diff --git a/packages/core/package.json b/packages/core/package.json index de3cb6d9..6746e666 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -9,6 +9,7 @@ "exports": { ".": { "types": "./dist/types/index.d.ts", + "development": "./src/index.ts", "bun": "./dist/bun/index.js", "default": "./dist/node/index.js" } diff --git a/packages/core/src/curator.ts b/packages/core/src/curator.ts index 4c474bd9..66bdd0bf 100644 --- a/packages/core/src/curator.ts +++ b/packages/core/src/curator.ts @@ -637,6 +637,7 @@ export async function run(input: { projectPath: string; sessionID: string; model?: { providerID: string; modelID: string }; + signal?: AbortSignal; /** Optional gateway worker-health hook — called when the LLM call returns * null. The gateway uses this to escalate to Sentry after sustained failure. */ workerHealth?: { @@ -707,7 +708,9 @@ export async function run(input: { export async function dedupePreferenceCreates( ops: CuratorOp[], projectPath: string, + signal?: AbortSignal, ): Promise { + signal?.throwIfAborted(); if (!embedding.isAvailable()) return ops; const pid = ensureProject(projectPath); // Track ids already matched this batch so two paraphrases in the SAME op list @@ -732,7 +735,9 @@ export async function dedupePreferenceCreates( projectId: op.scope === "global" ? null : pid, threshold: ltm.PREFERENCE_DEDUP_THRESHOLD, }); + signal?.throwIfAborted(); } catch (err) { + signal?.throwIfAborted(); log.warn( "preference dedup: findSemanticDuplicate failed (non-fatal):", err, @@ -778,6 +783,7 @@ async function runInner(input: { projectPath: string; sessionID: string; model?: { providerID: string; modelID: string }; + signal?: AbortSignal; workerHealth?: { recordFailure(reason: string): void; recordSuccess(): void; @@ -792,6 +798,7 @@ async function runInner(input: { relationsCreated: number; changedEntries: ChangedEntry[]; }> { + input.signal?.throwIfAborted(); const cfg = config(); // Get recent undistilled messages since last curation. @@ -887,7 +894,9 @@ async function runInner(input: { projectPath: input.projectPath, sessionID: input.sessionID, }); + input.signal?.throwIfAborted(); } catch (err) { + input.signal?.throwIfAborted(); log.warn("instruction-detect failed (non-fatal):", err); } @@ -949,7 +958,9 @@ async function runInner(input: { sessionID: input.sessionID, maxTokens: 2048, temperature: 0, + signal: input.signal, }); + input.signal?.throwIfAborted(); if (!responseText) { // Transport failure / empty completion already recorded by the LLM // adapter (single owner of transport-failure attribution) — avoid @@ -985,7 +996,12 @@ async function runInner(input: { // embedding similarity at a preference-specific (looser) threshold to redirect // a near-dup create onto the existing entry. Async embedding work lives here // (runInner is async) so applyOps can stay synchronous. - const ops = await dedupePreferenceCreates(response.ops, input.projectPath); + const ops = await dedupePreferenceCreates( + response.ops, + input.projectPath, + input.signal, + ); + input.signal?.throwIfAborted(); const result = applyOps(ops, { projectPath: input.projectPath, @@ -1003,7 +1019,9 @@ async function runInner(input: { // similarity when available, falls back to word-overlap. if (result.created > 0) { try { + input.signal?.throwIfAborted(); const dupes = await ltm.deduplicate(input.projectPath, { dryRun: false }); + input.signal?.throwIfAborted(); if (dupes.totalRemoved > 0) { log.info( `post-curation dedup: merged ${dupes.totalRemoved} duplicate entries`, @@ -1023,6 +1041,7 @@ async function runInner(input: { } } } catch (err) { + input.signal?.throwIfAborted(); log.warn("post-curation dedup failed (non-fatal):", err); } @@ -1034,6 +1053,7 @@ async function runInner(input: { // callers that don't pre-check). if (cfg.crossProject && embedding.isAvailable()) { try { + input.signal?.throwIfAborted(); const promotion = ltm.promoteCrossProject({ dryRun: false }); if (promotion.promoted > 0) { log.info( @@ -1041,6 +1061,7 @@ async function runInner(input: { ); } } catch (err) { + input.signal?.throwIfAborted(); log.warn("cross-project promotion failed (non-fatal):", err); } } @@ -1056,9 +1077,11 @@ async function runInner(input: { // alias-overlap signals still fire, and the next run catches the rest. if (result.entitiesCreated > 0 && embedding.isAvailable()) { try { + input.signal?.throwIfAborted(); const dupes = await entities.deduplicateEntities(input.projectPath, { dryRun: false, }); + input.signal?.throwIfAborted(); const autoMerged = dupes.merged.reduce((n, c) => n + c.merged.length, 0); if (autoMerged > 0) { log.info( @@ -1077,12 +1100,16 @@ async function runInner(input: { } } } catch (err) { + input.signal?.throwIfAborted(); log.warn("post-curation entity dedup failed (non-fatal):", err); } } - // Soft-cap enforcement: after creates and the dedup sweeps settle the count, - // evict the lowest-value project-scoped entries back down to maxEntries. + // The maintenance operations below are synchronous even though the dedup + // APIs return already-settled promises. Abort checks bracket every await and + // write phase so reset cannot resume a stale writer, while foreground runs + // still enforce the same duplicate, cap, and cursor invariants as idle runs. + input.signal?.throwIfAborted(); const evictedCount = enforceEntryCap( input.projectPath, cfg.curator.maxEntries, @@ -1094,6 +1121,7 @@ async function runInner(input: { ); } + input.signal?.throwIfAborted(); const now = Date.now(); lastCuratedAt.set(input.sessionID, now); saveSessionTracking(input.sessionID, { lastCuratedAt: now }); diff --git a/packages/core/src/data.ts b/packages/core/src/data.ts index 60f00e1e..81118fac 100644 --- a/packages/core/src/data.ts +++ b/packages/core/src/data.ts @@ -13,6 +13,7 @@ import { statSync, unlinkSync, existsSync, rmSync } from "node:fs"; import { resolve as resolvePath } from "node:path"; import { db, + databaseInTransaction, ensureProject, projectId, projectPath as getProjectPathById, @@ -156,10 +157,12 @@ onProjectMutation(() => { /** List all projects with summary counts. */ export function listProjects(): ProjectSummary[] { const now = Date.now(); - if (projectsCache && now - projectsCacheAt < LIST_CACHE_TTL_MS) { + const database = db(); + const cacheable = !databaseInTransaction(database); + if (cacheable && projectsCache && now - projectsCacheAt < LIST_CACHE_TTL_MS) { return projectsCache; } - const result = db() + const result = database .query( `SELECT p.id, p.path, p.name, p.git_remote, p.created_at, COALESCE(k.cnt, 0) AS knowledge_count, @@ -187,8 +190,10 @@ export function listProjects(): ProjectSummary[] { ORDER BY p.created_at DESC`, ) .all() as ProjectSummary[]; - projectsCache = result; - projectsCacheAt = now; + if (cacheable) { + projectsCache = result; + projectsCacheAt = now; + } return result; } @@ -881,6 +886,11 @@ export function deleteProject(projectId: string): ClearResult | null { database .query("DELETE FROM project_path_aliases WHERE project_id = ?") .run(projectId); + database + .query( + "DELETE FROM project_id_aliases WHERE retired_id = ? OR project_id = ?", + ) + .run(projectId, projectId); database .query("DELETE FROM warmup_histograms WHERE project_id = ?") .run(projectId); @@ -1604,6 +1614,13 @@ export type MergeResult = { * Returns counts of moved rows for reporting. */ export function mergeProjects(sourceId: string, targetId: string): MergeResult { + if (sourceId === targetId) { + return { + knowledge_moved: 0, + messages_moved: 0, + distillations_moved: 0, + }; + } const database = db(); // Count before merging (result.changes is inflated by FTS triggers) diff --git a/packages/core/src/db.ts b/packages/core/src/db.ts index c62d7975..062f9294 100644 --- a/packages/core/src/db.ts +++ b/packages/core/src/db.ts @@ -13,26 +13,40 @@ import { getGitRemote } from "./git"; import { isHostedMode } from "./hosted"; import { dataDir } from "./data-dir"; import { tracedDatabase } from "./db/traced"; +import { + decodeWarmupHistogram, + encodeWarmupHistogram, + mergeWarmupHistogramCounts, +} from "./warmup-histogram"; import { currentTenantId } from "./tenant"; -/** - * Callback fired when project rows are created or mutated (merge, rename, etc.). - * Used by data.ts to invalidate its listing caches without a circular import. - */ -let onProjectMutationCb: (() => void) | null = null; +export type ProjectMutation = + | { type: "create"; projectId: string } + | { + type: "merge"; + sourceId: string; + targetId: string; + sourcePath?: string; + }; + +/** Project mutation listeners, kept here to avoid higher-layer import cycles. */ +const projectMutationListeners = new Set<(mutation: ProjectMutation) => void>(); -/** Register a callback for project mutations. Only one callback is supported. */ -export function onProjectMutation(cb: () => void): void { - onProjectMutationCb = cb; +/** Register a callback for project mutations. */ +export function onProjectMutation( + cb: (mutation: ProjectMutation) => void, +): () => void { + projectMutationListeners.add(cb); + return () => projectMutationListeners.delete(cb); } /** Fire the project mutation callback (if registered). */ -function fireProjectMutation(): void { +function fireProjectMutation(mutation: ProjectMutation): void { // Project creation and merge (mergeProjectInternal) both route through here, // and a merge re-points a path to a different project id. Drop the memo so a // stale path→id can never survive a mutation. invalidateProjectIdCache(); - onProjectMutationCb?.(); + for (const listener of projectMutationListeners) listener(mutation); } /** @@ -88,6 +102,7 @@ export function fireProjectRemoteBackfilled(projectId: string): void { * branch is intentionally left uncached so lazy backfill keeps retrying. */ const projectIdByPathCache = new WeakMap>(); +const projectIdCacheDataVersion = new WeakMap(); function tenantPathKey(path: string): string { return `${currentTenantId()}\x1f${path}`; @@ -102,6 +117,75 @@ function projectIdCacheFor(conn: Database): Map { return m; } +function projectDataVersion(conn: Database): number { + const row = conn.query("PRAGMA data_version").get() as { + data_version: number; + } | null; + if (!row || !Number.isSafeInteger(row.data_version)) { + throw new Error("invalid SQLite data_version"); + } + return row.data_version; +} + +function cachedProjectId( + conn: Database, + path: string, + dataVersion: number, +): string | undefined { + const cache = projectIdCacheFor(conn); + const cachedVersion = projectIdCacheDataVersion.get(conn); + if (cachedVersion !== undefined && cachedVersion !== dataVersion) { + cache.clear(); + } + projectIdCacheDataVersion.set(conn, dataVersion); + return cache.get(path); +} + +/** True while either supported SQLite driver has an open transaction. */ +export function databaseInTransaction(conn: Database): boolean { + const transactionState = conn as Database & { + /** node:sqlite transaction-state getter. */ + isTransaction?: boolean; + /** bun:sqlite transaction-state getter. */ + inTransaction?: boolean; + }; + if (typeof transactionState.isTransaction === "boolean") { + return transactionState.isTransaction; + } + if (typeof transactionState.inTransaction === "boolean") { + return transactionState.inTransaction; + } + + // node:sqlite shipped before isTransaction (Node 22.5-22.15). Ask SQLite + // directly instead of maintaining a depth counter that can drift after raw + // SQL, failed commits, or SQLite's automatic rollback conditions. + try { + conn.exec("BEGIN DEFERRED; ROLLBACK"); + return false; + } catch (error) { + if ( + error instanceof Error && + /cannot start a transaction within a transaction/i.test(error.message) + ) { + return true; + } + throw error; + } +} + +/** Publish a stable path mapping only after it is outside rollback scope. */ +function memoizeProjectId( + conn: Database, + path: string, + projectID: string, + dataVersion: number, +): void { + if (!databaseInTransaction(conn)) { + projectIdCacheFor(conn).set(path, projectID); + projectIdCacheDataVersion.set(conn, dataVersion); + } +} + /** * Drop the memoized path→id map for the current connection. Called on every * project mutation (create/merge/backfill via the fire-hooks) and by data.ts @@ -130,7 +214,7 @@ export function repoNameFromRemote(remote: string | null): string | null { return name.length > 0 ? name : null; } -const MIGRATIONS: string[] = [ +export const MIGRATIONS: readonly string[] = Object.freeze([ ` -- Version 1: Initial schema @@ -1947,7 +2031,21 @@ const MIGRATIONS: string[] = [ // Version 82: tenant/session-safe temporal and tool-call identities. Applied // idempotently by applyTemporalIdentity() because tool_calls needs a PK rebuild. `-- Version 82: see applyTemporalIdentity — no-op SQL marker.`, -]; + // Version 83: persist session-scoped amnesia across eviction and restart. + // Local-only privacy state; never synchronized between devices. + `ALTER TABLE session_state ADD COLUMN amnesia INTEGER NOT NULL DEFAULT 0;`, + ` + -- Version 84: durable local redirects for retired project UUIDs. + -- Paths can be reused or repointed, so cross-process consumers need the + -- immutable source UUID to recover the live merge target safely. + CREATE TABLE IF NOT EXISTS project_id_aliases ( + retired_id TEXT PRIMARY KEY, + project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_project_id_aliases_project + ON project_id_aliases(project_id); + `, +]); // Index of the migration whose work is performed by a column-presence-aware JS // step instead of plain SQL, because it is destructive (DROP COLUMN) and its @@ -2816,6 +2914,8 @@ export function dbPath(): string { } let instance: Database | undefined; +/** Actual file backing `instance`; absent for private in-memory databases. */ +let instanceFilePath: string | undefined; export function db(): Database { if (instance) return instance; @@ -2917,6 +3017,13 @@ export function db(): Database { // LORE_NO_DB_TRACING=1 returns the raw connection instead of the query-tracing Proxy (disables automatic per-query DB spans). const dbTracingDisabled = process.env.LORE_NO_DB_TRACING === "1"; + const mainDatabase = database + .query("PRAGMA database_list") + .all() + .find((row) => row.name === "main") as + | { name: string; file: string } + | undefined; + instanceFilePath = mainDatabase?.file || undefined; instance = dbTracingDisabled ? database : tracedDatabase(database); return instance; } @@ -3970,6 +4077,17 @@ function recoverMissingObjects(database: Database) { `); } } + // Version 81: local retired-project UUID redirects. These are deliberately + // not synchronized; they repair process-local state after another local + // connection merges a project. + database.exec(` + CREATE TABLE IF NOT EXISTS project_id_aliases ( + retired_id TEXT PRIMARY KEY, + project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_project_id_aliases_project + ON project_id_aliases(project_id); + `); // Version 54: knowledge_session_injections.verdict (outcome impact, #497). // The verdict-keyed index MUST be created here, AFTER the column is ensured — // never in the big exec above, which runs before this ALTER and would throw @@ -4051,7 +4169,8 @@ function recoverMissingObjects(database: Database) { * ping-pong (a non-deterministic "local wins" would make the two devices merge in * opposite directions forever). mergeProjectInternal re-keys the content + re-enqueues it * under the winner. MUST run AFTER a pull (post-content, so the re-key finds the applied - * rows) and OUTSIDE any transaction (mergeProjectInternal opens its own BEGIN IMMEDIATE). + * rows). mergeProjectInternal is savepoint-backed, so callers may include the + * merge in a larger atomic write unit. */ export function convergeProjectsByRemote(): void { const tenantId = currentTenantId(); @@ -4073,20 +4192,169 @@ export function convergeProjectsByRemote(): void { } } -export function mergeProjectInternal(sourceId: string, targetId: string): void { - const d = db(); - const owners = d - .query("SELECT id, tenant_id FROM projects WHERE id IN (?, ?)") - .all(sourceId, targetId) as Array<{ id: string; tenant_id: string }>; +/** Every durable table whose rows must follow a project merge. */ +export const PROJECT_MERGE_TABLES = Object.freeze([ + "cache_bust_stats", + "dedup_feedback", + "distillation_vec", + "distillations", + "entities", + "import_history", + "knowledge", + "knowledge_contradictions", + "knowledge_session_injections", + "knowledge_tombstones", + "knowledge_transfers", + "lat_sections", + "project_id_aliases", + "project_path_aliases", + "session_prompt_deltas", + "session_rollup", + "temporal_messages", + "temporal_vec", + "tool_calls", + "warmup_histograms", +] as const); + +const SQLITE_MAX_INTEGER = "9223372036854775807"; + +function assertProjectMergeCountersSafe( + database: Database, + sourceId: string, + targetId: string, +): void { + const invalidCacheSource = database + .query( + `SELECT 1 FROM cache_bust_stats + WHERE project_id = ? + AND (typeof(turns) != 'integer' + OR typeof(write_tokens) != 'integer' + OR turns < 0 OR write_tokens < 0) + LIMIT 1`, + ) + .get(sourceId); + const invalidTransferSource = database + .query( + `SELECT 1 FROM knowledge_transfers + WHERE recalled_in_project_id = ? + AND (typeof(hit_count) != 'integer' OR hit_count < 0) + LIMIT 1`, + ) + .get(sourceId); + const cacheOverflow = database + .query( + `SELECT 1 + FROM cache_bust_stats AS source + JOIN cache_bust_stats AS target + ON target.project_id = ? + AND target.cause = source.cause + AND target.relocatable = source.relocatable + WHERE source.project_id = ? + AND (typeof(source.turns) != 'integer' + OR typeof(target.turns) != 'integer' + OR typeof(source.write_tokens) != 'integer' + OR typeof(target.write_tokens) != 'integer' + OR source.turns < 0 OR target.turns < 0 + OR source.write_tokens < 0 OR target.write_tokens < 0 + OR target.turns > ${SQLITE_MAX_INTEGER} - source.turns + OR target.write_tokens > ${SQLITE_MAX_INTEGER} - source.write_tokens) + LIMIT 1`, + ) + .get(targetId, sourceId); + const transferOverflow = database + .query( + `SELECT 1 + FROM knowledge_transfers AS source + JOIN knowledge_transfers AS target + ON target.recalled_in_project_id = ? + AND target.knowledge_id = source.knowledge_id + WHERE source.recalled_in_project_id = ? + AND (typeof(source.hit_count) != 'integer' + OR typeof(target.hit_count) != 'integer' + OR source.hit_count < 0 OR target.hit_count < 0 + OR target.hit_count > ${SQLITE_MAX_INTEGER} - source.hit_count) + LIMIT 1`, + ) + .get(targetId, sourceId); if ( - owners.length !== 2 || - owners[0].tenant_id !== owners[1].tenant_id || - owners[0].tenant_id !== currentTenantId() + invalidCacheSource || + invalidTransferSource || + cacheOverflow || + transferOverflow ) { - throw new Error("cannot merge projects across tenant boundaries"); + throw new Error("project merge counter overflow"); } - d.exec("BEGIN IMMEDIATE"); +} + +function assertRollupSourcesSafe( + database: Database, + projectId: string, + sessionId: string, +): void { + const invalidTemporal = database + .query( + `SELECT 1 FROM temporal_messages + WHERE project_id = ? AND session_id = ? AND tokens IS NOT NULL + AND (typeof(tokens) != 'integer' OR tokens < 0 + OR tokens > ${Number.MAX_SAFE_INTEGER}) + LIMIT 1`, + ) + .get(projectId, sessionId); + const invalidDistillation = database + .query( + `SELECT 1 FROM distillations + WHERE project_id = ? AND session_id = ? AND token_count IS NOT NULL + AND (typeof(token_count) != 'integer' OR token_count < 0 + OR token_count > ${Number.MAX_SAFE_INTEGER}) + LIMIT 1`, + ) + .get(projectId, sessionId); + const unsafeAggregate = database + .query( + `SELECT 1 + WHERE (SELECT TOTAL(tokens) FROM temporal_messages + WHERE project_id = ? AND session_id = ?) > ${Number.MAX_SAFE_INTEGER} + OR (SELECT TOTAL(token_count) FROM distillations + WHERE project_id = ? AND session_id = ?) > ${Number.MAX_SAFE_INTEGER} + LIMIT 1`, + ) + .get(projectId, sessionId, projectId, sessionId); + if (invalidTemporal || invalidDistillation || unsafeAggregate) { + throw new Error("project merge rollup counter invalid"); + } +} + +function parseSqliteInteger(value: string): bigint | null { + if (!/^(?:0|[1-9][0-9]*)$/.test(value)) return null; try { + return BigInt(value); + } catch { + return null; + } +} + +export function mergeProjectInternal(sourceId: string, targetId: string): void { + if (sourceId === targetId) return; + const d = db(); + let sourcePath: string | undefined; + let merged = false; + withSavepoint("merge_project", () => { + const sourceRow = d + .query("SELECT path, tenant_id FROM projects WHERE id = ?") + .get(sourceId) as { path: string; tenant_id: string } | null; + if (!sourceRow) return; + sourcePath = sourceRow.path; + const targetRow = d + .query("SELECT tenant_id FROM projects WHERE id = ?") + .get(targetId) as { tenant_id: string } | null; + if ( + !targetRow || + sourceRow.tenant_id !== targetRow.tenant_id || + sourceRow.tenant_id !== currentTenantId() + ) { + throw new Error("cannot merge projects across tenant boundaries"); + } + assertProjectMergeCountersSafe(d, sourceId, targetId); d.query("UPDATE knowledge SET project_id = ? WHERE project_id = ?").run( targetId, sourceId, @@ -4103,13 +4371,34 @@ export function mergeProjectInternal(sourceId: string, targetId: string): void { // ⇒ no session filter. Inside the transaction so a failure rolls the merge // back; no-op in blob mode. (knowledge_vec/entity_vec have no partition key.) repartitionVec0Project(d, sourceId, targetId); - // Re-point the session rollup set-based: the temporal/distillation project_id - // UPDATEs above do NOT fire the rollup triggers (scoped to content/tokens/ - // metadata + insert/delete), so move the rows here. session_id is globally - // unique ⇒ no (target, session_id) PK collision. + // Re-point rollups set-based, but rebuild collisions from the authoritative + // temporal/distillation rows after those rows have moved above. + const rollupCollisions = d + .query( + `SELECT target.session_id + FROM session_rollup AS target + JOIN session_rollup AS source + ON source.project_id = ? + AND source.session_id = target.session_id + WHERE target.project_id = ?`, + ) + .all(sourceId, targetId) as Array<{ session_id: string }>; + d.query( + `DELETE FROM session_rollup AS source + WHERE source.project_id = ? + AND EXISTS ( + SELECT 1 FROM session_rollup AS target + WHERE target.project_id = ? + AND target.session_id = source.session_id + )`, + ).run(sourceId, targetId); d.query( "UPDATE session_rollup SET project_id = ? WHERE project_id = ?", ).run(targetId, sourceId); + for (const collision of rollupCollisions) { + assertRollupSourcesSafe(d, targetId, collision.session_id); + recomputeSessionRollupRow(d, targetId, collision.session_id); + } d.query("UPDATE lat_sections SET project_id = ? WHERE project_id = ?").run( targetId, sourceId, @@ -4122,6 +4411,154 @@ export function mergeProjectInternal(sourceId: string, targetId: string): void { targetId, sourceId, ); + // Prompt deltas are globally keyed by (session_id, seq), so project_id is + // not part of their identity and can be re-keyed directly. + d.query( + "UPDATE session_prompt_deltas SET project_id = ? WHERE project_id = ?", + ).run(targetId, sourceId); + // Import identity is (project, agent, source). Preserve the newest record + // when both projects imported the same source, then re-key the remainder. + d.query( + `DELETE FROM import_history + WHERE project_id = ? + AND EXISTS ( + SELECT 1 FROM import_history AS target + WHERE target.project_id = ? + AND target.agent_name = import_history.agent_name + AND target.source_id = import_history.source_id + AND target.imported_at >= import_history.imported_at + )`, + ).run(sourceId, targetId); + d.query( + `DELETE FROM import_history + WHERE project_id = ? + AND EXISTS ( + SELECT 1 FROM import_history AS source + WHERE source.project_id = ? + AND source.agent_name = import_history.agent_name + AND source.source_id = import_history.source_id + AND source.imported_at > import_history.imported_at + )`, + ).run(targetId, sourceId); + d.query( + "UPDATE import_history SET project_id = ? WHERE project_id = ?", + ).run(targetId, sourceId); + // Histogram rows collide by time slot. Sum valid count vectors and totals; + // if either persisted row is malformed, retain the newer whole row. + const sourceHistograms = d + .query( + `SELECT time_slot, counts, CAST(total AS TEXT) AS total, + CAST(updated_at AS TEXT) AS updated_at + FROM warmup_histograms WHERE project_id = ?`, + ) + .all(sourceId) as Array<{ + time_slot: string; + counts: string; + total: string; + updated_at: string; + }>; + for (const source of sourceHistograms) { + const target = d + .query( + `SELECT counts, CAST(total AS TEXT) AS total, + CAST(updated_at AS TEXT) AS updated_at + FROM warmup_histograms WHERE project_id = ? AND time_slot = ?`, + ) + .get(targetId, source.time_slot) as { + counts: string; + total: string; + updated_at: string; + } | null; + if (!target) continue; + let counts = target.counts; + let total: number | undefined; + const sourceUpdatedAt = parseSqliteInteger(source.updated_at); + const targetUpdatedAt = parseSqliteInteger(target.updated_at); + const sourceIsNewer = + sourceUpdatedAt !== null && + (targetUpdatedAt === null || sourceUpdatedAt > targetUpdatedAt); + try { + const sourceValues = decodeWarmupHistogram(source.counts, source.total); + const targetValues = decodeWarmupHistogram(target.counts, target.total); + if (!sourceValues || !targetValues) { + throw new Error("invalid histogram counts"); + } + const encoded = encodeWarmupHistogram( + mergeWarmupHistogramCounts(targetValues, sourceValues), + ); + counts = encoded.counts; + total = encoded.total; + } catch { + if (sourceIsNewer) { + d.query( + `UPDATE warmup_histograms + SET counts = (SELECT counts FROM warmup_histograms + WHERE project_id = ? AND time_slot = ?), + total = (SELECT total FROM warmup_histograms + WHERE project_id = ? AND time_slot = ?), + updated_at = (SELECT updated_at FROM warmup_histograms + WHERE project_id = ? AND time_slot = ?) + WHERE project_id = ? AND time_slot = ?`, + ).run( + sourceId, + source.time_slot, + sourceId, + source.time_slot, + sourceId, + source.time_slot, + targetId, + source.time_slot, + ); + } + d.query( + "DELETE FROM warmup_histograms WHERE project_id = ? AND time_slot = ?", + ).run(sourceId, source.time_slot); + continue; + } + d.query( + `UPDATE warmup_histograms + SET counts = ?, total = ?, + updated_at = MAX( + updated_at, + (SELECT updated_at FROM warmup_histograms + WHERE project_id = ? AND time_slot = ?) + ) + WHERE project_id = ? AND time_slot = ?`, + ).run( + counts, + total, + sourceId, + source.time_slot, + targetId, + source.time_slot, + ); + d.query( + "DELETE FROM warmup_histograms WHERE project_id = ? AND time_slot = ?", + ).run(sourceId, source.time_slot); + } + d.query( + "UPDATE warmup_histograms SET project_id = ? WHERE project_id = ?", + ).run(targetId, sourceId); + d.query( + "UPDATE dedup_feedback SET project_id = ? WHERE project_id = ?", + ).run(targetId, sourceId); + d.query( + "UPDATE knowledge_tombstones SET project_id = ? WHERE project_id = ?", + ).run(targetId, sourceId); + d.query( + `INSERT INTO cache_bust_stats + (project_id, cause, relocatable, turns, write_tokens, updated_at) + SELECT ?, cause, relocatable, turns, write_tokens, updated_at + FROM cache_bust_stats WHERE project_id = ? + ON CONFLICT(project_id, cause, relocatable) DO UPDATE SET + turns = cache_bust_stats.turns + excluded.turns, + write_tokens = cache_bust_stats.write_tokens + excluded.write_tokens, + updated_at = MAX(cache_bust_stats.updated_at, excluded.updated_at)`, + ).run(targetId, sourceId); + d.query("DELETE FROM cache_bust_stats WHERE project_id = ?").run(sourceId); + d.query( + "UPDATE knowledge_contradictions SET project_id = ? WHERE project_id = ?", + ).run(targetId, sourceId); // Outcome-reward injection log (#497): carries a project_id that must follow // the entries to the target, or its rows orphan once the source project row // is deleted below AND crediting breaks (creditSessionOutcome queries by the @@ -4157,25 +4594,34 @@ export function mergeProjectInternal(sourceId: string, targetId: string): void { ).run(targetId, targetId); // entity_relations references entities by FK — no project_id column to update. // Relations move implicitly when their parent entities move. + // Flatten every prior redirect to the new live winner before retiring this + // source UUID. This makes transitive merge resolution a single indexed read. + d.query("DELETE FROM project_id_aliases WHERE retired_id = ?").run( + targetId, + ); + d.query( + "UPDATE project_id_aliases SET project_id = ? WHERE project_id = ?", + ).run(targetId, sourceId); d.query( - "UPDATE OR IGNORE project_path_aliases SET project_id = ? WHERE project_id = ?", + `INSERT INTO project_id_aliases (retired_id, project_id) VALUES (?, ?) + ON CONFLICT(retired_id) DO UPDATE SET project_id = excluded.project_id`, + ).run(sourceId, targetId); + d.query( + "UPDATE project_path_aliases SET project_id = ? WHERE project_id = ?", ).run(targetId, sourceId); // Register source's path as alias of target - const sourceRow = d - .query("SELECT path, tenant_id FROM projects WHERE id = ?") - .get(sourceId) as { path: string; tenant_id: string } | null; - if (sourceRow) { - d.query( - "INSERT OR IGNORE INTO project_path_aliases (tenant_id, path, project_id) VALUES (?, ?, ?)", - ).run(sourceRow.tenant_id, sourceRow.path, targetId); - } + d.query( + `INSERT INTO project_path_aliases (tenant_id, path, project_id) + VALUES (?, ?, ?) + ON CONFLICT(tenant_id, path) DO UPDATE SET project_id = excluded.project_id`, + ).run(sourceRow.tenant_id, sourceRow.path, targetId); d.query("DELETE FROM projects WHERE id = ?").run(sourceId); - d.exec("COMMIT"); - fireProjectMutation(); - } catch (e) { - d.exec("ROLLBACK"); - throw e; - } + merged = true; + }); + if (!merged) return; + // Invalidate after all merge statements have succeeded. If an outer + // transaction later rolls back, conservative cache invalidation is harmless. + fireProjectMutation({ type: "merge", sourceId, targetId, sourcePath }); } export function close() { @@ -4193,6 +4639,7 @@ export function close() { instance.close(); instance = undefined; } + instanceFilePath = undefined; // The sqlite-vec extension is loaded per-connection; reset loader state so a // subsequent db() on a fresh connection re-attempts the load. This also clears // the sticky vec0 storage-mode latch (a fresh connection may point at a @@ -4589,10 +5036,11 @@ export function ensureProject( // 0. Memoized fast path — the same session path is resolved many times per // request (see projectIdByPathCache docs / LOREAI-GATEWAY-3K). + const connection = db(); + const dataVersion = projectDataVersion(connection); const tenantId = currentTenantId(); const cacheKey = tenantPathKey(path); - const cache = projectIdCacheFor(db()); - const cached = cache.get(cacheKey); + const cached = cachedProjectId(connection, cacheKey, dataVersion); if (cached !== undefined) return cached; // 1. Exact path match (fast path) @@ -4629,7 +5077,7 @@ export function ensureProject( // above (it is the merge TARGET) — memoize so the next call for this // path skips the exact-path lookup. fireProjectRemoteBackfilled cleared // the map, so this set must come AFTER it. - cache.set(cacheKey, existing.id); + memoizeProjectId(connection, cacheKey, existing.id, dataVersion); return existing.id; } // Still remote-less (no remote resolved) — leave uncached so a later call @@ -4637,7 +5085,7 @@ export function ensureProject( return existing.id; } // Settled remote-backed row — stable mapping, safe to memoize. - cache.set(cacheKey, existing.id); + memoizeProjectId(connection, cacheKey, existing.id, dataVersion); return existing.id; } @@ -4648,7 +5096,7 @@ export function ensureProject( ) .get(tenantId, path) as { project_id: string } | null; if (alias) { - cache.set(cacheKey, alias.project_id); + memoizeProjectId(connection, cacheKey, alias.project_id, dataVersion); return alias.project_id; } @@ -4667,7 +5115,7 @@ export function ensureProject( "INSERT OR IGNORE INTO project_path_aliases (tenant_id, path, project_id) VALUES (?, ?, ?)", ) .run(tenantId, path, byRemote.id); - cache.set(cacheKey, byRemote.id); + memoizeProjectId(connection, cacheKey, byRemote.id, dataVersion); return byRemote.id; } } @@ -4692,17 +5140,71 @@ export function ensureProject( // already settled (has a remote): a remote-less project can still be // git_remote-backfilled by a later ensureProject(path, suppliedGitRemote) // call, so leave it uncached (mirrors the NULL-git_remote existing branch). - fireProjectMutation(); - if (gitRemote) cache.set(cacheKey, id); + fireProjectMutation({ type: "create", projectId: id }); + if (gitRemote) memoizeProjectId(connection, cacheKey, id, dataVersion); return id; } +function canonicalProjectIdFrom( + connection: Database, + id: string, +): string | undefined { + const tenantId = currentTenantId(); + const row = connection + .query( + `SELECT alias.project_id AS id + FROM project_id_aliases AS alias + JOIN projects AS target ON target.id = alias.project_id + WHERE alias.retired_id = ? AND target.tenant_id = ? + UNION ALL + SELECT id FROM projects WHERE id = ? AND tenant_id = ? + LIMIT 1`, + ) + .get(id, tenantId, id, tenantId) as { id: string } | null; + return row?.id; +} + +/** + * Resolve a live project UUID or a durable redirect left by a local merge. + * A committed read inside a private in-memory transaction returns undefined: + * SQLite cannot expose that connection's pre-transaction snapshot to a reader. + */ +export function canonicalProjectId( + id: string, + options?: { committed?: boolean }, +): string | undefined { + const connection = db(); + if (!options?.committed || !databaseInTransaction(connection)) { + return canonicalProjectIdFrom(connection, id); + } + + // The writer sees its own uncommitted redirects. A short-lived WAL reader + // observes only committed state, so read-only caches cannot publish a merge + // that an outer savepoint may still roll back. Use the file captured from the + // live connection rather than dbPath(): LORE_DB_PATH is mutable, and changing + // it must not redirect a reader away from the already-open writer. + // + // Private in-memory databases have no second connection that can observe the + // writer's last committed snapshot. Fail closed instead of exposing its + // speculative redirects or opening a distinct, empty `:memory:` database. + if (!instanceFilePath) return undefined; + const reader = new Database(instanceFilePath); + try { + reader.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`); + reader.exec("PRAGMA query_only = TRUE"); + return canonicalProjectIdFrom(reader, id); + } finally { + reader.close(); + } +} + export function projectId(path: string): string | undefined { const tenantId = currentTenantId(); const cacheKey = tenantPathKey(path); // Shares ensureProject's per-connection memo (LOREAI-GATEWAY-3K). - const cache = projectIdCacheFor(db()); - const cached = cache.get(cacheKey); + const connection = db(); + const dataVersion = projectDataVersion(connection); + const cached = cachedProjectId(connection, cacheKey, dataVersion); if (cached !== undefined) return cached; const row = db() @@ -4714,7 +5216,9 @@ export function projectId(path: string): string | undefined { // Mirror ensureProject: only memoize a settled (remote-backed) exact-path // row so a NULL-git_remote project still gets its lazy backfill retried // there. An unsettled row is returned but left uncached. - if (row.git_remote) cache.set(cacheKey, row.id); + if (row.git_remote) { + memoizeProjectId(connection, cacheKey, row.id, dataVersion); + } return row.id; } @@ -4726,7 +5230,7 @@ export function projectId(path: string): string | undefined { ) .get(tenantId, path) as { project_id: string } | null; if (alias) { - cache.set(cacheKey, alias.project_id); + memoizeProjectId(connection, cacheKey, alias.project_id, dataVersion); return alias.project_id; } return undefined; @@ -5359,6 +5863,8 @@ export type SessionTrackingState = { projectPathProvisional?: boolean; // v37: compaction anomaly pending flag compactionAnomalyPending?: boolean; + // v80: session-scoped privacy mode + amnesia?: boolean; }; /** @@ -5520,6 +6026,10 @@ export function saveSessionTracking( sets.push("compaction_anomaly_pending = ?"); vals.push(state.compactionAnomalyPending ? 1 : 0); } + if (state.amnesia !== undefined) { + sets.push("amnesia = ?"); + vals.push(state.amnesia ? 1 : 0); + } // Update only the specified columns db() .query(`UPDATE session_state SET ${sets.join(", ")} WHERE session_id = ?`) @@ -5571,6 +6081,8 @@ export type LoadedSessionTracking = { projectPathProvisional: boolean; // v37: compaction anomaly pending flag compactionAnomalyPending: boolean; + // v80: session-scoped privacy mode + amnesia: boolean; }; export type SessionPromptDelta = { @@ -5898,7 +6410,7 @@ export function loadSessionTracking( last_turn_at, last_bust_at, parent_session_id, is_subagent, project_path, project_path_provisional, - compaction_anomaly_pending + compaction_anomaly_pending, amnesia FROM session_state WHERE session_id = ?`, ) .get(sessionID) as { @@ -5935,6 +6447,7 @@ export function loadSessionTracking( project_path: string | null; project_path_provisional: number; compaction_anomaly_pending: number; + amnesia: number; } | null; if (!row) return null; return { @@ -5971,6 +6484,7 @@ export function loadSessionTracking( projectPath: row.project_path, projectPathProvisional: row.project_path_provisional === 1, compactionAnomalyPending: row.compaction_anomaly_pending === 1, + amnesia: row.amnesia === 1, }; } @@ -5986,20 +6500,41 @@ export function loadSessionTracking( */ export function findSessionStatesByFingerprint( fingerprint: string, - options?: { legacyUnownedOnly?: boolean }, -): Array<{ session_id: string; message_count: number; is_subagent: number }> { + options?: { + legacyUnownedOnly?: boolean; + credentialFingerprint?: string; + }, +): Array<{ + session_id: string; + message_count: number; + is_subagent: number; + project_path: string | null; + project_path_provisional: number; +}> { if (!fingerprint) return []; + const ownerFilter = options?.legacyUnownedOnly + ? "AND credential_fingerprint = ''" + : options?.credentialFingerprint !== undefined + ? "AND credential_fingerprint = ?" + : ""; + const params = + options?.credentialFingerprint !== undefined && !options.legacyUnownedOnly + ? [fingerprint, options.credentialFingerprint] + : [fingerprint]; return db() .query( - `SELECT session_id, message_count, is_subagent + `SELECT session_id, message_count, is_subagent, + project_path, project_path_provisional FROM session_state - WHERE fingerprint = ? AND fingerprint != '' - ${options?.legacyUnownedOnly ? "AND credential_fingerprint = ''" : ""}`, + WHERE fingerprint = ? AND fingerprint != '' + ${ownerFilter}`, ) - .all(fingerprint) as Array<{ + .all(...params) as Array<{ session_id: string; message_count: number; is_subagent: number; + project_path: string | null; + project_path_provisional: number; }>; } diff --git a/packages/core/src/distillation.ts b/packages/core/src/distillation.ts index 22062b70..33938736 100644 --- a/packages/core/src/distillation.ts +++ b/packages/core/src/distillation.ts @@ -1307,18 +1307,22 @@ async function distillSegment(input: { sessionID: input.sessionID, llm: input.llm, model: input.model, + signal: input.signal, // #627 Phase 1: propagate gitHead so echo-extracted preferences also // get stamped. metadata: input.metadata, }); - if (input.urgent) await echoPromise; - else trackBackground(echoPromise); + if (input.urgent) { + await echoPromise; + input.signal?.throwIfAborted(); + } else trackBackground(echoPromise); } else if (embedding.isAvailable()) { embedding.embedDistillation(distillId, result.observations); } // Fire-and-forget: extract decision/preference patterns → knowledge entries if (config().knowledge.enabled) { + input.signal?.throwIfAborted(); const patterns = extractPatterns(result.observations); for (const pat of patterns) { try { diff --git a/packages/core/src/gradient.ts b/packages/core/src/gradient.ts index 821adf64..30841c7b 100644 --- a/packages/core/src/gradient.ts +++ b/packages/core/src/gradient.ts @@ -2198,7 +2198,9 @@ export async function prewarmDistillationSnapshot( projectPath: string, sessionID: string | undefined, messages: MessageWithParts[], + signal?: AbortSignal, ): Promise { + signal?.throwIfAborted(); if (!sessionID) return; const sessState = getSessionState(sessionID); const lastUserMsgId = lastUserMessageId(messages); @@ -2209,6 +2211,7 @@ export async function prewarmDistillationSnapshot( if (snapshot && snapshot.lastUserMsgId === lastUserMsgId) return; const rows = await loadDistillationsOffloaded(projectPath, sessionID); + signal?.throwIfAborted(); if (rows === null) return; // worker timeout → let transform() do the sync load sessState.distillationSnapshot = { rows, lastUserMsgId }; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 223cd1ba..435c2ac2 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -89,6 +89,15 @@ export type { LLMClient, } from "./types"; export { isTextPart, isReasoningPart, isToolPart } from "./types"; +export { + WARMUP_HISTOGRAM_BIN_COUNT, + MAX_WARMUP_HISTOGRAM_TOTAL, + emptyWarmupHistogramCounts, + mergeWarmupHistogramCounts, + normalizeWarmupHistogram, + encodeWarmupHistogram, + decodeWarmupHistogram, +} from "./warmup-histogram"; export { dataDir } from "./data-dir"; export { currentTenantId, withTenant, LOCAL_TENANT_ID } from "./tenant"; @@ -111,6 +120,7 @@ export { setLastImportAt, isFirstRun, projectId, + canonicalProjectId, projectName, projectPath, projectKnownPaths, @@ -123,6 +133,9 @@ export { resolveWritableScope, scopeMemberRole, resolveProjectByRemoteOrPath, + onProjectMutation, + databaseInTransaction, + type ProjectMutation, mergeProjectInternal, convergeProjectsByRemote, UNATTRIBUTED_PROJECT_PREFIX, diff --git a/packages/core/src/ltm.ts b/packages/core/src/ltm.ts index 89fbd757..cb3d9fde 100644 --- a/packages/core/src/ltm.ts +++ b/packages/core/src/ltm.ts @@ -1615,7 +1615,9 @@ export async function validateProjectReferences( projectPath: string, resolver: ReferenceResolver, now: number = Date.now(), + signal?: AbortSignal, ): Promise { + signal?.throwIfAborted(); const pid = ensureProject(projectPath); const proj = db() .query("SELECT last_refcheck_at FROM projects WHERE id = ?") @@ -1662,6 +1664,7 @@ export async function validateProjectReferences( } const statusMap = await resolver.resolve(unionList); + signal?.throwIfAborted(); if (statusMap == null) { // Whole batch unverifiable (probe error/timeout / no FS) → strict no-op. // Consume the gate so a failing probe isn't re-hammered every idle tick. @@ -2476,6 +2479,8 @@ export const CONTEXT_SOURCE_LIMIT = 12; export const RECALLED_CONTEXT_CATEGORY = "recalled"; export type ForSessionOptions = { + /** Abort stale request work before reinforcement/transfer side effects. */ + signal?: AbortSignal; /** Caller-provided context (e.g., user's current message) for relevance * scoring when no session context exists in the DB yet. */ contextHint?: string; @@ -2565,6 +2570,7 @@ export async function forSession( maxTokens: number, options?: ForSessionOptions, ): Promise { + options?.signal?.throwIfAborted(); // Measure this hot per-turn path's main-thread blocking cost (#966 B). The // awaits below (embed + the pool-backed vector search) are wrapped so the // wall-time remainder is the synchronous entry-load / FTS / scoring / packing @@ -2717,10 +2723,12 @@ export async function forSession( // preference injected into system[1] every turn would still age out and be // pruned by decayProject/pruneDeadEntries after the grace window — silently // deleting an actively-used directive. Resets the decay clock only. + options?.signal?.throwIfAborted(); try { markInjected(result.map((e) => e.id)); recordSessionInjections(sessionID, projectPath, result); } catch (err) { + options?.signal?.throwIfAborted(); log.warn( "forSession(preference): reinforcement failed (non-fatal):", err, @@ -2784,6 +2792,7 @@ export async function forSession( ); vectorScores = new Map(hits.map((h) => [h.id, h.similarity])); } catch (err) { + options?.signal?.throwIfAborted(); log.warn("Vector scoring failed, falling back to FTS5:", err); vectorScores = new Map(); } @@ -3017,6 +3026,7 @@ export async function forSession( // (project_id === pid) are not transfers; lat.md synthetics are skipped (they // are not knowledge rows). The in-memory throttle bounds writes so this // every-message path does not hammer SQLite. + options?.signal?.throwIfAborted(); try { for (const entry of result) { if (entry.category === "lat.md") continue; @@ -3030,6 +3040,7 @@ export async function forSession( }); } } catch (err) { + options?.signal?.throwIfAborted(); log.warn("forSession: transfer recording failed (non-fatal):", err); } @@ -3037,6 +3048,7 @@ export async function forSession( // Being selected for the prompt resets each entry's decay clock — it is "still // relevant" — WITHOUT bumping confidence (that would re-flatten everything to // 1.0 and destroy the decay signal). lat.md synthetics are not knowledge rows. + options?.signal?.throwIfAborted(); try { // Only real knowledge rows get reinforced / recorded — lat.md and // recalled-context synthetics are not knowledge and have no confidence @@ -3048,6 +3060,7 @@ export async function forSession( markInjected(knowledgeResult.map((e) => e.id)); recordSessionInjections(sessionID, projectPath, knowledgeResult); } catch (err) { + options?.signal?.throwIfAborted(); log.warn("forSession: reinforcement failed (non-fatal):", err); } @@ -3059,6 +3072,7 @@ export async function forSession( // (lat.md synthetics are packed separately in step 6), so this never leaks a // lat.md row, which has no recall id. if (options?.overflowSink) { + options.signal?.throwIfAborted(); const selectedIds = new Set(result.map((e) => e.id)); for (const { entry } of allScored) { // Recalled-context synthetics are not knowledge rows — keep the ToC diff --git a/packages/core/src/pattern-echo.ts b/packages/core/src/pattern-echo.ts index 3005074a..6719a9be 100644 --- a/packages/core/src/pattern-echo.ts +++ b/packages/core/src/pattern-echo.ts @@ -65,7 +65,7 @@ export const PATTERN_COOLDOWN_MS = 10 * 60 * 1000; // Rate limit state // --------------------------------------------------------------------------- -const lastExtraction = new Map(); +const lastExtraction = new Map(); /** Test seam: clear the per-session cooldown state so suites don't leak the * module-global map across cases. Never called in production. */ @@ -95,10 +95,18 @@ export function detectPatternEchoes(input: { sessionID: string; llm: LLMClient; model?: { providerID: string; modelID: string }; + signal?: AbortSignal; /** Per-entry metadata stamped on every echo entry minted (#627 Phase 1). */ metadata?: KnowledgeMetadata; }): Promise { - const p = _detect(input).catch((err) => { + const cooldownOwner = Symbol(input.sessionID); + const p = _detect({ ...input, cooldownOwner }).catch((err) => { + if (input.signal?.aborted) { + if (lastExtraction.get(input.sessionID)?.owner === cooldownOwner) { + lastExtraction.delete(input.sessionID); + } + return; + } log.error("pattern echo detection failed:", err); }); return p; @@ -115,7 +123,9 @@ async function _detect(input: { sessionID: string; llm: LLMClient; model?: { providerID: string; modelID: string }; + signal?: AbortSignal; metadata?: KnowledgeMetadata; + cooldownOwner: symbol; }): Promise { // Step 1: Embed the new distillation and store it. This is the // embedDistillation() replacement at the gen-0 hook (see distillation.ts), so @@ -124,6 +134,7 @@ async function _detect(input: { // rate-limit check sat above this, so a segment arriving within the cooldown // got no embedding stored at all: a latent recall gap.) const [vec] = await embedding.embed([input.observations], "document"); + input.signal?.throwIfAborted(); storeEmbedding(db(), "distillations", input.distillId, vec); // Rate limit the EXPENSIVE pattern detection that follows (project-wide vector @@ -133,17 +144,22 @@ async function _detect(input: { // ltm.create() (below), so the common "no pattern this time" outcome never // armed it and the full search + cluster ran on every gen-0 distillation. const now = Date.now(); - const lastTime = lastExtraction.get(input.sessionID) ?? 0; + const lastTime = lastExtraction.get(input.sessionID)?.timestamp ?? 0; if (now - lastTime < PATTERN_COOLDOWN_MS) return; // Arm the cooldown, and opportunistically evict entries that have already aged // out of the window. A stale entry is a no-op for the check above, so eviction // is behavior-neutral — it just keeps this per-session map from growing without // bound now that we write one entry per distilling session (not only per // pattern created). This runs at most once per session per cooldown. - for (const [sid, ts] of lastExtraction) { - if (now - ts >= PATTERN_COOLDOWN_MS) lastExtraction.delete(sid); + for (const [sid, state] of lastExtraction) { + if (now - state.timestamp >= PATTERN_COOLDOWN_MS) { + lastExtraction.delete(sid); + } } - lastExtraction.set(input.sessionID, now); + lastExtraction.set(input.sessionID, { + timestamp: now, + owner: input.cooldownOwner, + }); // Step 2: Search for similar distillations across the project (wide net) const pid = ensureProject(input.projectPath); @@ -152,6 +168,7 @@ async function _detect(input: { pid, MAX_CANDIDATES, ); + input.signal?.throwIfAborted(); // Step 3: Filter candidates — above lower threshold, exclude self const candidates = hits.filter( @@ -209,8 +226,10 @@ async function _detect(input: { sessionID: input.sessionID, maxTokens: 512, temperature: 0, + signal: input.signal, }, ); + input.signal?.throwIfAborted(); if (!responseText) return; @@ -243,6 +262,7 @@ async function _detect(input: { content: pattern.content, projectId: pid, }); + input.signal?.throwIfAborted(); if (semanticDup) { log.info( `pattern echo: skipping near-duplicate (sim=${semanticDup.similarity.toFixed(3)}): "${pattern.title}"`, diff --git a/packages/core/src/recall.ts b/packages/core/src/recall.ts index 7260774b..316e5c68 100644 --- a/packages/core/src/recall.ts +++ b/packages/core/src/recall.ts @@ -85,6 +85,13 @@ export type RecallInput = { * don't inflate counts. Default false. */ recordTransfers?: boolean; + /** + * Defer transfer-counter writes until the caller commits a larger persistence + * unit. When omitted, transfers retain their existing immediate-write + * behavior. The callback is synchronous so callers can run it inside their + * own SQLite savepoint. + */ + deferTransferRecording?: (record: () => void) => void; /** * Set of knowledge entry IDs (`019fxxxx-...` UUIDs — the canonical form * without the `k:` prefix) that are already present in the model's visible @@ -1753,27 +1760,34 @@ export async function runRecall(input: RecallInput): Promise { // - "knowledge": promoted cross_project=1 entries whose project_id is a // DIFFERENT non-null project than the current one. if (input.recordTransfers) { - try { - const pid = ensureProject(input.projectPath); - for (const { item: tagged } of fused) { - if ( - tagged.source !== "cross-knowledge" && - tagged.source !== "knowledge" - ) - continue; - const entry = tagged.item; - // For same-source knowledge results, only promoted cross-project entries - // from another project count as transfers. - if (tagged.source === "knowledge" && entry.cross_project !== 1) - continue; - if (!entry.project_id || entry.project_id === pid) continue; - ltm.recordTransfer({ - knowledgeId: entry.logical_id, - recalledInProjectId: pid, - }); + const recordTransfers = (): void => { + try { + const pid = ensureProject(input.projectPath); + for (const { item: tagged } of fused) { + if ( + tagged.source !== "cross-knowledge" && + tagged.source !== "knowledge" + ) + continue; + const entry = tagged.item; + // For same-source knowledge results, only promoted cross-project entries + // from another project count as transfers. + if (tagged.source === "knowledge" && entry.cross_project !== 1) + continue; + if (!entry.project_id || entry.project_id === pid) continue; + ltm.recordTransfer({ + knowledgeId: entry.logical_id, + recalledInProjectId: pid, + }); + } + } catch (err) { + log.warn("recall: transfer recording failed (non-fatal):", err); } - } catch (err) { - log.warn("recall: transfer recording failed (non-fatal):", err); + }; + if (input.deferTransferRecording) { + input.deferTransferRecording(recordTransfers); + } else { + recordTransfers(); } } diff --git a/packages/core/src/warmup-histogram.ts b/packages/core/src/warmup-histogram.ts new file mode 100644 index 00000000..4712b9e2 --- /dev/null +++ b/packages/core/src/warmup-histogram.ts @@ -0,0 +1,245 @@ +export const WARMUP_HISTOGRAM_BIN_COUNT = 21; +export const MAX_WARMUP_HISTOGRAM_TOTAL = Math.floor( + Number.MAX_SAFE_INTEGER / 2, +); +const MAX_STORED_HISTOGRAM_CHARS = 64 * 1024; +const COMPACTED_EXACT_TOTAL = 1_000_000_000_000n; + +/** Overflow rows retain exact weights so later merges remain associative. */ +interface ExactWarmupHistogram { + v: 1; + counts: number[]; + exact: string[]; +} + +/** Oversized rows retain their decimal magnitude with compacted proportions. */ +interface CompactedWarmupHistogram { + v: 2; + counts: number[]; + exact: string[]; + scale: number; +} + +function totalCounts(counts: readonly bigint[]): bigint { + return counts.reduce((sum, count) => sum + count, 0n); +} + +function storedTotalMatches(value: unknown, expected: bigint): boolean { + if (typeof value === "bigint") return value === expected; + if (typeof value === "number") { + return Number.isSafeInteger(value) && BigInt(value) === expected; + } + return typeof value === "string" && value === expected.toString(); +} + +function parseLegacyCounts(value: unknown): bigint[] | undefined { + if ( + !Array.isArray(value) || + value.length !== WARMUP_HISTOGRAM_BIN_COUNT || + !value.every((count) => Number.isSafeInteger(count) && count >= 0) + ) { + return undefined; + } + return value.map((count) => BigInt(count)); +} + +function parseExactCounts(value: unknown): bigint[] | undefined { + if ( + !Array.isArray(value) || + value.length !== WARMUP_HISTOGRAM_BIN_COUNT || + !value.every( + (count) => typeof count === "string" && /^(?:0|[1-9][0-9]*)$/.test(count), + ) + ) { + return undefined; + } + return value.map((count) => BigInt(count)); +} + +export function emptyWarmupHistogramCounts(): bigint[] { + return Array.from({ length: WARMUP_HISTOGRAM_BIN_COUNT }, () => 0n); +} + +export function mergeWarmupHistogramCounts( + target: readonly bigint[], + source: readonly bigint[], +): bigint[] { + if ( + target.length !== WARMUP_HISTOGRAM_BIN_COUNT || + source.length !== WARMUP_HISTOGRAM_BIN_COUNT + ) { + throw new Error("invalid warmup histogram length"); + } + return target.map((count, index) => count + source[index]); +} + +function scaleWarmupHistogram( + counts: readonly bigint[], + limit: bigint, +): bigint[] { + if ( + counts.length !== WARMUP_HISTOGRAM_BIN_COUNT || + counts.some((count) => count < 0n) || + limit < BigInt(WARMUP_HISTOGRAM_BIN_COUNT) + ) { + throw new Error("invalid warmup histogram counts"); + } + const total = totalCounts(counts); + if (total <= limit) return [...counts]; + + const positive = counts.reduce( + (count, value) => count + (value > 0n ? 1 : 0), + 0, + ); + const reserved = BigInt(positive); + const distributable = limit - reserved; + const mass = total - reserved; + const scaledCounts = Array.from( + { length: WARMUP_HISTOGRAM_BIN_COUNT }, + () => 0n, + ); + const remainders: Array<{ index: number; remainder: bigint }> = []; + let scaledTotal = 0n; + for (let index = 0; index < counts.length; index++) { + const count = counts[index]; + if (count === 0n) continue; + const weighted = (count - 1n) * distributable; + const scaled = 1n + weighted / mass; + scaledCounts[index] = scaled; + scaledTotal += scaled; + remainders.push({ index, remainder: weighted % mass }); + } + remainders.sort((left, right) => + left.remainder === right.remainder + ? left.index - right.index + : left.remainder > right.remainder + ? -1 + : 1, + ); + const remaining = Number(limit - scaledTotal); + for (let index = 0; index < remaining; index++) { + scaledCounts[remainders[index].index]++; + } + return scaledCounts; +} + +export function normalizeWarmupHistogram(counts: readonly bigint[]): { + counts: number[]; + total: number; +} { + const scaled = scaleWarmupHistogram( + counts, + BigInt(MAX_WARMUP_HISTOGRAM_TOTAL), + ); + return { + counts: scaled.map(Number), + total: Number(totalCounts(scaled)), + }; +} + +export function encodeWarmupHistogram(counts: readonly bigint[]): { + counts: string; + total: number; + weights: bigint[]; +} { + let weights = [...counts]; + let normalized = normalizeWarmupHistogram(weights); + if (totalCounts(counts) <= BigInt(MAX_WARMUP_HISTOGRAM_TOTAL)) { + return { + counts: JSON.stringify(normalized.counts), + total: normalized.total, + weights, + }; + } + let serialized = JSON.stringify({ + v: 1, + counts: normalized.counts, + exact: weights.map(String), + } satisfies ExactWarmupHistogram); + if (serialized.length > MAX_STORED_HISTOGRAM_CHARS) { + const compacted = scaleWarmupHistogram(weights, COMPACTED_EXACT_TOTAL); + const scale = + totalCounts(weights).toString().length - + totalCounts(compacted).toString().length; + const compactedDigits = compacted.reduce( + (max, count) => Math.max(max, count.toString().length), + 1, + ); + if (scale <= 0 || scale + compactedDigits > MAX_STORED_HISTOGRAM_CHARS) { + throw new Error("warmup histogram encoding exceeds storage limit"); + } + const multiplier = 10n ** BigInt(scale); + weights = compacted.map((count) => count * multiplier); + normalized = normalizeWarmupHistogram(weights); + serialized = JSON.stringify({ + v: 2, + counts: normalized.counts, + exact: compacted.map(String), + scale, + } satisfies CompactedWarmupHistogram); + } + if (serialized.length > MAX_STORED_HISTOGRAM_CHARS) { + throw new Error("warmup histogram encoding exceeds storage limit"); + } + return { counts: serialized, total: normalized.total, weights }; +} + +export function decodeWarmupHistogram( + rawCounts: string, + rawTotal: unknown, +): bigint[] | undefined { + if (rawCounts.length > MAX_STORED_HISTOGRAM_CHARS) { + return undefined; + } + try { + const parsed = JSON.parse(rawCounts) as unknown; + const legacy = parseLegacyCounts(parsed); + if (legacy) { + return storedTotalMatches(rawTotal, totalCounts(legacy)) + ? legacy + : undefined; + } + + if ( + typeof parsed !== "object" || + parsed === null || + Array.isArray(parsed) + ) { + return undefined; + } + const stored = parsed as Partial< + ExactWarmupHistogram | CompactedWarmupHistogram + >; + if (stored.v !== 1 && stored.v !== 2) return undefined; + let exact = parseExactCounts(stored.exact); + const visible = parseLegacyCounts(stored.counts); + if (!exact || !visible) return undefined; + if (stored.v === 2) { + const scale = stored.scale; + if ( + !Number.isSafeInteger(scale) || + (scale ?? 0) <= 0 || + (scale ?? 0) + + exact.reduce( + (max, count) => Math.max(max, count.toString().length), + 1, + ) > + MAX_STORED_HISTOGRAM_CHARS + ) { + return undefined; + } + const multiplier = 10n ** BigInt(scale ?? 0); + exact = exact.map((count) => count * multiplier); + } + const normalized = normalizeWarmupHistogram(exact); + if ( + !storedTotalMatches(rawTotal, BigInt(normalized.total)) || + visible.some((count, index) => count !== BigInt(normalized.counts[index])) + ) { + return undefined; + } + return exact; + } catch { + return undefined; + } +} diff --git a/packages/core/test/curator-changed-entries.test.ts b/packages/core/test/curator-changed-entries.test.ts index 9fa0d49f..7d40b864 100644 --- a/packages/core/test/curator-changed-entries.test.ts +++ b/packages/core/test/curator-changed-entries.test.ts @@ -1,11 +1,14 @@ -import { describe, expect, test } from "vitest"; -import { applyOps, enforceEntryCap } from "../src/curator"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { applyOps, enforceEntryCap, run as runCurator } from "../src/curator"; import type { ChangedEntry } from "../src/curator"; import { db, ensureProject } from "../src/db"; import * as ltm from "../src/ltm"; +import type { LLMClient } from "../src/types"; const PROJECT = "/tmp/lore-curator-delta/project"; +afterEach(() => vi.restoreAllMocks()); + describe("curator applyOps changedEntries", () => { test("returns createdEntries for genuine creates", () => { const result = applyOps( @@ -230,3 +233,161 @@ describe("curator enforceEntryCap (soft-cap eviction)", () => { expect(ltm.forProject(CAP_PROJECT, false)).toHaveLength(3); }); }); + +describe("signal-bound curator maintenance", () => { + test("runs duplicate cleanup, cap enforcement, and cursor persistence", async () => { + const projectPath = "/tmp/lore-curator-signal-maintenance/project"; + const sessionID = "signal-maintenance-session"; + const pid = ensureProject(projectPath); + db().query("DELETE FROM knowledge WHERE project_id = ?").run(pid); + db().query("DELETE FROM temporal_messages WHERE project_id = ?").run(pid); + db().query("DELETE FROM session_state WHERE session_id = ?").run(sessionID); + const now = Date.now(); + for (let i = 0; i < 3; i++) { + db() + .query( + `INSERT INTO temporal_messages + (id, project_id, session_id, role, content, tokens, distilled, created_at, metadata) + VALUES (?, ?, ?, 'user', ?, 1, 0, ?, '{}')`, + ) + .run( + `signal-maintenance-message-${i}`, + pid, + sessionID, + `message ${i}`, + now + i, + ); + } + for (let i = 0; i < 201; i++) { + ltm.create({ + projectPath, + category: "gotcha", + title: `Signal maintenance ${i}`, + content: `Distinct maintenance content ${i}`, + scope: "project", + confidence: 0.5, + }); + } + const deduplicate = vi.spyOn(ltm, "deduplicate"); + const llm: LLMClient = { + prompt: vi.fn().mockResolvedValue( + JSON.stringify({ + ops: [ + { + op: "create", + category: "gotcha", + title: "Foreground maintenance trigger", + content: "A distinct entry that triggers the post-create sweep.", + scope: "project", + }, + ], + entities: [], + relations: [], + }), + ), + }; + + await runCurator({ + llm, + projectPath, + sessionID, + signal: new AbortController().signal, + }); + + expect(deduplicate).toHaveBeenCalledWith(projectPath, { dryRun: false }); + expect(ltm.forProject(projectPath, false)).toHaveLength(200); + expect( + db() + .query("SELECT last_curated_at FROM session_state WHERE session_id = ?") + .get(sessionID), + ).toMatchObject({ last_curated_at: expect.any(Number) }); + }); + + test("aborting after dedup prevents stale cap and cursor writes", async () => { + const projectPath = "/tmp/lore-curator-signal-abort/project"; + const sessionID = "signal-maintenance-abort-session"; + const pid = ensureProject(projectPath); + db().query("DELETE FROM knowledge WHERE project_id = ?").run(pid); + db().query("DELETE FROM temporal_messages WHERE project_id = ?").run(pid); + db().query("DELETE FROM session_state WHERE session_id = ?").run(sessionID); + const now = Date.now(); + for (let i = 0; i < 3; i++) { + db() + .query( + `INSERT INTO temporal_messages + (id, project_id, session_id, role, content, tokens, distilled, created_at, metadata) + VALUES (?, ?, ?, 'user', ?, 1, 0, ?, '{}')`, + ) + .run( + `signal-abort-message-${i}`, + pid, + sessionID, + `message ${i}`, + now + i, + ); + } + for (let i = 0; i < 201; i++) { + ltm.create({ + projectPath, + category: "gotcha", + title: `Signal abort ${i}`, + content: `Distinct abort content ${i}`, + scope: "project", + confidence: 0.5, + }); + } + const beforeCuration = ltm.forProject(projectPath, false).length; + let releaseDedup!: () => void; + const dedupBlocked = new Promise< + Awaited> + >((resolve) => { + releaseDedup = () => + resolve({ + clusters: [], + totalRemoved: 0, + pairSimilarities: new Map(), + entryTitles: new Map(), + }); + }); + const deduplicate = vi + .spyOn(ltm, "deduplicate") + .mockReturnValueOnce(dedupBlocked); + const controller = new AbortController(); + const llm: LLMClient = { + prompt: vi.fn().mockResolvedValue( + JSON.stringify({ + ops: [ + { + op: "create", + category: "gotcha", + title: "Abort maintenance trigger", + content: "Trigger the dedup phase before cancellation.", + scope: "project", + }, + ], + entities: [], + relations: [], + }), + ), + }; + + const pending = runCurator({ + llm, + projectPath, + sessionID, + signal: controller.signal, + }); + await vi.waitFor(() => expect(deduplicate).toHaveBeenCalledTimes(1)); + controller.abort(new DOMException("pipeline reset", "AbortError")); + releaseDedup(); + + await expect(pending).rejects.toMatchObject({ name: "AbortError" }); + expect(ltm.forProject(projectPath, false)).toHaveLength(beforeCuration + 1); + expect(beforeCuration + 1).toBeGreaterThan(200); + expect( + db() + .query("SELECT last_curated_at FROM session_state WHERE session_id = ?") + .get(sessionID), + ).toBeNull(); + }); +}); diff --git a/packages/core/test/db-helpers.test.ts b/packages/core/test/db-helpers.test.ts index 08fa8b87..056beb6d 100644 --- a/packages/core/test/db-helpers.test.ts +++ b/packages/core/test/db-helpers.test.ts @@ -1,5 +1,172 @@ import { describe, test, expect, beforeEach } from "vitest"; -import { db, runUpsert, withTransaction, withSavepoint } from "../src/db"; +import { + close, + databaseInTransaction, + db, + runUpsert, + withTransaction, + withSavepoint, +} from "../src/db"; + +function withoutNativeTransactionState( + connection: ReturnType, +): ReturnType { + return new Proxy(connection, { + get(target, property) { + if (property === "isTransaction" || property === "inTransaction") { + return undefined; + } + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} + +describe("databaseInTransaction", () => { + test("fallback leaves an autocommit connection unchanged", () => { + const connection = db(); + const fallback = withoutNativeTransactionState(connection); + + expect(databaseInTransaction(fallback)).toBe(false); + expect(connection.isTransaction).toBe(false); + }); + + test("fallback observes raw exec and prepared transaction statements", () => { + const connection = db(); + const fallback = withoutNativeTransactionState(connection); + + connection.exec("BEGIN IMMEDIATE"); + try { + expect(databaseInTransaction(fallback)).toBe(true); + } finally { + connection.exec("COMMIT"); + } + expect(databaseInTransaction(fallback)).toBe(false); + + connection.query("BEGIN").run(); + try { + expect(databaseInTransaction(fallback)).toBe(true); + } finally { + connection.query("ROLLBACK").run(); + } + expect(databaseInTransaction(fallback)).toBe(false); + }); + + test("fallback follows nested and duplicate savepoint semantics", () => { + const connection = db(); + const fallback = withoutNativeTransactionState(connection); + + connection.exec("SAVEPOINT outer_sp"); + try { + expect(databaseInTransaction(fallback)).toBe(true); + connection.exec("SAVEPOINT inner_sp; SAVEPOINT inner_sp"); + expect(databaseInTransaction(fallback)).toBe(true); + connection.exec("ROLLBACK TO outer_sp"); + expect(databaseInTransaction(fallback)).toBe(true); + connection.exec("RELEASE outer_sp"); + } catch (error) { + if (connection.isTransaction) connection.exec("ROLLBACK"); + throw error; + } + expect(databaseInTransaction(fallback)).toBe(false); + + connection.exec("BEGIN; SAVEPOINT outer_tx; SAVEPOINT inner_tx"); + try { + connection.exec("RELEASE outer_tx"); + expect(databaseInTransaction(fallback)).toBe(true); + } finally { + connection.exec("ROLLBACK"); + } + expect(databaseInTransaction(fallback)).toBe(false); + }); + + test("fallback reflects failed transaction boundaries", () => { + const connection = db(); + const fallback = withoutNativeTransactionState(connection); + connection.exec(` + CREATE TABLE IF NOT EXISTS _t_tx_parent (id INTEGER PRIMARY KEY); + CREATE TABLE IF NOT EXISTS _t_tx_child ( + parent_id INTEGER REFERENCES _t_tx_parent(id) + DEFERRABLE INITIALLY DEFERRED + ); + DELETE FROM _t_tx_child; + DELETE FROM _t_tx_parent; + BEGIN; + INSERT INTO _t_tx_child(parent_id) VALUES (1); + `); + try { + expect(() => connection.exec("COMMIT")).toThrow(); + expect(databaseInTransaction(fallback)).toBe(true); + expect(() => connection.exec("ROLLBACK TO missing_savepoint")).toThrow(); + expect(databaseInTransaction(fallback)).toBe(true); + } finally { + connection.exec("ROLLBACK"); + } + expect(databaseInTransaction(fallback)).toBe(false); + }); + + test("fallback propagates probe cleanup failures", () => { + const connection = db(); + const fallback = withoutNativeTransactionState(connection); + const cleanupFailure = new Error("simulated rollback failure"); + const brokenCleanup = new Proxy(fallback, { + get(target, property) { + if (property === "exec") { + return (sql: string) => { + if (sql === "BEGIN DEFERRED; ROLLBACK") { + connection.exec("BEGIN DEFERRED"); + throw cleanupFailure; + } + return connection.exec(sql); + }; + } + return Reflect.get(target, property, target); + }, + }); + + try { + expect(() => databaseInTransaction(brokenCleanup)).toThrow( + cleanupFailure, + ); + expect(connection.isTransaction).toBe(true); + } finally { + if (connection.isTransaction) connection.exec("ROLLBACK"); + } + }); + + test("fallback observes transaction wrappers and callback rollback", () => { + const connection = db(); + const fallback = withoutNativeTransactionState(connection); + + withTransaction(() => { + expect(databaseInTransaction(fallback)).toBe(true); + withSavepoint("fallback_nested", () => { + expect(databaseInTransaction(fallback)).toBe(true); + }); + expect(databaseInTransaction(fallback)).toBe(true); + }); + expect(databaseInTransaction(fallback)).toBe(false); + + expect(() => + withTransaction(() => { + expect(databaseInTransaction(fallback)).toBe(true); + throw new Error("rollback fallback wrapper"); + }), + ).toThrow("rollback fallback wrapper"); + expect(databaseInTransaction(fallback)).toBe(false); + }); + + test("fallback starts clean after close and reopen", () => { + const connection = db(); + connection.exec("BEGIN IMMEDIATE"); + close(); + + const reopened = db(); + expect(databaseInTransaction(withoutNativeTransactionState(reopened))).toBe( + false, + ); + }); +}); describe("runUpsert", () => { beforeEach(() => { diff --git a/packages/core/test/db.test.ts b/packages/core/test/db.test.ts index 433460cd..55c262ef 100644 --- a/packages/core/test/db.test.ts +++ b/packages/core/test/db.test.ts @@ -1,11 +1,16 @@ import { describe, test, expect, beforeEach, afterEach } from "vitest"; +import { DatabaseSync } from "node:sqlite"; +import { registerSink, type LogSink } from "../src/log"; import { db, close, ensureProject, projectId, + canonicalProjectId, + onProjectMutation, invalidateProjectIdCache, mergeProjectInternal, + PROJECT_MERGE_TABLES, loadForceMinLayer, saveForceMinLayer, getMeta, @@ -20,6 +25,7 @@ import { loadSessionTracking, findSessionStatesByFingerprint, countMatchingTemporalIds, + withSavepoint, appendSessionPromptDelta, listSessionPromptDeltas, loadHeaderSessionIndex, @@ -35,9 +41,28 @@ import { isUnattributedProjectPath, UNATTRIBUTED_PROJECT_PREFIX, assertFts5Available, + MIGRATIONS, } from "../src/db"; import { enableHostedMode, _resetHostedModeForTest } from "../src/hosted"; -import { deleteProject } from "../src/data"; +import { + deleteProject, + invalidateProjectsCache, + listProjects, + mergeProjects, +} from "../src/data"; +import { + decodeWarmupHistogram, + encodeWarmupHistogram, + emptyWarmupHistogramCounts, + normalizeWarmupHistogram, +} from "../src/warmup-histogram"; + +const passthroughLogSink: LogSink = { + info() {}, + warn() {}, + error() {}, + captureException() {}, +}; describe("db", () => { test("initializes and creates tables", () => { @@ -119,7 +144,7 @@ describe("db", () => { const row = db().query("SELECT version FROM schema_version").get() as { version: number; }; - expect(row.version).toBe(82); + expect(row.version).toBe(MIGRATIONS.length); }); test("v81 quarantines sync bookkeeping that predates tenant provenance", () => { @@ -153,7 +178,7 @@ describe("db", () => { migrated.query("SELECT row_id, tenant_id FROM sync_state").all(), ).toEqual([{ row_id: "legacy-state", tenant_id: null }]); expect(migrated.query("SELECT version FROM schema_version").get()).toEqual({ - version: 82, + version: MIGRATIONS.length, }); expect( migrated @@ -225,7 +250,7 @@ describe("db", () => { const migrated = db(); expect(migrated.query("SELECT version FROM schema_version").get()).toEqual({ - version: 82, + version: MIGRATIONS.length, }); expect( migrated @@ -324,7 +349,7 @@ describe("db", () => { const ver = fresh.query("SELECT version FROM schema_version").get() as { version: number; }; - expect(ver.version).toBe(82); + expect(ver.version).toBe(MIGRATIONS.length); // Register + JOIN view were rebuilt and are queryable (confidence exposed). expect( fresh @@ -361,7 +386,7 @@ describe("db", () => { const ver = fresh.query("SELECT version FROM schema_version").get() as { version: number; }; - expect(ver.version).toBe(82); + expect(ver.version).toBe(MIGRATIONS.length); }); test("v56: knowledge_ref_validity table + projects.last_refcheck_at exist after recovery", () => { @@ -834,6 +859,11 @@ describe("db", () => { "INSERT OR IGNORE INTO project_path_aliases (path, project_id) VALUES (?, ?)", ) .run(worktree, id1); + db() + .query( + "INSERT INTO project_id_aliases (retired_id, project_id) VALUES (?, ?)", + ) + .run("retired-before-delete", id1); expect(ensureProject(worktree)).toBe(id1); // prime the memo deleteProject(id1); @@ -846,6 +876,13 @@ describe("db", () => { expect( db().query("SELECT 1 AS n FROM projects WHERE id = ?").get(id2), ).toBeTruthy(); + expect( + db() + .query( + "SELECT project_id FROM project_id_aliases WHERE retired_id = ?", + ) + .get("retired-before-delete"), + ).toBeNull(); }); test("mergeProjectInternal invalidates the memo (source path resolves to the winner, not the deleted source)", () => { @@ -868,6 +905,88 @@ describe("db", () => { expect(ensureProject(loserPath)).toBe(winner); expect(projectId(loserPath)).toBe(winner); }); + + test("does not publish an alias to the memo before its transaction commits", () => { + enableHostedMode(); + try { + const canonical = `/test/memo-rollback/canonical-${crypto.randomUUID()}`; + const worktree = `/test/memo-rollback/worktree-${crypto.randomUUID()}`; + const remote = `github.com/test/memo-rollback-${crypto.randomUUID()}`; + const id = ensureProject(canonical, undefined, remote); + + withSavepoint("project_alias_publication", () => { + expect(ensureProject(worktree, undefined, remote)).toBe(id); + // Delete the still-uncommitted alias directly. If ensureProject had + // speculatively memoized it, projectId would keep returning `id` even + // though no SQL row now supports that mapping. + db() + .query("DELETE FROM project_path_aliases WHERE path = ?") + .run(worktree); + expect(projectId(worktree)).toBeUndefined(); + }); + } finally { + _resetHostedModeForTest(); + } + }); + + test("does not publish a lookup under a data version committed after its query", () => { + const path = `/test/query-memo-race-${crypto.randomUUID()}`; + const movedPath = `${path}-moved`; + const oldId = crypto.randomUUID(); + db() + .query( + "INSERT INTO projects (id, path, name, git_remote, created_at) VALUES (?, ?, ?, ?, ?)", + ) + .run( + oldId, + path, + "old owner", + `github.com/test/query-memo-race-${crypto.randomUUID()}`, + Date.now(), + ); + invalidateProjectIdCache(); + + const freshId = crypto.randomUUID(); + const external = new DatabaseSync(process.env.LORE_DB_PATH as string); + external.exec("PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON"); + let reassigned = false; + registerSink({ + ...passthroughLogSink, + withDbSpan(sql: string, fn: () => T): T { + const result = fn(); + if ( + !reassigned && + sql.includes("SELECT id, git_remote FROM projects") && + sql.includes("path = ?") + ) { + reassigned = true; + external.exec("BEGIN IMMEDIATE"); + external + .prepare("UPDATE projects SET path = ? WHERE id = ?") + .run(movedPath, oldId); + external + .prepare( + "INSERT INTO projects (id, path, name, created_at) VALUES (?, ?, ?, ?)", + ) + .run(freshId, path, "fresh owner", Date.now()); + external.exec("COMMIT"); + } + return result; + }, + }); + try { + // This query legitimately saw oldId, but the external commit lands + // after the SQL completes and before memo publication. + expect(projectId(path)).toBe(oldId); + } finally { + registerSink(passthroughLogSink); + external.close(); + } + + expect(reassigned).toBe(true); + // The lookup-start data_version must invalidate the stale result here. + expect(projectId(path)).toBe(freshId); + }); }); test("ensureProject deduplicates via git_remote", () => { @@ -981,14 +1100,110 @@ describe("db", () => { expect( ensureProject(path, undefined, "github.com/test/backfill-cache"), ).toBe(id); - // Raw-delete the row WITHOUT invalidating: a cache HIT must still return - // id, proving the post-backfill mapping was memoized (so the next call - // skips the exact-path lookup — the extra lookup Seer flagged). - db().query("DELETE FROM projects WHERE id = ?").run(id); - expect( - ensureProject(path, undefined, "github.com/test/backfill-cache"), - ).toBe(id); + const queries: string[] = []; + registerSink({ + ...passthroughLogSink, + withDbSpan(sql: string, fn: () => T): T { + queries.push(sql); + return fn(); + }, + }); + try { + expect( + ensureProject(path, undefined, "github.com/test/backfill-cache"), + ).toBe(id); + expect(queries).toContain("PRAGMA data_version"); + expect( + queries.some((sql) => + sql.includes("SELECT id, git_remote FROM projects WHERE path = ?"), + ), + ).toBe(false); + } finally { + registerSink(passthroughLogSink); + } }); + + test("external path reassignment invalidates a live cached UUID", () => { + const path = `/test/external-path-owner-${crypto.randomUUID()}`; + const movedPath = `${path}-moved`; + const oldId = ensureProject( + path, + undefined, + `github.com/test/external-path-${crypto.randomUUID()}`, + ); + expect(projectId(path)).toBe(oldId); // prime the remote-backed memo + + const freshId = crypto.randomUUID(); + const external = new DatabaseSync(process.env.LORE_DB_PATH as string); + try { + external.exec("PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON"); + external.exec("BEGIN IMMEDIATE"); + external + .prepare("UPDATE projects SET path = ? WHERE id = ?") + .run(movedPath, oldId); + external + .prepare( + "INSERT INTO projects (id, path, name, created_at) VALUES (?, ?, ?, ?)", + ) + .run(freshId, path, "fresh external owner", Date.now()); + external.exec("COMMIT"); + } finally { + external.close(); + } + + expect(projectId(path)).toBe(freshId); + expect(ensureProject(path)).toBe(freshId); + }); + }); + + test("committed canonical reads stay bound to the live database path", () => { + const sourceId = ensureProject( + `/test/committed-reader-source-${crypto.randomUUID()}`, + ); + const targetId = ensureProject( + `/test/committed-reader-target-${crypto.randomUUID()}`, + ); + mergeProjectInternal(sourceId, targetId); + + const originalPath = process.env.LORE_DB_PATH; + process.env.LORE_DB_PATH = ":memory:"; + try { + withSavepoint("committed_reader_live_path", () => { + expect(canonicalProjectId(sourceId, { committed: true })).toBe( + targetId, + ); + }); + } finally { + if (originalPath === undefined) delete process.env.LORE_DB_PATH; + else process.env.LORE_DB_PATH = originalPath; + } + }); + + test("committed canonical reads fail closed for private in-memory databases", () => { + const originalPath = process.env.LORE_DB_PATH; + close(); + try { + const memoryPaths = [ + ":memory:", + `file:lore-committed-${crypto.randomUUID()}?mode=memory&cache=shared`, + ]; + for (const [index, memoryPath] of memoryPaths.entries()) { + process.env.LORE_DB_PATH = memoryPath; + const sourceId = ensureProject(`/test/in-memory-source-${index}`); + const targetId = ensureProject(`/test/in-memory-target-${index}`); + withSavepoint("in_memory_committed_reader", () => { + mergeProjectInternal(sourceId, targetId); + expect( + canonicalProjectId(sourceId, { committed: true }), + ).toBeUndefined(); + }); + close(); + } + } finally { + close(); + if (originalPath === undefined) delete process.env.LORE_DB_PATH; + else process.env.LORE_DB_PATH = originalPath; + } }); // Regression: the "git-remote magnet" bug. A non-repo path (e.g. a parent @@ -1183,6 +1398,759 @@ describe("db", () => { .get("/test/merge/source") as { project_id: string } | null; expect(alias).not.toBeNull(); expect(alias?.project_id).toBe(targetId); + expect(canonicalProjectId(sourceId)).toBe(targetId); + }); + + test("project self-merges preserve the project and all scoped data", () => { + const project = ensureProject("/test/merge/self"); + db() + .query( + "INSERT INTO temporal_messages (id, project_id, session_id, role, content, created_at) VALUES (?, ?, ?, ?, ?, ?)", + ) + .run( + crypto.randomUUID(), + project, + "session-self-merge", + "user", + "must survive", + Date.now(), + ); + const mutations: unknown[] = []; + const unsubscribe = onProjectMutation((mutation) => + mutations.push(mutation), + ); + + try { + mergeProjectInternal(project, project); + expect(mergeProjects(project, project)).toEqual({ + knowledge_moved: 0, + messages_moved: 0, + distillations_moved: 0, + }); + + expect( + db().query("SELECT path FROM projects WHERE id = ?").get(project), + ).toEqual({ path: "/test/merge/self" }); + expect( + db() + .query("SELECT content FROM temporal_messages WHERE project_id = ?") + .all(project), + ).toEqual([{ content: "must survive" }]); + expect(mutations).toEqual([]); + } finally { + unsubscribe(); + } + }); + + test("project ID redirects flatten transitive merges and ignore stale sources", () => { + const source = ensureProject("/test/merge-id-alias/source"); + const middle = ensureProject("/test/merge-id-alias/middle"); + const target = ensureProject("/test/merge-id-alias/target"); + const staleTarget = ensureProject("/test/merge-id-alias/stale-target"); + + mergeProjectInternal(source, middle); + mergeProjectInternal(middle, target); + mergeProjectInternal(source, staleTarget); + + expect(canonicalProjectId(source)).toBe(target); + expect(canonicalProjectId(middle)).toBe(target); + expect(canonicalProjectId(target)).toBe(target); + expect( + db() + .query( + "SELECT retired_id, project_id FROM project_id_aliases WHERE retired_id IN (?, ?) ORDER BY retired_id", + ) + .all(source, middle), + ).toEqual( + [ + { retired_id: source, project_id: target }, + { retired_id: middle, project_id: target }, + ].sort((a, b) => a.retired_id.localeCompare(b.retired_id)), + ); + }); + + test("mergeProjectInternal covers every project-scoped table", () => { + const d = db(); + expect(Object.isFrozen(PROJECT_MERGE_TABLES)).toBe(true); + const tables = ( + d + .query( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'", + ) + .all() as Array<{ name: string }> + ) + .map((row) => row.name) + .filter( + (name) => + !name.startsWith("temporal_vec_") && + !name.startsWith("distillation_vec_"), + ); + const projectScoped = tables.filter((table) => { + const quoted = table.replaceAll('"', '""'); + const columns = d.query(`PRAGMA table_info("${quoted}")`).all() as Array<{ + name: string; + }>; + return columns.some( + (column) => + column.name === "project_id" || + column.name === "recalled_in_project_id", + ); + }); + const registered = PROJECT_MERGE_TABLES.filter((table) => + tables.includes(table), + ); + expect(projectScoped.sort()).toEqual([...registered].sort()); + }); + + test("mergeProjectInternal preserves collision-prone project sidecars", () => { + const d = db(); + const sourceId = ensureProject("/test/merge-sidecars/source"); + const targetId = ensureProject("/test/merge-sidecars/target"); + const sourceCounts = Array.from({ length: 21 }, () => 1); + const targetCounts = Array.from({ length: 21 }, () => 2); + + d.query( + `INSERT INTO session_prompt_deltas + (session_id, seq, project_id, selector, content) + VALUES ('source-session', 1, ?, 'after-system', 'source delta')`, + ).run(sourceId); + d.query( + `INSERT INTO import_history + (id, project_id, agent_name, source_id, source_hash, + entries_created, entries_updated, imported_at) + VALUES ('target-import', ?, 'opencode', 'shared', 'old', 1, 2, 100), + ('source-import', ?, 'opencode', 'shared', 'new', 3, 4, 200), + ('source-only-import', ?, 'claude', 'unique', 'only', 5, 6, 150)`, + ).run(targetId, sourceId, sourceId); + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, 21, 100), (?, 'all', ?, 42, 200)`, + ).run( + sourceId, + JSON.stringify(sourceCounts), + targetId, + JSON.stringify(targetCounts), + ); + d.query( + `INSERT INTO dedup_feedback + (project_id, entry_a_title, entry_b_title, similarity, accepted, + source, created_at, kind) + VALUES (?, 'source-a', 'source-b', 0.9, 1, 'manual', 100, 'knowledge'), + (?, 'target-a', 'target-b', 0.8, 0, 'manual', 200, 'knowledge')`, + ).run(sourceId, targetId); + d.query( + `INSERT INTO knowledge_tombstones (id, project_id, deleted_at) + VALUES ('source-tombstone', ?, 100), ('target-tombstone', ?, 200)`, + ).run(sourceId, targetId); + d.query( + `INSERT INTO cache_bust_stats + (project_id, cause, relocatable, turns, write_tokens, updated_at) + VALUES (?, 'system-host-change', 1, 2, 20, 100), + (?, 'system-host-change', 1, 3, 30, 200)`, + ).run(sourceId, targetId); + d.query( + `INSERT INTO knowledge_contradictions + (logical_id_a, logical_id_b, project_id, similarity, status, + detected_at, updated_at) + VALUES ('source-a', 'source-b', ?, 0.9, 'open', 100, 100), + ('target-a', 'target-b', ?, 0.8, 'resolved', 200, 200)`, + ).run(sourceId, targetId); + + mergeProjectInternal(sourceId, targetId); + + for (const table of [ + "session_prompt_deltas", + "import_history", + "warmup_histograms", + "dedup_feedback", + "knowledge_tombstones", + "cache_bust_stats", + "knowledge_contradictions", + ]) { + const quoted = table.replaceAll('"', '""'); + const source = d + .query(`SELECT COUNT(*) AS count FROM "${quoted}" WHERE project_id = ?`) + .get(sourceId) as { count: number }; + expect(source.count, table).toBe(0); + } + expect( + d + .query( + "SELECT project_id FROM session_prompt_deltas WHERE session_id = 'source-session'", + ) + .get(), + ).toEqual({ project_id: targetId }); + expect( + d + .query( + `SELECT source_hash, entries_created, entries_updated, imported_at + FROM import_history + WHERE project_id = ? AND agent_name = 'opencode' AND source_id = 'shared'`, + ) + .get(targetId), + ).toEqual({ + source_hash: "new", + entries_created: 3, + entries_updated: 4, + imported_at: 200, + }); + expect( + d + .query( + "SELECT COUNT(*) AS count FROM import_history WHERE project_id = ?", + ) + .get(targetId), + ).toEqual({ count: 2 }); + const histogram = d + .query( + "SELECT counts, total, updated_at FROM warmup_histograms WHERE project_id = ? AND time_slot = 'all'", + ) + .get(targetId) as { counts: string; total: number; updated_at: number }; + expect(JSON.parse(histogram.counts)).toEqual( + Array.from({ length: 21 }, () => 3), + ); + expect(histogram).toMatchObject({ total: 63, updated_at: 200 }); + expect( + d + .query( + "SELECT COUNT(*) AS count FROM dedup_feedback WHERE project_id = ?", + ) + .get(targetId), + ).toEqual({ count: 2 }); + expect( + d + .query( + "SELECT COUNT(*) AS count FROM knowledge_tombstones WHERE project_id = ?", + ) + .get(targetId), + ).toEqual({ count: 2 }); + expect( + d + .query( + `SELECT turns, write_tokens, updated_at FROM cache_bust_stats + WHERE project_id = ? AND cause = 'system-host-change' AND relocatable = 1`, + ) + .get(targetId), + ).toEqual({ turns: 5, write_tokens: 50, updated_at: 200 }); + expect( + d + .query( + "SELECT COUNT(*) AS count FROM knowledge_contradictions WHERE project_id = ?", + ) + .get(targetId), + ).toEqual({ count: 2 }); + }); + + test("mergeProjectInternal rebuilds colliding session rollups from source rows", () => { + const d = db(); + const sourceId = ensureProject("/test/merge-rollup/source"); + const targetId = ensureProject("/test/merge-rollup/target"); + const sessionId = "shared-rollup-session"; + d.query( + `INSERT INTO temporal_messages + (id, project_id, session_id, role, content, tokens, created_at) + VALUES ('source-rollup-message', ?, ?, 'user', 'source', 5, 100), + ('target-rollup-message', ?, ?, 'assistant', 'target', 7, 200)`, + ).run(sourceId, sessionId, targetId, sessionId); + expect( + d + .query( + "SELECT COUNT(*) AS count FROM session_rollup WHERE session_id = ?", + ) + .get(sessionId), + ).toEqual({ count: 2 }); + + mergeProjectInternal(sourceId, targetId); + + expect( + d + .query( + `SELECT project_id, message_count, token_sum, first_message_at, + last_message_at, first_assistant_metadata, dirty + FROM session_rollup WHERE session_id = ?`, + ) + .all(sessionId), + ).toEqual([ + { + project_id: targetId, + message_count: 2, + token_sum: 12, + first_message_at: 100, + last_message_at: 200, + first_assistant_metadata: null, + dirty: 0, + }, + ]); + }); + + test("mergeProjectInternal never combines invalid warmup histograms", () => { + const d = db(); + const sourceId = ensureProject("/test/merge-histogram/source"); + const targetId = ensureProject("/test/merge-histogram/target"); + const valid = JSON.stringify(Array.from({ length: 21 }, () => 2)); + const invalid = new Map([ + ["wrong-length", JSON.stringify([1, 1])], + [ + "non-finite", + `[${Array.from({ length: 21 }, () => "1") + .slice(0, 20) + .join(",")},1e400]`, + ], + ["malformed", "{bad"], + [ + "negative", + JSON.stringify([-1, ...Array.from({ length: 20 }, () => 1)]), + ], + [ + "fractional", + JSON.stringify([0.5, ...Array.from({ length: 20 }, () => 1)]), + ], + ]); + for (const [slot, counts] of invalid) { + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, ?, ?, 21, 100), (?, ?, ?, 42, 200)`, + ).run(sourceId, slot, counts, targetId, slot, valid); + } + + mergeProjectInternal(sourceId, targetId); + + const rows = d + .query( + "SELECT time_slot, counts, total, updated_at FROM warmup_histograms WHERE project_id = ? ORDER BY time_slot", + ) + .all(targetId) as Array<{ + time_slot: string; + counts: string; + total: number; + updated_at: number; + }>; + expect(rows).toHaveLength(invalid.size); + for (const row of rows) { + expect(row.counts, row.time_slot).toBe(valid); + expect(row.total, row.time_slot).toBe(42); + expect(row.updated_at, row.time_slot).toBe(200); + } + }); + + test("mergeProjectInternal makes the source canonical path target the winner", () => { + const d = db(); + const sourcePath = "/test/merge-alias-conflict/source"; + const sourceId = ensureProject(sourcePath); + const targetId = ensureProject("/test/merge-alias-conflict/target"); + const unrelatedId = ensureProject("/test/merge-alias-conflict/unrelated"); + d.query( + "INSERT INTO project_path_aliases (path, project_id) VALUES (?, ?)", + ).run(sourcePath, unrelatedId); + + mergeProjectInternal(sourceId, targetId); + + expect(projectId(sourcePath)).toBe(targetId); + expect( + d + .query("SELECT project_id FROM project_path_aliases WHERE path = ?") + .get(sourcePath), + ).toEqual({ project_id: targetId }); + }); + + test("mergeProjectInternal rejects 64-bit counter overflow atomically", () => { + const d = db(); + const cacheSource = ensureProject("/test/merge-overflow/cache-source"); + const cacheTarget = ensureProject("/test/merge-overflow/cache-target"); + d.exec(` + INSERT INTO cache_bust_stats + (project_id, cause, relocatable, turns, write_tokens, updated_at) + VALUES ('${cacheSource}', 'overflow', 0, 1, 1, 100), + ('${cacheTarget}', 'overflow', 0, 9223372036854775807, + 9223372036854775807, 200); + `); + expect(() => mergeProjectInternal(cacheSource, cacheTarget)).toThrow( + /counter overflow/, + ); + expect( + d + .query("SELECT COUNT(*) AS count FROM projects WHERE id IN (?, ?)") + .get(cacheSource, cacheTarget), + ).toEqual({ count: 2 }); + expect( + d + .query( + "SELECT COUNT(*) AS count FROM cache_bust_stats WHERE project_id IN (?, ?)", + ) + .get(cacheSource, cacheTarget), + ).toEqual({ count: 2 }); + + const transferSource = ensureProject( + "/test/merge-overflow/transfer-source", + ); + const transferTarget = ensureProject( + "/test/merge-overflow/transfer-target", + ); + d.exec(` + INSERT INTO knowledge_transfers + (knowledge_id, recalled_in_project_id, hit_count, + first_recalled_at, last_recalled_at) + VALUES ('overflow-entry', '${transferSource}', 1, 100, 100), + ('overflow-entry', '${transferTarget}', 9223372036854775807, + 200, 200); + `); + expect(() => mergeProjectInternal(transferSource, transferTarget)).toThrow( + /counter overflow/, + ); + expect( + d + .query("SELECT COUNT(*) AS count FROM projects WHERE id IN (?, ?)") + .get(transferSource, transferTarget), + ).toEqual({ count: 2 }); + expect( + d + .query( + "SELECT COUNT(*) AS count FROM knowledge_transfers WHERE recalled_in_project_id IN (?, ?)", + ) + .get(transferSource, transferTarget), + ).toEqual({ count: 2 }); + }); + + test.each([ + ["turns", 1.5, 2], + ["write tokens", 1, 2.5], + ] as const)( + "mergeProjectInternal rejects fractional source-only cache %s atomically", + (_field, turns, writeTokens) => { + const d = db(); + const cacheSource = ensureProject( + `/test/merge-invalid/cache-source-${_field}`, + ); + const cacheTarget = ensureProject( + `/test/merge-invalid/cache-target-${_field}`, + ); + d.query( + `INSERT INTO cache_bust_stats + (project_id, cause, relocatable, turns, write_tokens, updated_at) + VALUES (?, ?, 0, ?, ?, 100)`, + ).run(cacheSource, `invalid-source-${_field}`, turns, writeTokens); + + expect(() => mergeProjectInternal(cacheSource, cacheTarget)).toThrow( + /counter overflow/, + ); + expect( + d + .query( + `SELECT project_id, turns, write_tokens + FROM cache_bust_stats WHERE project_id = ?`, + ) + .get(cacheSource), + ).toEqual({ + project_id: cacheSource, + turns, + write_tokens: writeTokens, + }); + expect( + d + .query("SELECT COUNT(*) AS count FROM projects WHERE id IN (?, ?)") + .get(cacheSource, cacheTarget), + ).toEqual({ count: 2 }); + }, + ); + + test("mergeProjectInternal rejects fractional source-only transfer counters atomically", () => { + const d = db(); + const transferSource = ensureProject("/test/merge-invalid/transfer-source"); + const transferTarget = ensureProject("/test/merge-invalid/transfer-target"); + d.exec(` + INSERT INTO knowledge_transfers + (knowledge_id, recalled_in_project_id, hit_count, + first_recalled_at, last_recalled_at) + VALUES ('invalid-source-entry', '${transferSource}', 3.5, 100, 100); + `); + expect(() => mergeProjectInternal(transferSource, transferTarget)).toThrow( + /counter overflow/, + ); + expect( + d + .query( + `SELECT recalled_in_project_id, hit_count + FROM knowledge_transfers + WHERE knowledge_id = 'invalid-source-entry'`, + ) + .get(), + ).toEqual({ recalled_in_project_id: transferSource, hit_count: 3.5 }); + expect( + d + .query("SELECT COUNT(*) AS count FROM projects WHERE id IN (?, ?)") + .get(transferSource, transferTarget), + ).toEqual({ count: 2 }); + }); + + test("mergeProjectInternal preserves valid histogram distributions across aggregate overflow", () => { + const d = db(); + const sourceId = ensureProject("/test/merge-overflow-histogram/source"); + const targetId = ensureProject("/test/merge-overflow-histogram/target"); + const laterId = ensureProject("/test/merge-overflow-histogram/later"); + const sourceTotal = Number.MAX_SAFE_INTEGER; + const emptyCounts = Array.from({ length: 21 }, () => 0); + const sourceCounts = [...emptyCounts]; + const targetCounts = [...emptyCounts]; + const laterCounts = [...emptyCounts]; + sourceCounts[0] = sourceTotal - 1; + sourceCounts[1] = 1; + targetCounts[2] = 1; + laterCounts[3] = 1; + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, ?, 100), + (?, 'all', ?, ?, 100), + (?, 'all', ?, ?, 100)`, + ).run( + sourceId, + JSON.stringify(sourceCounts), + sourceTotal, + targetId, + JSON.stringify(targetCounts), + 1, + laterId, + JSON.stringify(laterCounts), + 1, + ); + + mergeProjectInternal(sourceId, targetId); + mergeProjectInternal(targetId, laterId); + + const row = d + .query( + "SELECT counts, total FROM warmup_histograms WHERE project_id = ? AND time_slot = 'all'", + ) + .get(laterId) as { counts: string; total: number }; + const exact = decodeWarmupHistogram(row.counts, row.total); + expect(exact).toBeDefined(); + const counts = normalizeWarmupHistogram(exact ?? []).counts; + expect(Number.isSafeInteger(row.total)).toBe(true); + expect(counts.reduce((sum, count) => sum + count, 0)).toBe(row.total); + expect(counts.slice(0, 4).every((count) => count > 0)).toBe(true); + expect( + d + .query( + "SELECT COUNT(*) AS count FROM warmup_histograms WHERE project_id IN (?, ?)", + ) + .get(sourceId, targetId), + ).toEqual({ count: 0 }); + }); + + test("mergeProjectInternal persists a decodable row when exact weights gain a digit", () => { + const d = db(); + const sourceId = ensureProject("/test/merge-exact-boundary/source"); + const targetId = ensureProject("/test/merge-exact-boundary/target"); + const boundary = 10n ** 1000n - 1n; + const sourceWeights = emptyWarmupHistogramCounts(); + const targetWeights = emptyWarmupHistogramCounts(); + sourceWeights[0] = boundary; + sourceWeights[1] = 1n; + targetWeights[0] = boundary; + targetWeights[2] = 1n; + const source = encodeWarmupHistogram(sourceWeights); + const target = encodeWarmupHistogram(targetWeights); + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, ?, 100), (?, 'all', ?, ?, 100)`, + ).run( + sourceId, + source.counts, + source.total, + targetId, + target.counts, + target.total, + ); + + mergeProjectInternal(sourceId, targetId); + + const row = d + .query( + "SELECT counts, total FROM warmup_histograms WHERE project_id = ? AND time_slot = 'all'", + ) + .get(targetId) as { counts: string; total: number }; + const decoded = decodeWarmupHistogram(row.counts, row.total); + expect(decoded).toBeDefined(); + expect(decoded?.[0].toString()).toHaveLength(1001); + expect(decoded?.slice(0, 3).every((count) => count > 0n)).toBe(true); + }); + + test.each([ + ["source", 200, 100], + ["target", 100, 200], + ] as const)( + "mergeProjectInternal retains newer %s histogram with unsafe totals", + (_newer, sourceUpdatedAt, targetUpdatedAt) => { + const d = db(); + const sourceId = ensureProject( + `/test/merge-unsafe-histogram/source-${sourceUpdatedAt}`, + ); + const targetId = ensureProject( + `/test/merge-unsafe-histogram/target-${targetUpdatedAt}`, + ); + const sourceCounts = JSON.stringify(Array.from({ length: 21 }, () => 1)); + const targetCounts = JSON.stringify(Array.from({ length: 21 }, () => 2)); + d.exec(` + INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES ('${sourceId}', 'all', '${sourceCounts}', + 9223372036854775807, ${sourceUpdatedAt}), + ('${targetId}', 'all', '${targetCounts}', + 9223372036854775806, ${targetUpdatedAt}); + `); + + mergeProjectInternal(sourceId, targetId); + + const row = d + .query( + `SELECT counts, CAST(total AS TEXT) AS total, updated_at + FROM warmup_histograms + WHERE project_id = ? AND time_slot = 'all'`, + ) + .get(targetId); + expect(row).toEqual( + sourceUpdatedAt > targetUpdatedAt + ? { + counts: sourceCounts, + total: "9223372036854775807", + updated_at: sourceUpdatedAt, + } + : { + counts: targetCounts, + total: "9223372036854775806", + updated_at: targetUpdatedAt, + }, + ); + }, + ); + + test.each([ + ["source", 200, 100], + ["target", 100, 200], + ] as const)( + "mergeProjectInternal retains newer %s histogram when totals disagree with counts", + (_newer, sourceUpdatedAt, targetUpdatedAt) => { + const d = db(); + const sourceId = ensureProject( + `/test/merge-inconsistent-histogram/source-${sourceUpdatedAt}`, + ); + const targetId = ensureProject( + `/test/merge-inconsistent-histogram/target-${targetUpdatedAt}`, + ); + const sourceCounts = JSON.stringify(Array.from({ length: 21 }, () => 1)); + const targetCounts = JSON.stringify(Array.from({ length: 21 }, () => 2)); + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, 22, ?), (?, 'all', ?, 43, ?)`, + ).run( + sourceId, + sourceCounts, + sourceUpdatedAt, + targetId, + targetCounts, + targetUpdatedAt, + ); + + mergeProjectInternal(sourceId, targetId); + + expect( + d + .query( + `SELECT counts, total, updated_at + FROM warmup_histograms + WHERE project_id = ? AND time_slot = 'all'`, + ) + .get(targetId), + ).toEqual( + sourceUpdatedAt > targetUpdatedAt + ? { counts: sourceCounts, total: 22, updated_at: sourceUpdatedAt } + : { counts: targetCounts, total: 43, updated_at: targetUpdatedAt }, + ); + }, + ); + + test("mergeProjectInternal rejects malformed rollup source counters", () => { + const d = db(); + const sourceId = ensureProject("/test/merge-invalid-rollup/source"); + const targetId = ensureProject("/test/merge-invalid-rollup/target"); + const sessionId = "invalid-rollup-session"; + d.exec(` + INSERT INTO temporal_messages + (id, project_id, session_id, role, content, tokens, created_at) + VALUES ('invalid-source-rollup', '${sourceId}', '${sessionId}', + 'user', 'source', 1.25, 100), + ('invalid-target-rollup', '${targetId}', '${sessionId}', + 'assistant', 'target', 2.5, 200); + `); + + expect(() => mergeProjectInternal(sourceId, targetId)).toThrow( + /rollup counter invalid/, + ); + expect( + d + .query("SELECT COUNT(*) AS count FROM projects WHERE id IN (?, ?)") + .get(sourceId, targetId), + ).toEqual({ count: 2 }); + expect( + d + .query( + "SELECT COUNT(*) AS count FROM session_rollup WHERE session_id = ?", + ) + .get(sessionId), + ).toEqual({ count: 2 }); + }); + + test("listProjects never caches an uncommitted nested merge", () => { + const sourcePath = "/test/merge-list-cache/source"; + const targetPath = "/test/merge-list-cache/target"; + const sourceId = ensureProject(sourcePath); + const targetId = ensureProject(targetPath); + invalidateProjectsCache(); + expect(listProjects().some((project) => project.id === sourceId)).toBe( + true, + ); + + expect(() => + withSavepoint("outer_merge_rollback", () => { + mergeProjectInternal(sourceId, targetId); + expect(listProjects().some((project) => project.id === sourceId)).toBe( + false, + ); + throw new Error("force outer rollback"); + }), + ).toThrow("force outer rollback"); + + expect(projectId(sourcePath)).toBe(sourceId); + expect(canonicalProjectId(sourceId)).toBe(sourceId); + expect( + db() + .query("SELECT project_id FROM project_id_aliases WHERE retired_id = ?") + .get(sourceId), + ).toBeNull(); + expect(listProjects().some((project) => project.id === sourceId)).toBe( + true, + ); + }); + + test("project mutation listeners fan out without replacing data cache invalidation", () => { + listProjects(); + const mutations: string[] = []; + const unsubscribe = onProjectMutation((mutation) => { + mutations.push(mutation.type); + }); + try { + const project = ensureProject( + `/test/listener-fanout-${crypto.randomUUID()}`, + ); + expect(mutations).toEqual(["create"]); + expect(listProjects().some(({ id }) => id === project)).toBe(true); + } finally { + unsubscribe(); + } }); test("recoverMissingObjects creates project_path_aliases when missing", () => { @@ -1210,6 +2178,28 @@ describe("db", () => { expect(after?.name).toBe("project_path_aliases"); }); + test("recoverMissingObjects creates project_id_aliases when missing", () => { + const d = db(); + d.exec("DROP TABLE IF EXISTS project_id_aliases"); + expect( + d + .query( + "SELECT name FROM sqlite_master WHERE type='table' AND name='project_id_aliases'", + ) + .get(), + ).toBeNull(); + + close(); + const fresh = db(); + expect( + fresh + .query( + "SELECT name FROM sqlite_master WHERE type='table' AND name='project_id_aliases'", + ) + .get(), + ).toEqual({ name: "project_id_aliases" }); + }); + test("saveSessionCosts and loadSessionCosts round-trip", () => { const sid = `test-costs-${crypto.randomUUID()}`; const snapshot = { @@ -1511,6 +2501,7 @@ describe("db", () => { stableLtmText: "frozen stable LTM text", stableLtmTokens: 77, recallStore: JSON.stringify([["all:q", { toolUseId: "t1" }]]), + amnesia: true, dedupDecisions: JSON.stringify([["m1:p1", true]]), lastKnownMessageCount: 137, lastUpstream: JSON.stringify({ model: "gpt-test" }), @@ -1530,6 +2521,7 @@ describe("db", () => { expect(loaded?.recallStore).toBe( JSON.stringify([["all:q", { toolUseId: "t1" }]]), ); + expect(loaded?.amnesia).toBe(true); expect(loaded?.dedupDecisions).toBe(JSON.stringify([["m1:p1", true]])); // v43: persisted for accurate calibrated-delta estimation after restart. expect(loaded?.lastKnownMessageCount).toBe(137); @@ -2250,7 +3242,12 @@ describe("db", () => { const sidOther = `adopt-other-${crypto.randomUUID()}`; const sidEmpty = `adopt-empty-${crypto.randomUUID()}`; const fp = `fp-${crypto.randomUUID().slice(0, 8)}`; - saveSessionTracking(sidA, { fingerprint: fp, messageCount: 100 }); + saveSessionTracking(sidA, { + fingerprint: fp, + messageCount: 100, + projectPath: "/tmp/adopt-provisional", + projectPathProvisional: true, + }); saveSessionTracking(sidB, { fingerprint: fp, messageCount: 200, @@ -2272,12 +3269,14 @@ describe("db", () => { expect(bySid.get(sidA)?.message_count).toBe(100); expect(bySid.get(sidB)?.is_subagent).toBe(1); expect(bySid.get(sidA)?.is_subagent).toBe(0); + expect(bySid.get(sidA)?.project_path).toBe("/tmp/adopt-provisional"); + expect(bySid.get(sidA)?.project_path_provisional).toBe(1); // Empty query never matches the empty-fingerprint rows. expect(findSessionStatesByFingerprint("")).toEqual([]); }); - test("findSessionStatesByFingerprint can restrict legacy candidates to unowned rows", () => { + test("findSessionStatesByFingerprint can restrict candidates by owner", () => { const fp = `legacy-fp-${crypto.randomUUID().slice(0, 8)}`; const unowned = `legacy-unowned-${crypto.randomUUID()}`; const owned = `legacy-owned-${crypto.randomUUID()}`; @@ -2295,6 +3294,16 @@ describe("db", () => { (row) => row.session_id, ), ).toEqual([unowned]); + expect( + findSessionStatesByFingerprint(fp, { + credentialFingerprint: "credential-a", + }).map((row) => row.session_id), + ).toEqual([owned]); + expect( + findSessionStatesByFingerprint(fp, { + credentialFingerprint: "credential-b", + }), + ).toEqual([]); }); test("countMatchingTemporalIds counts only same-project, same-session ids", () => { diff --git a/packages/core/test/knowledge-transfers.test.ts b/packages/core/test/knowledge-transfers.test.ts index 253bc159..9e8ccbc1 100644 --- a/packages/core/test/knowledge-transfers.test.ts +++ b/packages/core/test/knowledge-transfers.test.ts @@ -1,6 +1,6 @@ import { describe, test, expect, beforeEach } from "vitest"; import { uuidv7 } from "uuidv7"; -import { db, ensureProject } from "../src/db"; +import { db, ensureProject, withSavepoint } from "../src/db"; import * as ltm from "../src/ltm"; import * as data from "../src/data"; import { runRecall } from "../src/recall"; @@ -224,6 +224,40 @@ describe("runRecall transfer gating", () => { expect(breakdown[0]?.recalled_in_project_id).toBe(fpid); }); + test("deferred transfer recording participates in the caller savepoint", async () => { + const id = createPromoted( + "Deferred zigzag indexing trick", + "Deferred zigzag indexing trick for sparse matrix traversal", + ); + const recordings: Array<() => void> = []; + + await runRecall({ + query: "deferred zigzag indexing sparse matrix", + scope: "all", + projectPath: FOREIGN, + sessionID: FOREIGN_SESSION, + knowledgeEnabled: true, + recordTransfers: true, + deferTransferRecording: (record) => recordings.push(record), + }); + + expect(recordings).toHaveLength(1); + expect(ltm.transferCount(id)).toBe(0); + expect(() => + withSavepoint("deferred_transfer_failure", () => { + for (const record of recordings) record(); + expect(ltm.transferCount(id)).toBe(1); + throw new Error("later persistence failed"); + }), + ).toThrow("later persistence failed"); + expect(ltm.transferCount(id)).toBe(0); + + withSavepoint("deferred_transfer_success", () => { + for (const record of recordings) record(); + }); + expect(ltm.transferCount(id)).toBe(1); + }); + test("recallById never records", async () => { const id = createPromoted("ById entry", "content for id lookup"); await runRecall({ diff --git a/packages/core/test/ltm-versioning.test.ts b/packages/core/test/ltm-versioning.test.ts index b38934ed..03b78be9 100644 --- a/packages/core/test/ltm-versioning.test.ts +++ b/packages/core/test/ltm-versioning.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import { db } from "../src/db"; +import { db, MIGRATIONS } from "../src/db"; import * as ltm from "../src/ltm"; const PROJECT = "/test/a2/versioning"; @@ -39,11 +39,11 @@ function ftsHits(token: string): number { } describe("A2 sub-PR 1: append-only knowledge scaffolding", () => { - test("schema version is 81", () => { + test("schema is at the latest version", () => { const v = db().query("SELECT version FROM schema_version").get() as { version: number; }; - expect(v.version).toBe(82); + expect(v.version).toBe(MIGRATIONS.length); }); test("create() defaults logical_id = id, version 1, current, not deleted", () => { diff --git a/packages/core/test/pattern-echo.test.ts b/packages/core/test/pattern-echo.test.ts index c0c24e8b..816afbaa 100644 --- a/packages/core/test/pattern-echo.test.ts +++ b/packages/core/test/pattern-echo.test.ts @@ -155,4 +155,87 @@ describe("pattern-echo cooldown", () => { await detectPatternEchoes({ ...base, distillId: "f2" }); expect(searchSpy).toHaveBeenCalledTimes(1); }); + + it("rolls back its cooldown when cancellation interrupts an armed attempt", async () => { + const pid = ensureProject(PROJECT); + insertDistill("c1", pid, "s-cancel"); + insertDistill("c2", pid, "s-cancel"); + const controller = new AbortController(); + let searchStarted!: () => void; + const started = new Promise((resolve) => { + searchStarted = resolve; + }); + searchSpy.mockImplementationOnce( + () => + new Promise((_, reject) => { + searchStarted(); + controller.signal.addEventListener( + "abort", + () => reject(controller.signal.reason), + { once: true }, + ); + }), + ); + const base = { + observations: "obs", + projectPath: PROJECT, + sessionID: "s-cancel", + llm: stubLLM(), + }; + + const cancelled = detectPatternEchoes({ + ...base, + distillId: "c1", + signal: controller.signal, + }); + await started; + controller.abort(new DOMException("session reset", "AbortError")); + await cancelled; + + await detectPatternEchoes({ ...base, distillId: "c2" }); + expect(searchSpy).toHaveBeenCalledTimes(2); + }); + + it("does not clear a newer cooldown owner when an older attempt cancels", async () => { + vi.useFakeTimers(); + try { + const pid = ensureProject(PROJECT); + insertDistill("o1", pid, "s-owner"); + insertDistill("o2", pid, "s-owner"); + insertDistill("o3", pid, "s-owner"); + const older = new AbortController(); + let rejectOlder!: (reason: unknown) => void; + searchSpy.mockImplementationOnce( + () => + new Promise((_, reject) => { + rejectOlder = reject; + }), + ); + const base = { + observations: "obs", + projectPath: PROJECT, + sessionID: "s-owner", + llm: stubLLM(), + }; + + const olderAttempt = detectPatternEchoes({ + ...base, + distillId: "o1", + signal: older.signal, + }); + await vi.waitFor(() => expect(searchSpy).toHaveBeenCalledTimes(1)); + vi.advanceTimersByTime(PATTERN_COOLDOWN_MS + 1); + await detectPatternEchoes({ ...base, distillId: "o2" }); + expect(searchSpy).toHaveBeenCalledTimes(2); + + older.abort(new DOMException("older generation reset", "AbortError")); + rejectOlder(older.signal.reason); + await olderAttempt; + + await detectPatternEchoes({ ...base, distillId: "o3" }); + expect(searchSpy).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); }); diff --git a/packages/core/test/warmup-histogram.test.ts b/packages/core/test/warmup-histogram.test.ts new file mode 100644 index 00000000..b932f047 --- /dev/null +++ b/packages/core/test/warmup-histogram.test.ts @@ -0,0 +1,283 @@ +import { describe, expect, test, vi } from "vitest"; +import { + MAX_WARMUP_HISTOGRAM_TOTAL, + WARMUP_HISTOGRAM_BIN_COUNT, + decodeWarmupHistogram, + emptyWarmupHistogramCounts, + encodeWarmupHistogram, + mergeWarmupHistogramCounts, + normalizeWarmupHistogram, +} from "../src/warmup-histogram"; + +function counts(...entries: Array<[number, bigint]>): bigint[] { + const result = emptyWarmupHistogramCounts(); + for (const [index, count] of entries) result[index] = count; + return result; +} + +describe("warmup histogram persistence", () => { + test("normalization preserves every nonzero bucket with a safe exact total", () => { + const normalized = normalizeWarmupHistogram( + counts([0, BigInt(Number.MAX_SAFE_INTEGER) - 1n], [1, 1n], [2, 1n]), + ); + + expect(normalized.counts.slice(0, 3).every((count) => count > 0)).toBe( + true, + ); + expect(normalized.counts.reduce((sum, count) => sum + count, 0)).toBe( + MAX_WARMUP_HISTOGRAM_TOTAL, + ); + expect(normalized.total).toBe(MAX_WARMUP_HISTOGRAM_TOTAL); + }); + + test("encoded exact weights make chained merges grouping-independent", () => { + const large = counts([0, BigInt(Number.MAX_SAFE_INTEGER) - 1n], [1, 1n]); + const second = counts([2, 1n]); + const third = counts([3, 1n]); + + const leftEncoded = encodeWarmupHistogram( + mergeWarmupHistogramCounts(large, second), + ); + const left = decodeWarmupHistogram(leftEncoded.counts, leftEncoded.total); + const rightEncoded = encodeWarmupHistogram( + mergeWarmupHistogramCounts(second, third), + ); + const right = decodeWarmupHistogram( + rightEncoded.counts, + rightEncoded.total, + ); + expect(left).toBeDefined(); + expect(right).toBeDefined(); + + const leftGrouped = mergeWarmupHistogramCounts(left ?? [], third); + const rightGrouped = mergeWarmupHistogramCounts(large, right ?? []); + expect(leftGrouped).toEqual(rightGrouped); + expect(normalizeWarmupHistogram(leftGrouped).counts.slice(0, 4)).toEqual( + normalizeWarmupHistogram(rightGrouped).counts.slice(0, 4), + ); + expect( + normalizeWarmupHistogram(leftGrouped) + .counts.slice(0, 4) + .every((count) => count > 0), + ).toBe(true); + }); + + test("round-trips a legacy safe-integer row above the normalization limit", () => { + const legacy = Array.from({ length: WARMUP_HISTOGRAM_BIN_COUNT }, () => 0); + legacy[0] = Number.MAX_SAFE_INTEGER; + const decoded = decodeWarmupHistogram( + JSON.stringify(legacy), + String(Number.MAX_SAFE_INTEGER), + ); + expect(decoded?.[0]).toBe(BigInt(Number.MAX_SAFE_INTEGER)); + + const encoded = encodeWarmupHistogram(decoded ?? []); + expect(decodeWarmupHistogram(encoded.counts, encoded.total)).toEqual( + decoded, + ); + }); + + test("remains decodable when accepted weights gain a digit during merge", () => { + const boundary = 10n ** 1000n - 1n; + const first = encodeWarmupHistogram(counts([0, boundary], [1, 1n])); + const second = encodeWarmupHistogram(counts([0, boundary], [2, 1n])); + const firstWeights = decodeWarmupHistogram(first.counts, first.total); + const secondWeights = decodeWarmupHistogram(second.counts, second.total); + expect(firstWeights).toBeDefined(); + expect(secondWeights).toBeDefined(); + + const merged = encodeWarmupHistogram( + mergeWarmupHistogramCounts(firstWeights ?? [], secondWeights ?? []), + ); + const decoded = decodeWarmupHistogram(merged.counts, merged.total); + expect(decoded).toBeDefined(); + expect(decoded?.[0].toString()).toHaveLength(1001); + expect(decoded?.slice(0, 3).every((count) => count > 0n)).toBe(true); + }); + + test("compacts encoder output to the decoder storage budget", () => { + const huge = 10n ** 4000n; + const weights = Array.from( + { length: WARMUP_HISTOGRAM_BIN_COUNT }, + () => huge, + ); + const visible = normalizeWarmupHistogram(weights); + const oversized = JSON.stringify({ + v: 1, + counts: visible.counts, + exact: weights.map(String), + }); + expect(oversized.length).toBeGreaterThan(64 * 1024); + expect(decodeWarmupHistogram(oversized, visible.total)).toBeUndefined(); + + const encoded = encodeWarmupHistogram(weights); + expect(encoded.counts.length).toBeLessThanOrEqual(64 * 1024); + const decoded = decodeWarmupHistogram(encoded.counts, encoded.total); + expect(decoded).toEqual(encoded.weights); + expect(decoded?.every((count) => count > 0n)).toBe(true); + + const merged = encodeWarmupHistogram( + mergeWarmupHistogramCounts(decoded ?? [], decoded ?? []), + ); + expect(decodeWarmupHistogram(merged.counts, merged.total)).toEqual( + merged.weights, + ); + }); + + test("retains compacted history magnitude when a smaller row is merged later", () => { + const huge = 10n ** 4000n; + const history = Array.from( + { length: WARMUP_HISTOGRAM_BIN_COUNT }, + (_, index) => (index < 17 ? huge : 0n), + ); + const encoded = encodeWarmupHistogram(history); + const retained = decodeWarmupHistogram(encoded.counts, encoded.total); + expect(retained?.[0].toString().length).toBeGreaterThan(3000); + + const later = counts([20, 10n ** 3000n]); + const merged = normalizeWarmupHistogram( + mergeWarmupHistogramCounts(retained ?? [], later), + ); + expect(merged.counts[0]).toBeGreaterThan(merged.counts[20]); + }); + + test.each([0, 64 * 1024])("rejects invalid compacted scale %i", (scale) => { + const huge = 10n ** 4000n; + const encoded = encodeWarmupHistogram( + Array.from({ length: WARMUP_HISTOGRAM_BIN_COUNT }, () => huge), + ); + const stored = JSON.parse(encoded.counts) as { scale: number }; + stored.scale = scale; + expect( + decodeWarmupHistogram(JSON.stringify(stored), encoded.total), + ).toBeUndefined(); + }); + + test("rejects noncanonical exact strings without another mismatch", () => { + const encoded = encodeWarmupHistogram( + counts([0, BigInt(Number.MAX_SAFE_INTEGER)], [1, 1n]), + ); + const stored = JSON.parse(encoded.counts) as { + exact: string[]; + }; + stored.exact[0] = `0${stored.exact[0]}`; + expect( + decodeWarmupHistogram(JSON.stringify(stored), encoded.total), + ).toBeUndefined(); + }); + + test("rejects incorrect visible counts with the correct visible total", () => { + const encoded = encodeWarmupHistogram( + counts([0, BigInt(Number.MAX_SAFE_INTEGER)], [1, 1n]), + ); + const stored = JSON.parse(encoded.counts) as { + counts: number[]; + }; + stored.counts[0]--; + stored.counts[1]++; + expect( + decodeWarmupHistogram(JSON.stringify(stored), encoded.total), + ).toBeUndefined(); + }); + + test("rejects an incorrect exact total with otherwise valid fields", () => { + const encoded = encodeWarmupHistogram( + counts([0, BigInt(Number.MAX_SAFE_INTEGER)], [1, 1n]), + ); + expect( + decodeWarmupHistogram(encoded.counts, encoded.total + 1), + ).toBeUndefined(); + }); + + test("rejects an unsafe numeric total that otherwise matches legacy counts", () => { + const legacy = Array.from({ length: WARMUP_HISTOGRAM_BIN_COUNT }, () => 0); + legacy[0] = Number.MAX_SAFE_INTEGER; + legacy[1] = 1; + expect( + decodeWarmupHistogram( + JSON.stringify(legacy), + Number.MAX_SAFE_INTEGER + 1, + ), + ).toBeUndefined(); + }); + + test("rejects negative bigint weights before normalization", () => { + expect(() => normalizeWarmupHistogram(counts([0, -1n], [1, 2n]))).toThrow( + "invalid warmup histogram counts", + ); + }); + + test("rejects an exact vector with the wrong shape", () => { + const encoded = encodeWarmupHistogram( + counts([0, BigInt(Number.MAX_SAFE_INTEGER)]), + ); + const stored = JSON.parse(encoded.counts) as { exact: string[] }; + stored.exact.pop(); + expect( + decodeWarmupHistogram(JSON.stringify(stored), encoded.total), + ).toBeUndefined(); + expect(() => + normalizeWarmupHistogram(Array.from({ length: 20 }, () => 0n)), + ).toThrow("invalid warmup histogram counts"); + }); + + test("rejects an oversized exact vector before bigint expansion", () => { + const bigint = vi.spyOn(globalThis, "BigInt"); + try { + const raw = JSON.stringify({ + v: 2, + counts: Array.from({ length: WARMUP_HISTOGRAM_BIN_COUNT }, () => 0), + exact: Array.from({ length: 1000 }, () => "1"), + scale: 100, + }); + expect(decodeWarmupHistogram(raw, "0")).toBeUndefined(); + expect(bigint).toHaveBeenCalledTimes(WARMUP_HISTOGRAM_BIN_COUNT); + expect(bigint.mock.calls.every(([value]) => value === 0)).toBe(true); + } finally { + bigint.mockRestore(); + } + }); + + test.each([ + ["invalid JSON", "{", "0"], + ["wrong legacy length", "[]", "0"], + [ + "negative legacy bucket", + JSON.stringify([-1, 2, ...Array.from({ length: 19 }, () => 0)]), + "1", + ], + [ + "unsafe legacy bucket", + JSON.stringify([ + Number.MAX_SAFE_INTEGER + 1, + ...Array.from({ length: 20 }, () => 0), + ]), + String(Number.MAX_SAFE_INTEGER + 1), + ], + [ + "legacy total mismatch", + JSON.stringify([1, ...Array.from({ length: 20 }, () => 0)]), + "2", + ], + [ + "unknown exact version", + JSON.stringify({ + v: 3, + counts: Array.from({ length: 21 }, () => 0), + exact: Array.from({ length: 21 }, () => "0"), + }), + "0", + ], + [ + "invalid exact bucket", + JSON.stringify({ + v: 1, + counts: Array.from({ length: 21 }, () => 0), + exact: ["-1", ...Array.from({ length: 20 }, () => "0")], + }), + "0", + ], + ] as const)("rejects %s", (_name, rawCounts, rawTotal) => { + expect(decodeWarmupHistogram(rawCounts, rawTotal)).toBeUndefined(); + }); +}); diff --git a/packages/gateway/src/cache-warmer.ts b/packages/gateway/src/cache-warmer.ts index a542b099..d251e838 100644 --- a/packages/gateway/src/cache-warmer.ts +++ b/packages/gateway/src/cache-warmer.ts @@ -27,6 +27,8 @@ import { log, config as loreConfig, db, + databaseInTransaction, + canonicalProjectId, projectId, getKV, setKV, @@ -38,6 +40,11 @@ import { estimateMetaDistillCostPerCall, getPrefixChurnRate, PREFIX_CHURN_WARM_BLOCK, + decodeWarmupHistogram, + emptyWarmupHistogramCounts, + encodeWarmupHistogram, + mergeWarmupHistogramCounts, + normalizeWarmupHistogram, } from "@loreai/core"; import { applyUpstreamExtraHeaders, @@ -704,6 +711,10 @@ const globalHistograms = new Map(); * Creates an empty histogram if none exists for the given pid. */ export function getGlobalHistogram(pid: string): InterTurnHistogram { + reconcileProjectMerges(); + const canonical = canonicalProjectId(pid, { committed: true }); + if (!canonical) return createHistogram(); + pid = canonical; let hist = globalHistograms.get(pid); if (!hist) { hist = createHistogram(); @@ -2560,6 +2571,85 @@ export function creditWarmupHit( /** Tracks which project IDs have been modified since last flush. */ const dirtyProjects = new Set(); +/** Unflushed observations, tracked separately so project merges can rebase them. */ +const dirtyHistograms = new Map(); + +function loadPersistedHistogramCounts(pid: string): bigint[] { + let merged = emptyWarmupHistogramCounts(); + const rows = db() + .query( + "SELECT counts, CAST(total AS TEXT) AS total FROM warmup_histograms WHERE project_id = ?", + ) + .all(pid) as Array<{ counts: string; total: string }>; + for (const row of rows) { + const counts = decodeWarmupHistogram(row.counts, row.total); + if (counts) merged = mergeWarmupHistogramCounts(merged, counts); + } + return merged; +} + +function loadPersistedHistogram(pid: string): InterTurnHistogram { + const merged = normalizeWarmupHistogram(loadPersistedHistogramCounts(pid)); + return merged; +} + +function addHistogram( + target: InterTurnHistogram, + source: InterTurnHistogram, +): void { + const merged = normalizeWarmupHistogram( + mergeWarmupHistogramCounts( + target.counts.map(BigInt), + source.counts.map(BigInt), + ), + ); + target.counts = merged.counts; + target.total = merged.total; +} + +function reconcileProjectMerges(): void { + const database = db(); + if (databaseInTransaction(database)) return; + const pendingByTarget = new Map(); + const affectedTargets = new Set(); + const trackedIds = new Set([ + ...globalHistograms.keys(), + ...dirtyHistograms.keys(), + ]); + for (const id of trackedIds) { + const targetId = canonicalProjectId(id); + if (targetId === id) continue; + if (!targetId) { + // Hide a deleted project's stale aggregate immediately, but retain its + // pending delta until flush confirms the deletion under a write lock. + globalHistograms.delete(id); + continue; + } + affectedTargets.add(targetId); + const dirty = dirtyHistograms.get(id); + if (dirty) { + const pending = pendingByTarget.get(targetId) ?? createHistogram(); + addHistogram(pending, dirty); + pendingByTarget.set(targetId, pending); + } + globalHistograms.delete(id); + dirtyHistograms.delete(id); + dirtyProjects.delete(id); + } + + for (const targetId of affectedTargets) { + const persisted = loadPersistedHistogram(targetId); + const pending = pendingByTarget.get(targetId) ?? createHistogram(); + const targetDirty = dirtyHistograms.get(targetId); + if (targetDirty) addHistogram(pending, targetDirty); + addHistogram(persisted, pending); + globalHistograms.set(targetId, persisted); + if (pending.total > 0) { + dirtyHistograms.set(targetId, pending); + dirtyProjects.add(targetId); + } + } +} /** * Load persisted global histograms for a project from SQLite. @@ -2577,35 +2667,18 @@ const dirtyProjects = new Set(); * histogram. New data is written under the "all" time_slot key. */ export function loadGlobalHistograms(projectPath: string): string | undefined { - const pid = projectId(projectPath); - if (!pid) return undefined; // project not yet in DB — nothing to load + reconcileProjectMerges(); + if (databaseInTransaction(db())) return undefined; + const resolvedPid = projectId(projectPath); + if (!resolvedPid) return undefined; // project not yet in DB — nothing to load + const pid = canonicalProjectId(resolvedPid); + if (!pid) return undefined; if (globalHistograms.has(pid)) return pid; // already loaded - const merged = createHistogram(); - + let merged = createHistogram(); try { - const rows = db() - .query( - "SELECT time_slot, counts, total FROM warmup_histograms WHERE project_id = ?", - ) - .all(pid) as Array<{ time_slot: string; counts: string; total: number }>; - - for (const row of rows) { - try { - const counts = JSON.parse(row.counts) as number[]; - if (Array.isArray(counts) && counts.length === BIN_COUNT) { - // Merge this row into the single histogram (handles both old - // slot-segmented rows and the new "all" row). - for (let i = 0; i < BIN_COUNT; i++) { - merged.counts[i] += counts[i]; - } - merged.total += row.total; - } - } catch { - // Corrupt JSON — skip this row - } - } + merged = loadPersistedHistogram(pid); log.info( `cache-warmer: loaded global histogram for project=${projectPath.slice(-30)} ` + @@ -2631,35 +2704,63 @@ export function loadGlobalHistograms(projectPath: string): string | undefined { * on the next load. */ export function flushGlobalHistograms(): void { + reconcileProjectMerges(); if (dirtyProjects.size === 0) return; const d = db(); const now = Date.now(); - for (const pid of dirtyProjects) { - const hist = globalHistograms.get(pid); - if (!hist) continue; + const projectsToFlush = Array.from(dirtyProjects); + for (const pid of projectsToFlush) { + const pending = dirtyHistograms.get(pid); + if (!pending) { + dirtyHistograms.delete(pid); + dirtyProjects.delete(pid); + continue; + } try { // Atomic: delete old slot rows + upsert the unified "all" row. // Without the transaction, a crash between DELETE and INSERT // would lose all histogram data for this project. - d.exec("BEGIN"); + d.exec("BEGIN IMMEDIATE"); + let resolvedPid: string | undefined; + let hist: InterTurnHistogram | undefined; try { - // Delete old slot-segmented rows (backward compat cleanup) - d.query( - "DELETE FROM warmup_histograms WHERE project_id = ? AND time_slot != 'all'", - ).run(pid); - - d.query( - `INSERT INTO warmup_histograms (project_id, time_slot, counts, total, updated_at) - VALUES (?, 'all', ?, ?, ?) - ON CONFLICT(project_id, time_slot) DO UPDATE SET - counts = excluded.counts, - total = excluded.total, - updated_at = excluded.updated_at`, - ).run(pid, JSON.stringify(hist.counts), hist.total, now); + // Re-read after taking the write lock, then apply only observations + // buffered by this process. This prevents stale cache state from + // overwriting a merge or another gateway's committed history. + resolvedPid = canonicalProjectId(pid); + if (resolvedPid) { + const exact = mergeWarmupHistogramCounts( + loadPersistedHistogramCounts(resolvedPid), + pending.counts.map(BigInt), + ); + const encoded = encodeWarmupHistogram(exact); + hist = normalizeWarmupHistogram(encoded.weights); + // Delete old slot-segmented rows (backward compat cleanup) + d.query( + "DELETE FROM warmup_histograms WHERE project_id = ? AND time_slot != 'all'", + ).run(resolvedPid); + + d.query( + `INSERT INTO warmup_histograms (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, ?, ?) + ON CONFLICT(project_id, time_slot) DO UPDATE SET + counts = excluded.counts, + total = excluded.total, + updated_at = excluded.updated_at`, + ).run(resolvedPid, encoded.counts, encoded.total, now); + } d.exec("COMMIT"); + if (resolvedPid && hist) { + if (resolvedPid !== pid) globalHistograms.delete(pid); + globalHistograms.set(resolvedPid, hist); + } else { + globalHistograms.delete(pid); + } + dirtyHistograms.delete(pid); + dirtyProjects.delete(pid); } catch (e) { d.exec("ROLLBACK"); throw e; @@ -2668,8 +2769,6 @@ export function flushGlobalHistograms(): void { log.warn(`cache-warmer: failed to flush histogram:`, e); } } - - dirtyProjects.clear(); } /** @@ -2683,6 +2782,9 @@ export function recordGlobalGap(projectPath: string, gapMs: number): void { if (!pid) return; // project not yet in DB — skip const hist = getGlobalHistogram(pid); recordGap(hist, gapMs); + const dirty = dirtyHistograms.get(pid) ?? createHistogram(); + recordGap(dirty, gapMs); + dirtyHistograms.set(pid, dirty); dirtyProjects.add(pid); } @@ -2695,6 +2797,7 @@ export function getGlobalHistogramsSnapshot(): ReadonlyMap< string, InterTurnHistogram > { + reconcileProjectMerges(); return globalHistograms; } @@ -2723,5 +2826,6 @@ export function _resetForTest(): void { } globalHistograms.clear(); dirtyProjects.clear(); + dirtyHistograms.clear(); authDisabledSessions.clear(); } diff --git a/packages/gateway/src/cli/agents.ts b/packages/gateway/src/cli/agents.ts index db8735a8..b058751d 100644 --- a/packages/gateway/src/cli/agents.ts +++ b/packages/gateway/src/cli/agents.ts @@ -83,7 +83,17 @@ function isLoopbackHost(hostname: string): boolean { if (h === "localhost" || h === "::1" || h === "0.0.0.0" || h === "::") return true; // 127.0.0.0/8 - return /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(h); + if (/^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(h)) return true; + // URL canonicalizes IPv4-mapped IPv6, e.g. ::ffff:127.0.0.1 becomes + // ::ffff:7f00:1. Decode its high IPv4 octet so mapped loopback/wildcard + // addresses cannot bypass the self-proxy guard. + const mapped = h.match( + /^(?:::ffff:|0:0:0:0:0:ffff:)([0-9a-f]{1,4}):([0-9a-f]{1,4})$/, + ); + if (!mapped) return false; + const high = Number.parseInt(mapped[1], 16); + const low = Number.parseInt(mapped[2], 16); + return high >>> 8 === 127 || (high === 0 && low === 0); } /** @@ -93,8 +103,11 @@ function isLoopbackHost(hostname: string): boolean { * Returns the first defined `upstreamEnvVars` value that points somewhere * OTHER than a loopback host, so we never "adopt" the gateway pointing at * itself and re-launches through `lore run` stay idempotent even if the - * gateway restarts on a different port. Returns undefined when the agent has + * gateway restarts on a different port. Returns null when the agent has * no adoptable base-URL var, none is set, or the only value is loopback. + * Throws when a non-empty configured value cannot be routed safely; callers + * must not mistake an invalid override for an absent one and launch with the + * override's credential against Lore's default upstream. */ export function captureUserUpstream( agent: AgentDef, @@ -105,6 +118,11 @@ export function captureUserUpstream( for (const key of agent.upstreamEnvVars) { const raw = env[key]; if (!raw) continue; + const invalid = () => { + throw new Error( + `${agent.displayName} has an unsafe or invalid upstream URL in ${key}`, + ); + }; // Strip control chars (CR/LF/etc.) up front — a newline in a base-URL env // var would otherwise ride through into an injected header (CRLF header // smuggling). `new URL()` tolerates an embedded newline, so we cannot rely @@ -112,20 +130,25 @@ export function captureUserUpstream( // normalize at the source so every consumer sees a clean value. // oxlint-disable-next-line no-control-regex -- intentional control-character sanitization const trimmed = raw.replace(/[\x00-\x1f\x7f]/g, "").trim(); - if (!trimmed) continue; + if (!trimmed) invalid(); // A real base URL has no internal whitespace. Reject anything with an // interior space/tab — after control-char stripping, a CRLF-smuggling // payload like "https://host/\nX-Api-Key: stolen" collapses to a value // with an interior space, which must not be adopted. - if (/\s/.test(trimmed)) continue; + if (/\s/.test(trimmed)) invalid(); let parsed: URL; try { parsed = new URL(trimmed); } catch { + invalid(); continue; } - // Only adopt a real http(s) URL that isn't the gateway itself. - if (parsed.protocol !== "http:" && parsed.protocol !== "https:") continue; + // Only adopt a real http(s) URL that isn't the gateway itself. Credentials + // belong in the agent's auth env, never in a base URL that may be forwarded + // in a routing header or surfaced in diagnostics. + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") invalid(); + if (parsed.username || parsed.password || parsed.search || parsed.hash) + invalid(); // Reject ANY loopback host, not just the exact gateway origin: the gateway // may restart on a different port (contention), so a stale // ANTHROPIC_BASE_URL=http://127.0.0.1: must not be adopted (would @@ -141,7 +164,9 @@ export function captureUserUpstream( /** * Sanitize + validate a base-URL env value. Returns a clean http(s) URL string * (control chars stripped, interior whitespace rejected — see the CRLF note in - * captureUserUpstream) or null if unusable. Loopback is NOT rejected here: + * captureUserUpstream) or null if unusable. Userinfo, queries, and fragments + * are rejected because gateway route normalization cannot preserve them. + * Loopback is NOT rejected here: * unlike `lore run`, `lore import` starts no long-lived gateway to point at, so * a loopback base URL (a running gateway the user already has) is a legitimate * extraction upstream. @@ -155,6 +180,8 @@ function cleanBaseUrl(raw: string | undefined): string | null { const parsed = new URL(trimmed); if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null; + if (parsed.username || parsed.password || parsed.search || parsed.hash) + return null; } catch { return null; } @@ -206,11 +233,12 @@ export function captureUserEnvCredential( // Capture the paired base URL (first defined + valid), if any. let upstreamUrl: string | null = null; for (const key of agent.upstreamEnvVars ?? []) { - const clean = cleanBaseUrl(env[key]); - if (clean) { - upstreamUrl = clean; - break; - } + const raw = env[key]; + if (!raw) continue; + const clean = cleanBaseUrl(raw); + if (!clean) return null; + upstreamUrl = clean; + break; } return { ...picked, upstreamUrl, envVarName }; } diff --git a/packages/gateway/src/cli/run.ts b/packages/gateway/src/cli/run.ts index 21c95d39..cc31981f 100644 --- a/packages/gateway/src/cli/run.ts +++ b/packages/gateway/src/cli/run.ts @@ -86,10 +86,20 @@ export interface AdoptedUpstream { agentDisplayName: string; } +/** Render an adopted base URL without exposing query-string credentials. */ +export function formatUpstreamForLog(url: string): string { + try { + const parsed = new URL(url); + const query = parsed.search ? "?" : ""; + return `${parsed.origin}${parsed.pathname}${query}`; + } catch { + return ""; + } +} + /** * Map an agent's wire protocol to the gateway env var that overrides the - * default upstream for that protocol. Gemini has no such default knob - * (the native endpoint is fixed), so it relies purely on the injected header. + * default upstream for that protocol. Gemini has no such default knob. */ function gatewayUpstreamEnvKey( wireProtocol: NonNullable, @@ -109,17 +119,18 @@ function gatewayUpstreamEnvKey( * env) so the gateway proxies THERE instead of the hardcoded default. Called * BEFORE `startGateway`/`loadConfig` so the gateway picks up the override. * - * Two mechanisms, applied together: + * Two mechanisms, applied together where the agent supports them: * 1. Set the gateway's own `LORE_UPSTREAM_` process env — the - * in-process gateway reads this at `loadConfig()`, so EVERY agent (even - * header-less ones like Gemini/Copilot) gets routed to the user's host. + * in-process gateway reads this at `loadConfig()`, so header-less agents + * with a configurable protocol default get routed to the user's host. * 2. Return an `AdoptedUpstream` so the per-agent launch env can ALSO inject * `X-Lore-Upstream-URL` (+ `X-Lore-Provider` for known hosts) — this is * what flips the wire protocol/auth-scheme for a known provider that * differs from the ingress shape (e.g. Claude Code → OpenRouter, which is * an OpenAI-protocol provider reached via an Anthropic-shape client). * - * Returns null when the user hasn't overridden the agent's base URL. + * Returns null when the user hasn't overridden the agent's base URL. Throws + * when an override exists but Lore has no safe routing mechanism for it. */ export function applyUpstreamAdoption( agent: AgentDef, @@ -128,6 +139,11 @@ export function applyUpstreamAdoption( const captured = captureUserUpstream(agent, gatewayUrl); if (!captured) return null; const gatewayEnvKey = gatewayUpstreamEnvKey(captured.wireProtocol); + if (!gatewayEnvKey && captured.wireProtocol !== "anthropic") { + throw new Error( + `${agent.displayName} cannot safely route its configured upstream through Lore`, + ); + } const providerID = providerForUpstreamOrigin(captured.url); // Set the gateway default upstream for this protocol, UNLESS the user has // explicitly set it already (their explicit LORE_UPSTREAM_* wins). @@ -153,7 +169,8 @@ export function applyUpstreamAdoption( * Remote-mode adoption: the remote gateway owns its own config, so we do NOT * set any local `LORE_UPSTREAM_*` env. We only compute the `AdoptedUpstream` * so the launch env can inject `X-Lore-Upstream-URL`/`X-Lore-Provider`, which - * the remote gateway honors per request. Returns null when nothing to adopt. + * the remote gateway honors per request. Returns null when nothing to adopt + * and throws when the selected agent cannot transport those headers. */ export function adoptForRemote( agent: AgentDef, @@ -161,6 +178,11 @@ export function adoptForRemote( ): AdoptedUpstream | null { const captured = captureUserUpstream(agent, gatewayUrl); if (!captured) return null; + if (captured.wireProtocol !== "anthropic") { + throw new Error( + `${agent.displayName} cannot safely route its configured upstream through a remote Lore gateway`, + ); + } return { url: captured.url, gatewayEnvKey: "", @@ -174,9 +196,10 @@ export function adoptForRemote( * agents that forward `ANTHROPIC_CUSTOM_HEADERS` to the gateway (Claude Code / * Pi). Sets `X-Lore-Upstream-URL` and, for a known host, `X-Lore-Provider`. * - * Agents without a header-forwarding mechanism (Codex/Hermes/Copilot/Gemini) - * still get routed via the `LORE_UPSTREAM_` gateway env set in - * `applyUpstreamAdoption`; the header is a no-op for them. + * Agents without a header-forwarding mechanism (Codex/Hermes/Copilot) get + * routed only via the startup-time `LORE_UPSTREAM_` gateway env set + * in `applyUpstreamAdoption`; reusing a gateway is therefore rejected for + * their adopted upstreams. Gemini has no equivalent default and fails closed. */ export function injectAdoptionHeaders( agent: AgentDef, @@ -400,10 +423,20 @@ export async function commandRun( // shift on conflict, but the user's ANTHROPIC_BASE_URL points at their own // provider host, not loopback, so the guard just needs a loopback origin. const prospectiveUrl = `http://127.0.0.1:${config.port}`; - adopted = applyUpstreamAdoption(selection.def, prospectiveUrl); + try { + adopted = applyUpstreamAdoption(selection.def, prospectiveUrl); + } catch (err) { + console.error( + `[lore] ${err instanceof Error ? err.message : String(err)}.`, + ); + console.error( + "[lore] Remove the agent's custom base URL or configure Lore's upstream explicitly.", + ); + return safeExit(1); + } if (adopted) { console.log( - `[lore] Adopting your ${adopted.agentDisplayName} upstream: ${adopted.url}` + + `[lore] Adopting your ${adopted.agentDisplayName} upstream: ${formatUpstreamForLog(adopted.url)}` + (adopted.providerID ? ` (provider: ${adopted.providerID})` : ""), ); } @@ -434,7 +467,17 @@ export async function commandRun( console.log(`[lore] Using remote gateway at ${gatewayUrl}`); // In remote mode, adopt via header injection only (no local gateway env). if (selection?.def) { - adopted = adoptForRemote(selection.def, gatewayUrl); + try { + adopted = adoptForRemote(selection.def, gatewayUrl); + } catch (err) { + console.error( + `[lore] ${err instanceof Error ? err.message : String(err)}.`, + ); + console.error( + "[lore] Remove the agent's custom base URL or configure the remote gateway's upstream.", + ); + return safeExit(1); + } } } else { // Local mode: start (or reuse) a local gateway. @@ -455,6 +498,18 @@ export async function commandRun( console.log(`[lore] Gateway listening on ${gatewayUrl}`); } else { console.log(`[lore] Reusing existing gateway at ${gatewayUrl}`); + if ( + adopted?.gatewayEnvKey && + selection?.def?.wireProtocol !== "anthropic" + ) { + console.error( + `[lore] Cannot adopt your ${adopted.agentDisplayName} upstream when reusing an existing gateway.`, + ); + console.error( + `[lore] Stop the existing gateway or start it with ${adopted.gatewayEnvKey} configured.`, + ); + return safeExit(1); + } } } console.log(`[lore] Dashboard: ${gatewayUrl}/ui`); diff --git a/packages/gateway/src/idle.ts b/packages/gateway/src/idle.ts index d0927669..b7641cb1 100644 --- a/packages/gateway/src/idle.ts +++ b/packages/gateway/src/idle.ts @@ -433,6 +433,8 @@ export function startIdleScheduler( doIdleWork: (sessionID: string, state: SessionState) => Promise, /** Optional callback to clean up pipeline-level satellite Maps when a session is evicted. */ onEvict?: (sessionID: string) => void, + /** Optional lifecycle gate for active work owned outside the idle module. */ + isExternallyActive?: (sessionID: string) => boolean, ): () => void { const inProgress = new Set(); const warmupInProgress = new Set(); @@ -614,6 +616,7 @@ export function startIdleScheduler( // --- Idle work (distillation, curation, etc.) --- for (const [sessionID, state] of sessions) { + if (state.amnesia) continue; if (inProgress.has(sessionID)) continue; if (now - state.lastRequestTime < timeoutMs) continue; @@ -715,6 +718,7 @@ export function startIdleScheduler( warmupInProgress, now, onEvict, + isExternallyActive, ); // --- Cache warming (separate from idle work — fires before TTL expiry) --- @@ -732,6 +736,7 @@ export function startIdleScheduler( for (const [sessionID, state] of sessions) { if (!warmingGloballyEnabled) break; + if (state.amnesia) continue; if (warmupInProgress.has(sessionID)) continue; // Skip sessions with stale auth credentials — warmup would just 401 @@ -848,6 +853,7 @@ export function evictIdleSessions( warmupInProgress: ReadonlySet, now: number, onEvict?: (sessionID: string) => void, + isExternallyActive?: (sessionID: string) => boolean, ): number { const evictionTimeoutMs = config.sessionEvictionTimeoutSeconds * 1000; let evicted = 0; @@ -856,6 +862,7 @@ export function evictIdleSessions( if (evictionTimeoutMs <= 0) break; // eviction disabled if (inProgress.has(sessionID)) continue; // don't evict during active idle work if (warmupInProgress.has(sessionID)) continue; + if (isExternallyActive?.(sessionID)) continue; // Sub-agent sessions are ephemeral — evict faster const timeout = state.isSubagent ? Math.min(evictionTimeoutMs, SUBAGENT_EVICTION_MS) diff --git a/packages/gateway/src/pipeline.ts b/packages/gateway/src/pipeline.ts index 5eb26d42..79ad3fc8 100644 --- a/packages/gateway/src/pipeline.ts +++ b/packages/gateway/src/pipeline.ts @@ -22,7 +22,9 @@ import { recordCacheBustObservation, findSessionStatesByFingerprint, countMatchingTemporalIds, + getGitRemote, projectId, + resolveProjectByRemoteOrPath, projectGitRemote, mergeProjectInternal, isUnattributedProjectPath, @@ -118,7 +120,6 @@ import type { GatewayConfig } from "./config"; import { getProjectPath, extractGitRemoteHeader, - extractProjectHeader, resolveUpstreamRoute, extractUpstreamUrlHeader, extractUpstreamPathHeader, @@ -141,7 +142,7 @@ import { KNOWN_SESSION_HEADERS, extractKnownSessionHeader, learnHeaders, - findRotationPredecessor, + observeHeaderValues, isCredentialHeaderName, } from "./session"; import { @@ -190,6 +191,11 @@ import { formatResponsesEvent, makeResponsesAccState, mapStatusFromStopReason, + isSupportedResponsesOutputItemType, + isValidResponsesOutputItemStatus, + responsesDoneItemMatchesAdded, + responsesTerminalItemMatches, + ResponsesTerminalError, type ResponsesAccState, } from "./stream/openai-responses"; import { @@ -631,6 +637,147 @@ export function setBeforeUpstreamCaptureForTest( beforeUpstreamCaptureForTest = hook; } +/** Test-only observer for pinning post-response lifecycle ordering. */ +let postResponseStartObserver: (() => void) | undefined; +let recallPersistenceCommitObserver: (() => void) | undefined; +let pipelineResetPauseForTest: Promise | undefined; +let pipelinePreUpstreamPauseForTest: + | { pause: Promise; onWait: () => void } + | undefined; +let provisionalFinalizerPauseForTest: + | { pause: Promise; onWait: () => void } + | undefined; +let pipelineResetSettleTimeoutMs = 5000; +let pipelineResetInProgress = false; +let pipelineResetPromise: Promise | undefined; + +interface ActivePipelineRequest { + admissionKey: string; + abort: (reason: unknown) => void; + settled: Promise; + sessionIDs: Set; +} + +const activePipelineRequests = new Set(); +const detachedPipelineRequests = new Set(); +const DEFAULT_MAX_ACTIVE_PIPELINE_REQUESTS = 64; +const MAX_ACTIVE_PIPELINE_REQUESTS_PER_ADMISSION_KEY = 16; +const MAX_ACTIVE_PIPELINE_REQUESTS_PER_SESSION = 1; +const MAX_PENDING_SESSION_CLAIMS = 64; +const MAX_DETACHED_PIPELINE_REQUESTS = 64; +let maxActivePipelineRequests = DEFAULT_MAX_ACTIVE_PIPELINE_REQUESTS; +let maxDetachedPipelineRequests = MAX_DETACHED_PIPELINE_REQUESTS; + +interface PendingSessionClaim { + active: ActivePipelineRequest; + sessionID: string; + signal: AbortSignal; + resolve: () => void; + reject: (reason: unknown) => void; + onAbort: () => void; +} + +const pendingSessionClaims = new Map(); + +class PipelineCapacityError extends Error {} + +function activePipelineRequestsForSession(sessionID: string): number { + let count = 0; + for (const request of activePipelineRequests) { + if (request.sessionIDs.has(sessionID)) count++; + } + return count; +} + +function activePipelineRequestsForAdmissionKey(admissionKey: string): number { + let count = 0; + for (const request of activePipelineRequests) { + if (request.admissionKey === admissionKey) count++; + } + return count; +} + +function pendingSessionClaimsForAdmissionKey(admissionKey: string): number { + let count = 0; + for (const claim of pendingSessionClaims.values()) { + if (claim.active.admissionKey === admissionKey) count++; + } + return count; +} + +function pipelineSessionHasCapacity(sessionID: string): boolean { + return ( + activePipelineRequestsForSession(sessionID) + + (streamingPostResponseFinalizers.get(sessionID)?.pending ?? 0) < + MAX_ACTIVE_PIPELINE_REQUESTS_PER_SESSION + ); +} + +function pumpPendingSessionClaims(): void { + for (const [sessionID, claim] of pendingSessionClaims) { + if ( + activePipelineRequests.size + streamingPostResponsePending >= + maxActivePipelineRequests + ) { + return; + } + if ( + activePipelineRequestsForAdmissionKey(claim.active.admissionKey) + + (streamingPostResponsePendingByAdmissionKey.get( + claim.active.admissionKey, + ) ?? 0) >= + MAX_ACTIVE_PIPELINE_REQUESTS_PER_ADMISSION_KEY + ) { + continue; + } + if (!pipelineSessionHasCapacity(sessionID)) continue; + pendingSessionClaims.delete(sessionID); + claim.signal.removeEventListener("abort", claim.onAbort); + if (claim.signal.aborted) { + claim.reject(claim.signal.reason); + continue; + } + claim.active.sessionIDs.add(sessionID); + activePipelineRequests.add(claim.active); + claim.resolve(); + } +} + +function isPipelineSessionActive(sessionID: string): boolean { + return ( + activePipelineRequestsForSession(sessionID) > 0 || + streamingPostResponseFinalizers.has(sessionID) + ); +} + +export function activePipelineRequestCountForTest(): number { + return activePipelineRequests.size; +} + +export function detachedPipelineRequestCountForTest(): number { + return detachedPipelineRequests.size; +} + +export function pendingPipelineSessionClaimCountForTest(): number { + return pendingSessionClaims.size; +} + +export function setMaxActivePipelineRequestsForTest( + limit = DEFAULT_MAX_ACTIVE_PIPELINE_REQUESTS, +): void { + maxActivePipelineRequests = limit; +} + +export function setMaxDetachedPipelineRequestsForTest( + limit = MAX_DETACHED_PIPELINE_REQUESTS, +): void { + maxDetachedPipelineRequests = limit; +} + +export function isPipelineSessionActiveForTest(sessionID: string): boolean { + return isPipelineSessionActive(sessionID); +} + /** * Set (or clear) the module-level upstream interceptor. * @@ -645,6 +792,42 @@ export function setUpstreamInterceptor( activeInterceptor = interceptor; } +export function setPostResponseStartObserverForTest( + observer: (() => void) | undefined, +): void { + postResponseStartObserver = observer; +} + +export function setRecallPersistenceCommitObserverForTest( + observer: (() => void) | undefined, +): void { + recallPersistenceCommitObserver = observer; +} + +export function setPipelineResetPauseForTest( + pause: Promise | undefined, +): void { + pipelineResetPauseForTest = pause; +} + +export function setPipelinePreUpstreamPauseForTest( + pause: Promise | undefined, + onWait: () => void = () => {}, +): void { + pipelinePreUpstreamPauseForTest = pause ? { pause, onWait } : undefined; +} + +export function setProvisionalFinalizerPauseForTest( + pause: Promise | undefined, + onWait: () => void = () => {}, +): void { + provisionalFinalizerPauseForTest = pause ? { pause, onWait } : undefined; +} + +export function setPipelineResetSettleTimeoutForTest(timeoutMs = 5000): void { + pipelineResetSettleTimeoutMs = timeoutMs; +} + /** * Reset all module-level singleton state. * @@ -655,16 +838,73 @@ export function setUpstreamInterceptor( export async function resetPipelineState(opts?: { fast?: boolean; }): Promise { - // Cancel client-facing request/stream work before clearing singleton state. - // Gateway shutdown starts listener closure first, then reaches this abort; - // active streams settle and allow node:http's server.close() to complete. + if (pipelineResetPromise) return pipelineResetPromise; + pipelineResetInProgress = true; + const reset = (async () => { + try { + await resetPipelineStateInner(opts); + } finally { + pipelineResetInProgress = false; + pipelineResetPromise = undefined; + } + })(); + pipelineResetPromise = reset; + return reset; +} + +async function resetPipelineStateInner(opts?: { + fast?: boolean; +}): Promise { + streamingPostResponsesAccepting = false; + await pipelineResetPauseForTest; + const resetReason = new DOMException("gateway pipeline reset", "AbortError"); + pipelineGenerationAbort.abort(resetReason); const foregroundControllers = [...activeForegroundAbortControllers]; activeForegroundAbortControllers.clear(); - const resetReason = new DOMException("gateway pipeline reset", "AbortError"); for (const controller of foregroundControllers) { if (!controller.signal.aborted) controller.abort(resetReason); } - + const activeRequests = [ + ...new Set([ + ...activePipelineRequests, + ...[...pendingSessionClaims.values()].map((claim) => claim.active), + ]), + ]; + for (const request of activeRequests) request.abort(resetReason); + await boundedSettle( + activeRequests.map((request) => request.settled), + pipelineResetSettleTimeoutMs, + ); + for (const request of activeRequests) { + if (!activePipelineRequests.has(request)) continue; + activePipelineRequests.delete(request); + request.sessionIDs.clear(); + if (detachedPipelineRequests.size < maxDetachedPipelineRequests) { + detachedPipelineRequests.add(request); + } else { + log.error( + "pipeline quarantine full; dropping stale lifecycle reservation", + ); + } + } + // Streaming responses register post-response finalizers before closing their + // bodies. Drain them before sessions or the DB-facing pipeline state are + // cleared; a finalizer may also schedule ordinary background work, which the + // non-fast drain below will then observe. + await boundedSettle( + [...streamingPostResponseFinalizers.values()].map((state) => state.tail), + pipelineResetSettleTimeoutMs, + ); + streamingPostResponseGeneration++; + pipelineGenerationAbort = new AbortController(); + streamingPostResponseFinalizers.clear(); + streamingPostResponsePendingByAdmissionKey.clear(); + streamingPostResponsePending = 0; + maxStreamingPostResponses = DEFAULT_MAX_STREAMING_POST_RESPONSES; + maxStreamingPostResponsesPerSession = + DEFAULT_MAX_STREAMING_POST_RESPONSES_PER_SESSION; + lastStreamingPostResponseOverflowLog = 0; + lastStreamingPostResponseResetLog = 0; // Quiesce background work before tearing anything down. Only the non-fast // path drains — today that's test/eval teardown (the fast process-exit path, // the sole production caller, skips this to keep Ctrl+C snappy). Stop the @@ -689,16 +929,24 @@ export async function resetPipelineState(opts?: { inFlightBackground.clear(); } initialized = false; + maxActivePipelineRequests = DEFAULT_MAX_ACTIVE_PIPELINE_REQUESTS; + maxDetachedPipelineRequests = MAX_DETACHED_PIPELINE_REQUESTS; sessions.clear(); cwdWarned.clear(); staleHeaderWarned.clear(); subagentParentPendingLogged.clear(); headerSessionIndex.clear(); + ambiguousHeaderSessionKeys.clear(); + provisionalHeaderSessionIndex.clear(); + identityAdmissionTails.clear(); + headerSessionIndexHydrated = false; ltmSessionCache.clear(); ltmPinnedText.clear(); lastSavedDedupDecisions.clear(); stableLtmCache.clear(); stableLtmInFlight.clear(); + sessionLifecycleAborts.clear(); + streamingPostResponseWaiters.clear(); // Shut down the batch queue before clearing the client. On process exit // (`fast`), skip the synchronous LLM drain — replaying queued background // prompts through retries/backoff is what made Ctrl+C hang for minutes; they @@ -713,6 +961,9 @@ export async function resetPipelineState(opts?: { llmClient = null; activeInterceptor = undefined; beforeUpstreamCaptureForTest = undefined; + postResponseStartObserver = undefined; + recallPersistenceCommitObserver = undefined; + provisionalFinalizerPauseForTest = undefined; if (stopFileWatcher) { stopFileWatcher(); stopFileWatcher = null; @@ -735,6 +986,192 @@ export async function resetPipelineState(opts?: { /** Per-session state tracked across requests. */ const sessions = new Map(); +const DEFAULT_MAX_STREAMING_POST_RESPONSES = 64; +// Production requests reserve capacity before upstream work. The limits remain +// as defense-in-depth for unreserved/test-only scheduling. +const DEFAULT_MAX_STREAMING_POST_RESPONSES_PER_SESSION = 2; + +/** + * Deferred streaming finalizers keyed by session. The streamer invokes its + * callback before closing so registration is atomic with terminal delivery, + * but the expensive synchronous accounting itself runs on the next event-loop + * turn, allowing the body reader (and Node bridge) to observe EOF first. The + * bounded registry preserves in-process ordering; a process crash in that one + * event-loop-turn window can still lose final accounting, which is the explicit + * availability trade-off required to avoid holding client EOF behind SQLite. + */ +const streamingPostResponseFinalizers = new Map< + string, + { tail: Promise; pending: number } +>(); +const streamingPostResponsePendingByAdmissionKey = new Map(); +let streamingPostResponsePending = 0; +let streamingPostResponseGeneration = 0; +let pipelineGenerationAbort = new AbortController(); +let streamingPostResponsesAccepting = true; +let maxStreamingPostResponses = DEFAULT_MAX_STREAMING_POST_RESPONSES; +let maxStreamingPostResponsesPerSession = + DEFAULT_MAX_STREAMING_POST_RESPONSES_PER_SESSION; +let lastStreamingPostResponseOverflowLog = 0; +let lastStreamingPostResponseResetLog = 0; +let streamingPostResponseWaitObserverForTest: (() => void) | undefined; + +export function setStreamingPostResponseLimitsForTest( + globalLimit?: number, + perSessionLimit?: number, +): void { + maxStreamingPostResponses = + globalLimit ?? DEFAULT_MAX_STREAMING_POST_RESPONSES; + maxStreamingPostResponsesPerSession = + perSessionLimit ?? DEFAULT_MAX_STREAMING_POST_RESPONSES_PER_SESSION; +} + +export function streamingPostResponsePendingForTest(): number { + return streamingPostResponsePending; +} + +export function setStreamingPostResponseWaitObserverForTest( + observer: (() => void) | undefined, +): void { + streamingPostResponseWaitObserverForTest = observer; +} + +export function scheduleStreamingPostResponseForTest( + sessionID: string, + operation: () => void | Promise, + onDrop: () => void = () => {}, +): void { + scheduleStreamingPostResponse( + sessionID, + streamingPostResponseGeneration, + operation, + onDrop, + ); +} + +function scheduleStreamingPostResponse( + sessionID: string, + generation: number, + operation: () => void | Promise, + onDrop: () => void, + // Conversation requests reserve global + session capacity before upstream. + // Unreserved callers still use the defensive queue limits below. + capacityReserved = false, + admissionKey?: string, +): void { + const drop = (): void => { + try { + onDrop(); + } catch (error) { + log.error("streaming post-response drop cleanup failed:", error); + } + }; + if ( + !streamingPostResponsesAccepting || + generation !== streamingPostResponseGeneration + ) { + const now = Date.now(); + if (now - lastStreamingPostResponseResetLog >= 30_000) { + lastStreamingPostResponseResetLog = now; + log.info("streaming post-response skipped during pipeline reset"); + } + drop(); + return; + } + const existing = streamingPostResponseFinalizers.get(sessionID); + if ( + (!capacityReserved && + streamingPostResponsePending >= maxStreamingPostResponses) || + (!capacityReserved && + (existing?.pending ?? 0) >= maxStreamingPostResponsesPerSession) + ) { + const now = Date.now(); + if (now - lastStreamingPostResponseOverflowLog >= 30_000) { + lastStreamingPostResponseOverflowLog = now; + log.warn("streaming post-response queue full; dropping finalizer"); + } + drop(); + return; + } + const state = existing ?? { tail: Promise.resolve(), pending: 0 }; + const previous = state.tail; + state.pending++; + streamingPostResponsePending++; + if (admissionKey !== undefined) { + streamingPostResponsePendingByAdmissionKey.set( + admissionKey, + (streamingPostResponsePendingByAdmissionKey.get(admissionKey) ?? 0) + 1, + ); + } + const current = (async () => { + await previous; + await new Promise((resolve) => setImmediate(resolve)); + if (generation !== streamingPostResponseGeneration) { + drop(); + return; + } + try { + await operation(); + } catch (error) { + log.error("streaming post-response processing failed:", error); + } + })(); + state.tail = current; + streamingPostResponseFinalizers.set(sessionID, state); + void current.finally(() => { + if (streamingPostResponseFinalizers.get(sessionID) !== state) return; + state.pending--; + streamingPostResponsePending--; + if (admissionKey !== undefined) { + const remaining = + (streamingPostResponsePendingByAdmissionKey.get(admissionKey) ?? 1) - 1; + if (remaining > 0) { + streamingPostResponsePendingByAdmissionKey.set(admissionKey, remaining); + } else { + streamingPostResponsePendingByAdmissionKey.delete(admissionKey); + } + } + if (state.tail === current && state.pending === 0) { + streamingPostResponseFinalizers.delete(sessionID); + } + pumpPendingSessionClaims(); + }); +} + +const MAX_STREAMING_POST_RESPONSE_WAITERS_PER_SESSION = 16; +const streamingPostResponseWaiters = new Map(); + +class StreamingPostResponseWaitCapacityError extends Error {} + +async function awaitStreamingPostResponse( + sessionID: string, + signal?: AbortSignal, +): Promise { + if (!streamingPostResponseFinalizers.has(sessionID)) return; + const waiters = streamingPostResponseWaiters.get(sessionID) ?? 0; + if (waiters >= MAX_STREAMING_POST_RESPONSE_WAITERS_PER_SESSION) { + throw new StreamingPostResponseWaitCapacityError( + "streaming post-response wait queue full", + ); + } + streamingPostResponseWaiters.set(sessionID, waiters + 1); + try { + for (;;) { + const state = streamingPostResponseFinalizers.get(sessionID); + if (!state) return; + const tail = state.tail; + streamingPostResponseWaitObserverForTest?.(); + await promiseAgainstAbort(() => tail, signal); + const latest = streamingPostResponseFinalizers.get(sessionID); + if (latest !== state || state.tail === tail) return; + } + } finally { + const remaining = (streamingPostResponseWaiters.get(sessionID) ?? 1) - 1; + if (remaining > 0) streamingPostResponseWaiters.set(sessionID, remaining); + else streamingPostResponseWaiters.delete(sessionID); + } +} + /** Sessions that have already logged the cwd-fallback warning (dedup). */ const cwdWarned = new Set(); @@ -795,6 +1232,22 @@ export function rebindActiveSession( * Key: `credentialFingerprint\x1fheaderName\x1fheaderValue`. */ const headerSessionIndex = new Map(); +const ambiguousHeaderSessionKeys = new Set(); +type ProvisionalHeaderMapping = { + sessionID: string; + createdAt: number; + guardProject: boolean; + adoptionFingerprint?: string; + expectedUnowned: boolean; +}; +const provisionalHeaderSessionIndex = new Map< + string, + ProvisionalHeaderMapping +>(); +const identityAdmissionTails = new Map>(); +const MAX_PROVISIONAL_HEADER_MAPPINGS = 1024; +const PROVISIONAL_HEADER_MAPPING_TTL_MS = 5 * 60_000; +let headerSessionIndexHydrated = false; const SESSION_INDEX_SEPARATOR = "\x1f"; const TENANT_FINGERPRINT_RE = /^[a-f0-9]{64}$/; @@ -857,6 +1310,464 @@ function sessionIndexKey( ); } +async function withIdentityAdmission( + req: GatewayRequest, + config: GatewayConfig, + operation: () => Promise, +): Promise { + const known = extractKnownSessionHeader(req.rawHeaders); + if (!known) return operation(); + const key = sessionIndexKey( + requestCredentialFingerprint(req.rawHeaders, config) ?? "", + known.headerName, + known.sessionId, + ); + const previous = identityAdmissionTails.get(key); + let release!: () => void; + const ownCompletion = new Promise((resolve) => { + release = resolve; + }); + const tail = previous ? previous.then(() => ownCompletion) : ownCompletion; + identityAdmissionTails.set(key, tail); + try { + if (previous) await promiseAgainstAbort(() => previous, req.signal); + return await operation(); + } finally { + release(); + if (identityAdmissionTails.get(key) === tail) { + identityAdmissionTails.delete(key); + } + } +} + +function setProvisionalHeaderMapping( + key: string, + sessionID: string, + guardProject = false, + adoptionFingerprint?: string, + expectedUnowned = false, +): void { + const now = Date.now(); + for (const [candidate, entry] of provisionalHeaderSessionIndex) { + if (now - entry.createdAt > PROVISIONAL_HEADER_MAPPING_TTL_MS) { + provisionalHeaderSessionIndex.delete(candidate); + } + } + const existing = getProvisionalHeaderMapping(key); + if (existing && existing !== sessionID) { + throw new Error("ambiguous session headers"); + } + provisionalHeaderSessionIndex.delete(key); + while ( + provisionalHeaderSessionIndex.size >= MAX_PROVISIONAL_HEADER_MAPPINGS + ) { + const oldest = provisionalHeaderSessionIndex.keys().next().value; + if (oldest === undefined) break; + provisionalHeaderSessionIndex.delete(oldest); + } + provisionalHeaderSessionIndex.set(key, { + sessionID, + createdAt: now, + guardProject, + adoptionFingerprint, + expectedUnowned, + }); +} + +function getProvisionalHeaderEntry( + key: string, +): ProvisionalHeaderMapping | null { + const entry = provisionalHeaderSessionIndex.get(key); + if (!entry) return null; + if (Date.now() - entry.createdAt > PROVISIONAL_HEADER_MAPPING_TTL_MS) { + provisionalHeaderSessionIndex.delete(key); + return null; + } + return entry; +} + +function getProvisionalHeaderMapping(key: string): string | undefined { + return getProvisionalHeaderEntry(key)?.sessionID; +} + +function provisionalMappingGuardsProject( + key: string, + sessionID: string, +): boolean { + if (getProvisionalHeaderMapping(key) !== sessionID) return false; + return provisionalHeaderSessionIndex.get(key)?.guardProject === true; +} + +/** @internal Test seam for exercising ownership expiry during an in-flight turn. */ +export function expireProvisionalHeaderMappingsForTest(): void { + provisionalHeaderSessionIndex.clear(); +} + +function provisionalKeyOwned(key: string, sessionID: string): boolean { + return ( + headerSessionIndex.get(key) === sessionID || + getProvisionalHeaderMapping(key) === sessionID + ); +} + +function dropOwnedProvisionalKey( + key: string | undefined, + sessionID: string, +): void { + if (key && getProvisionalHeaderMapping(key) === sessionID) { + provisionalHeaderSessionIndex.delete(key); + } +} + +function conflictsWithConfidentSessionProject( + sessionID: string, + pathResult: ProjectPathResult, +): boolean { + if (pathResult.source !== "header" && pathResult.source !== "inferred") { + return false; + } + const live = sessions.get(sessionID); + if (live?.projectPath && live.projectPathProvisional === false) { + return live.projectPath !== pathResult.path; + } + const persisted = loadSessionTracking(sessionID); + return ( + !!persisted?.projectPath && + persisted.projectPathProvisional === false && + persisted.projectPath !== pathResult.path + ); +} + +function legacyAdoptionTargetIsUnowned(sessionID: string): boolean { + return loadSessionTracking(sessionID)?.credentialFingerprint === ""; +} + +function isConfidentlyBoundToProject( + state: SessionState, + projectPath: string, +): boolean { + return ( + state.projectPathProvisional !== true && state.projectPath === projectPath + ); +} + +function hydrateHeaderSessionIndex(config: GatewayConfig): void { + if (headerSessionIndexHydrated) return; + restoreHeaderSessionMappings(config); + headerSessionIndexHydrated = true; +} + +function findIndexedKnownSessionID( + req: GatewayRequest, + config: GatewayConfig, +): string | undefined { + const credentialFingerprint = requestCredentialFingerprint( + req.rawHeaders, + config, + ); + if (credentialFingerprint === null) return undefined; + hydrateHeaderSessionIndex(config); + const known = extractKnownSessionHeader(req.rawHeaders); + if (!known) return undefined; + return headerSessionIndex.get( + sessionIndexKey(credentialFingerprint, known.headerName, known.sessionId), + ); +} + +function hasConflictingConfirmedHeader( + req: GatewayRequest, + expectedSessionID: string, + excludedKey: string, + config: GatewayConfig, +): boolean { + const credentialFingerprint = requestCredentialFingerprint( + req.rawHeaders, + config, + ); + if (credentialFingerprint === null) return true; + hydrateHeaderSessionIndex(config); + for (const [key, sessionID] of headerSessionIndex) { + if (key === excludedKey || sessionID === expectedSessionID) continue; + const parsed = parseSessionIndexKey(key); + if (!parsed || parsed.headerName === "context-marker") continue; + if (parsed.credentialFingerprint !== credentialFingerprint) continue; + if (req.rawHeaders[parsed.headerName] === parsed.headerValue) return true; + } + return false; +} + +type IndexedSessionResolution = + | { + kind: "match"; + sessionID: string; + provisional?: boolean; + provisionalKey?: string; + } + | { kind: "ambiguous" } + | { kind: "none" }; + +function resolveIndexedSession( + req: GatewayRequest, + config: GatewayConfig, + includeProvisional = false, +): IndexedSessionResolution { + const known = extractKnownSessionHeader(req.rawHeaders); + if (known) { + if (includeProvisional) { + const credentialFingerprint = requestCredentialFingerprint( + req.rawHeaders, + config, + ); + if (credentialFingerprint === null) return { kind: "none" }; + const key = sessionIndexKey( + credentialFingerprint, + known.headerName, + known.sessionId, + ); + const confirmedSessionID = findIndexedKnownSessionID(req, config); + const sessionID = confirmedSessionID ?? getProvisionalHeaderMapping(key); + return sessionID + ? { + kind: "match", + sessionID, + provisional: !confirmedSessionID, + ...(!confirmedSessionID ? { provisionalKey: key } : {}), + } + : { kind: "none" }; + } + const sessionID = findIndexedKnownSessionID(req, config); + return sessionID ? { kind: "match", sessionID } : { kind: "none" }; + } + + const credentialFingerprint = requestCredentialFingerprint( + req.rawHeaders, + config, + ); + if (credentialFingerprint === null) return { kind: "none" }; + hydrateHeaderSessionIndex(config); + let match: string | undefined; + let provisional = false; + let provisionalKey: string | undefined; + for (const [key, sessionID] of headerSessionIndex) { + const parsed = parseSessionIndexKey(key); + if (!parsed || parsed.headerName === "context-marker") continue; + if (parsed.credentialFingerprint !== credentialFingerprint) continue; + if (req.rawHeaders[parsed.headerName] !== parsed.headerValue) continue; + if (match && match !== sessionID) return { kind: "ambiguous" }; + match = sessionID; + } + if (includeProvisional) { + for (const [key, entry] of provisionalHeaderSessionIndex) { + const sessionID = getProvisionalHeaderMapping(key); + if (!sessionID || sessionID !== entry.sessionID) continue; + const parsed = parseSessionIndexKey(key); + if (!parsed || parsed.headerName === "context-marker") continue; + if (parsed.credentialFingerprint !== credentialFingerprint) continue; + if (req.rawHeaders[parsed.headerName] !== parsed.headerValue) continue; + if (match && match !== sessionID) return { kind: "ambiguous" }; + match = sessionID; + provisional = true; + provisionalKey ??= key; + } + } + if (match) { + return { kind: "match", sessionID: match, provisional, provisionalKey }; + } + + const markerSid = extractSessionMarker(req.messages); + if (!markerSid) return { kind: "none" }; + const sessionID = headerSessionIndex.get( + sessionIndexKey(credentialFingerprint, "context-marker", markerSid), + ); + return sessionID ? { kind: "match", sessionID } : { kind: "none" }; +} + +function findIndexedSessionID( + req: GatewayRequest, + config: GatewayConfig, +): string | undefined { + const resolution = resolveIndexedSession(req, config); + return resolution.kind === "match" ? resolution.sessionID : undefined; +} + +/** + * Revalidate an authenticated index lookup after an async wait. Affinity + * rotation can revoke the request's alias while it is queued for the session; + * callers must fail closed instead of continuing with the stale session ID. + */ +function confirmedIndexedIdentityResolvesTo( + req: GatewayRequest, + expectedSessionID: string, + config: GatewayConfig, +): boolean { + const resolution = resolveIndexedSession(req, config); + return ( + resolution.kind === "match" && + resolution.sessionID === expectedSessionID && + resolution.provisional !== true + ); +} + +function findLiveSessionState( + req: GatewayRequest, + config: GatewayConfig, + allSessions: ReadonlyMap = sessions, +): SessionState | undefined { + const known = extractKnownSessionHeader(req.rawHeaders); + if (known) { + // An indexed higher-priority header is authoritative even when its session + // is not currently hydrated; never fall through to a conflicting alias. + const indexedSid = findIndexedKnownSessionID(req, config); + return indexedSid ? allSessions.get(indexedSid) : undefined; + } + const indexedSid = findIndexedSessionID(req, config); + return indexedSid ? allSessions.get(indexedSid) : undefined; +} + +function resolveAuthenticatedDirectSession( + req: GatewayRequest, + projectPath: string, + config: GatewayConfig, + knownHeaderOnly = true, +): SessionState | undefined { + if (knownHeaderOnly && !extractKnownSessionHeader(req.rawHeaders)) + return undefined; + const sessionID = knownHeaderOnly + ? findIndexedKnownSessionID(req, config) + : findIndexedSessionID(req, config); + if (!sessionID) return undefined; + try { + return getOrCreateSession( + sessionID, + projectPath, + "header", + requestCredentialFingerprint(req.rawHeaders, config) ?? "", + config, + ); + } catch (error) { + if (error instanceof SessionTenantMismatchError) return undefined; + throw error; + } +} + +function knownSessionHeaderForRequest( + req: GatewayRequest, + sessionID: string, + config: GatewayConfig, +): { headerName: string; sessionId: string } | null { + const credentialFingerprint = requestCredentialFingerprint( + req.rawHeaders, + config, + ); + if (credentialFingerprint === null) return null; + let known = extractKnownSessionHeader(req.rawHeaders); + if (!known) { + for (const [key, entry] of provisionalHeaderSessionIndex) { + if (entry.sessionID !== sessionID) continue; + const parsed = parseSessionIndexKey(key); + if (!parsed || parsed.headerName === "context-marker") continue; + if (parsed.credentialFingerprint !== credentialFingerprint) continue; + if (req.rawHeaders[parsed.headerName] !== parsed.headerValue) continue; + known = { + headerName: parsed.headerName, + sessionId: parsed.headerValue, + }; + break; + } + } + return known; +} + +function publishKnownSessionHeader( + known: { headerName: string; sessionId: string }, + state: SessionState, + credentialFingerprint: string, +): void { + const confirmedKey = sessionIndexKey( + credentialFingerprint, + known.headerName, + known.sessionId, + ); + if (credentialFingerprint) { + for (const [key, sessionID] of headerSessionIndex) { + if (sessionID !== state.sessionID) continue; + const parsed = parseSessionIndexKey(key); + if (parsed?.credentialFingerprint === "") { + headerSessionIndex.delete(key); + } + } + } + if (isRotationEligible(known.headerName)) { + for (const [key, sessionID] of headerSessionIndex) { + if (key === confirmedKey || sessionID !== state.sessionID) continue; + const parsed = parseSessionIndexKey(key); + if ( + parsed?.credentialFingerprint === credentialFingerprint && + parsed.headerName === known.headerName + ) { + headerSessionIndex.delete(key); + } + } + } + provisionalHeaderSessionIndex.delete(confirmedKey); + headerSessionIndex.set(confirmedKey, state.sessionID); + state.headerSessionId = known.sessionId; + state.headerName = known.headerName; + state.credentialFingerprint = credentialFingerprint; +} + +function confirmKnownSessionHeader( + req: GatewayRequest, + state: SessionState, + config: GatewayConfig, + tracking: Parameters[1] = {}, + persistTurn?: () => void, +): void { + const credentialFingerprint = requestCredentialFingerprint( + req.rawHeaders, + config, + ); + if (credentialFingerprint === null) return; + const known = knownSessionHeaderForRequest(req, state.sessionID, config); + if (!known) return; + withSavepoint("confirm_session_header", () => { + persistTurn?.(); + saveSessionTracking(state.sessionID, { + ...tracking, + headerSessionId: known.sessionId, + headerName: known.headerName, + credentialFingerprint, + }); + }); + publishKnownSessionHeader(known, state, credentialFingerprint); +} + +export function evictLiveSessionForTest( + req: GatewayRequest, + config?: GatewayConfig, +): boolean { + const credential = extractAuth(req.rawHeaders); + const credentialFingerprint = config + ? requestCredentialFingerprint(req.rawHeaders, config) + : credential + ? authFingerprint(credential) + : ""; + if (credentialFingerprint === null) return false; + for (const headerName of KNOWN_SESSION_HEADERS) { + const headerValue = req.rawHeaders[headerName]; + if (!headerValue) continue; + const sid = headerSessionIndex.get( + sessionIndexKey(credentialFingerprint, headerName, headerValue), + ); + if (sid) { + const removed = sessions.delete(sid); + if (removed) evictPipelineSessionState(sid); + return removed; + } + } + return false; +} + function parseSessionIndexKey(key: string): { credentialFingerprint: string; headerName: string; @@ -901,14 +1812,19 @@ function restoreHeaderSessionMappings(config: GatewayConfig): { ) { continue; } - headerSessionIndex.set( - sessionIndexKey( - entry.credentialFingerprint, - entry.headerName, - entry.headerSessionId, - ), - entry.sessionId, + const key = sessionIndexKey( + entry.credentialFingerprint, + entry.headerName, + entry.headerSessionId, ); + if (ambiguousHeaderSessionKeys.has(key)) continue; + const existing = headerSessionIndex.get(key); + if (existing && existing !== entry.sessionId) { + headerSessionIndex.delete(key); + ambiguousHeaderSessionKeys.add(key); + continue; + } + headerSessionIndex.set(key, entry.sessionId); restored++; } return { restored, cleared }; @@ -2531,6 +3447,54 @@ const stableLtmCache = new Map< * into stableLtmCache), so a LATER miss after a restart recomputes fresh. */ const stableLtmInFlight = new Map>(); +const sessionLifecycleAborts = new Map(); + +function sessionLifecycleSignal(sessionID: string): AbortSignal { + let controller = sessionLifecycleAborts.get(sessionID); + if (!controller) { + controller = new AbortController(); + sessionLifecycleAborts.set(sessionID, controller); + } + return controller.signal; +} + +function stableLtmComputeSignal(sessionID: string): AbortSignal { + return AbortSignal.any([ + pipelineGenerationAbort.signal, + sessionLifecycleSignal(sessionID), + ]); +} + +function evictStableLtmSession(sessionID: string): void { + sessionLifecycleAborts + .get(sessionID) + ?.abort(new DOMException("stable LTM session was evicted", "AbortError")); + sessionLifecycleAborts.delete(sessionID); + stableLtmCache.delete(sessionID); + stableLtmInFlight.delete(sessionID); +} + +function evictPipelineSessionState(sessionID: string): void { + // Keep the persisted header→session mapping warm. Eviction removes only the + // heavy live state; dropping this index would force an unbounded DB reload on + // the next request and would make state-changing slash commands unable to + // rehydrate the authoritative canonical session safely. + ltmSessionCache.delete(sessionID); + ltmPinnedText.delete(sessionID); + lastSavedDedupDecisions.delete(sessionID); + evictStableLtmSession(sessionID); + cwdWarned.delete(sessionID); + staleHeaderWarned.delete(sessionID); + for (const key of subagentParentPendingLogged) { + if (key.startsWith(`${sessionID}:`)) + subagentParentPendingLogged.delete(key); + } +} + +/** Test seam for exercising the same cleanup used by idle session eviction. */ +export function evictStableLtmSessionForTest(sessionID: string): void { + evictStableLtmSession(sessionID); +} /** * Run a stable-LTM compute under single-flight dedup for a session. If a @@ -2541,7 +3505,10 @@ const stableLtmInFlight = new Map>(); */ export async function singleFlightStableLtm( sessionID: string, - compute: () => Promise<{ formatted: string; tokenCount: number } | undefined>, + compute: ( + signal: AbortSignal, + ) => Promise<{ formatted: string; tokenCount: number } | undefined>, + callerSignal?: AbortSignal, ): Promise<{ formatted: string; tokenCount: number } | undefined> { // Cache hit — fast path. Reading the cache FIRST is essential: a previous // caller may have already settled and deleted its in-flight entry, so a @@ -2550,19 +3517,24 @@ export async function singleFlightStableLtm( if (cached) return cached; const inFlight = stableLtmInFlight.get(sessionID); if (inFlight) { - await inFlight; + await promiseAgainstAbort(() => inFlight, callerSignal); return stableLtmCache.get(sessionID); } - const promise = (async () => { + const signal = stableLtmComputeSignal(sessionID); + let promise!: Promise; + promise = (async () => { try { - const result = await compute(); + const result = await promiseAgainstAbort(() => compute(signal), signal); + signal.throwIfAborted(); if (result) stableLtmCache.set(sessionID, result); } finally { - stableLtmInFlight.delete(sessionID); + if (stableLtmInFlight.get(sessionID) === promise) { + stableLtmInFlight.delete(sessionID); + } } })(); stableLtmInFlight.set(sessionID, promise); - await promise; + await promiseAgainstAbort(() => promise, callerSignal); return stableLtmCache.get(sessionID); } @@ -2578,8 +3550,11 @@ async function computeStableLtm( cfg: ReturnType, contextHint: string | undefined, prefBudget: number, + signal?: AbortSignal, + requestGeneration?: number, ): Promise<{ formatted: string; tokenCount: number } | undefined> { const prefEntries = await ltm.forSession(projectPath, sessionID, prefBudget, { + signal, categories: ["preference"], ...(contextHint ? { contextHint } : {}), }); @@ -2639,6 +3614,11 @@ async function computeStableLtm( ] .filter(Boolean) .join("\n\n"); + if (requestGeneration !== undefined) { + assertCurrentPipelineGeneration(signal, requestGeneration); + } else { + signal?.throwIfAborted(); + } const tokenCount = formatted ? coreEstimateTokens(formatted) : 0; const stable = { formatted, tokenCount }; stableLtmCache.set(sessionID, stable); @@ -2675,6 +3655,7 @@ async function precomputeStableLtmForIdleSession( sessionID: string, state: SessionState, ): Promise { + const requestGeneration = streamingPostResponseGeneration; try { if (stableLtmCache.has(sessionID)) return; const cfg = loreConfig(); @@ -2690,8 +3671,16 @@ async function precomputeStableLtmForIdleSession( log.info( `idle precompute: warming stable LTM for session ${sessionID.slice(0, 16)} (pref=${prefBudget})`, ); - await singleFlightStableLtm(sessionID, () => - computeStableLtm(sessionID, projectPath, cfg, undefined, prefBudget), + await singleFlightStableLtm(sessionID, (signal) => + computeStableLtm( + sessionID, + projectPath, + cfg, + undefined, + prefBudget, + signal, + requestGeneration, + ), ); } catch (err) { log.warn( @@ -3119,7 +4108,13 @@ async function initIfNeeded( projectPath: string, config: GatewayConfig, gitRemote?: string, + signal?: AbortSignal, + requestGeneration?: number, ): Promise { + if (!pipelineResetInProgress) streamingPostResponsesAccepting = true; + if (requestGeneration !== undefined) { + assertCurrentPipelineGeneration(signal, requestGeneration); + } if (initialized) return; // Enable hosted mode before any FS operations — once set, all core @@ -3129,6 +4124,9 @@ async function initIfNeeded( } await load(projectPath); + if (requestGeneration !== undefined) { + assertCurrentPipelineGeneration(signal, requestGeneration); + } ensureProject(projectPath, undefined, gitRemote); initialized = true; @@ -3248,35 +4246,18 @@ async function initIfNeeded( config, sessions, idleHandler, - (sessionID) => { - // Clean up pipeline-level satellite Maps on session eviction. - // The headerSessionIndex entries are keyed by header values pointing - // TO this sessionID — remove them too. - for (const [key, sid] of headerSessionIndex) { - if (sid === sessionID) headerSessionIndex.delete(key); - } - ltmSessionCache.delete(sessionID); - ltmPinnedText.delete(sessionID); - lastSavedDedupDecisions.delete(sessionID); - stableLtmCache.delete(sessionID); - stableLtmInFlight.delete(sessionID); - cwdWarned.delete(sessionID); - staleHeaderWarned.delete(sessionID); - // Clear subagent parent-pending dedup entries for this session — - // keys are `${sessionID}:${parentClientId}`, so filter by prefix. - for (const key of subagentParentPendingLogged) { - if (key.startsWith(`${sessionID}:`)) { - subagentParentPendingLogged.delete(key); - } - } - }, + evictPipelineSessionState, + isPipelineSessionActive, ); } // Start background cloud sync (no-op until the user runs `lore sync enable`). if (!stopSyncScheduler) { const { startSyncScheduler } = await import("./sync"); - stopSyncScheduler = startSyncScheduler(config); + if (requestGeneration !== undefined) { + assertCurrentPipelineGeneration(signal, requestGeneration); + } + if (!stopSyncScheduler) stopSyncScheduler = startSyncScheduler(config); } log.info(`gateway pipeline initialized: ${projectPath}`); @@ -3533,8 +4514,18 @@ export function resolveSessionProjectPath( ); } + if (!healed && previous) { + // Keep writing to the original bucket until re-attribution succeeds. + // Moving the binding to projectPath here would lose `previous`, so the + // next confident turn could never retry and the old rows would remain + // permanently split from the session. + sessionState.projectPath = previous; + sessionState.projectPathProvisional = true; + return previous; + } + sessionState.projectPath = projectPath; - sessionState.projectPathProvisional = !healed; + sessionState.projectPathProvisional = false; // Backfill git_remote on the (now confident) project row — idempotent. if (effectiveRemote) { @@ -3678,7 +4669,9 @@ export function applySyntheticResolution( const wasProvisional = sessionState.projectPathProvisional === true; if (wasProvisional && previous && previous !== newPath) { - reattributeProvisionalProject(previous, newPath, gitRemote); + if (!reattributeProvisionalProject(previous, newPath, gitRemote)) { + return currentProjectPath; + } } sessionState.projectPath = newPath; @@ -3902,7 +4895,15 @@ export function matchingProviderSnapshotForTest( return matchingProviderSnapshot(state, providerID); } -function serializeUpstreamState(state: SessionState): string { +type MutableUpstreamState = Pick< + SessionState, + | "lastUpstream" + | "upstreamByProvider" + | "_upstreamRequestOrder" + | "_upstreamRequestOrderByProvider" +>; + +function serializeUpstreamState(state: MutableUpstreamState): string { const stripHeaders = (snapshot: UpstreamSnapshot) => ({ ...snapshot, headers: {}, @@ -4063,20 +5064,26 @@ function buildRequestUpstreamSnapshot( return freezeUpstreamSnapshot(snapshot); } -/** - * Capture request routing before awaiting the upstream. Failed tightening - * requests remain authoritative, while request-start order prevents an older - * concurrent turn from rolling back a newer policy when it resumes later. - */ -function captureRequestUpstream( +function prepareRequestUpstream( req: GatewayRequest, - state: SessionState, config: GatewayConfig, - requestOrder: number, -): ResolvedRequestUpstreamRoute { +): { + route: ResolvedRequestUpstreamRoute; + snapshot: UpstreamSnapshot; +} { const route = resolveRequestUpstreamRoute(req, config); const snapshot = buildRequestUpstreamSnapshot(req, route); + return { route, snapshot }; +} + +function applyRequestUpstream( + state: MutableUpstreamState, + snapshot: UpstreamSnapshot, + requestOrder: number, + config: GatewayConfig, +): { changed: boolean; resetCache: boolean } { let changed = false; + let resetCache = false; if (requestOrder >= (state._upstreamRequestOrder ?? 0)) { const previous = state.lastUpstream; @@ -4097,7 +5104,7 @@ function captureRequestUpstream( // transcript. Clear it synchronously with route capture so a failed or // in-flight policy-tightening request cannot let the idle warmer replay // that body (or admin extras) to the newly selected destination. - state.cacheAnalytics.lastRequestBody = null; + resetCache = true; } state.lastUpstream = snapshot; state._upstreamRequestOrder = requestOrder; @@ -4128,12 +5135,30 @@ function captureRequestUpstream( } } + return { changed, resetCache }; +} + +function captureRequestUpstream( + req: GatewayRequest, + state: SessionState, + config: GatewayConfig, + requestOrder: number, +): ResolvedRequestUpstreamRoute { + const prepared = prepareRequestUpstream(req, config); + const { changed, resetCache } = applyRequestUpstream( + state, + prepared.snapshot, + requestOrder, + config, + ); + if (resetCache) state.cacheAnalytics.lastRequestBody = null; + if (changed) { saveSessionTracking(state.sessionID, { lastUpstream: serializeUpstreamState(state), }); } - return route; + return prepared.route; } class SessionTenantMismatchError extends Error { @@ -4225,6 +5250,7 @@ function getOrCreateSession( messageCount: persisted?.messageCount ?? 0, turnsSinceCuration: persisted?.turnsSinceCuration ?? 0, consecutiveTextOnlyTurns: persisted?.consecutiveTextOnlyTurns ?? 0, + amnesia: persisted?.amnesia ?? false, recallStore: new Map(), upstreamByProvider: new Map(), cacheAnalytics: { @@ -4505,36 +5531,68 @@ const ADOPT_MIN_OVERLAP = 2; * Confirmation uses user messages only: temporal storage persists user messages * with position-stable deterministic IDs, while assistant responses are stored * under a synthetic index-0 ID — so only user messages are a reliable - * cross-restart match signal. The project_id scope of the overlap query also - * enforces same-project (a cross-project fingerprint twin yields zero overlap). - * Subagent status must match, and a fork guard rejects a count that dropped far - * below the candidate's stored count. + * cross-restart match signal. Confidently bound candidates require overlap in + * the incoming project. A provisionally bound candidate instead checks its + * existing bucket, allowing a later confident path to self-heal that bucket + * without weakening the cross-project guard for confident bindings. Subagent + * status must match, and a fork guard rejects a count that dropped far below the + * candidate's stored count. * * Called from BOTH mint paths: the Tier-1 path (known header present but its * value is new — the opencode restart case; `known` is rebound to the adopted * sid for a future Tier-1 fast path) and the Tier-3 path (no known header). */ +function trustedAdoptionRemote( + projectPath: string, + headers: Record, +): string | undefined { + const supplied = extractGitRemoteHeader(headers); + // Adoption is read-only, so it cannot call ensureProject's trusted-remote + // resolver. Match the current path's on-disk remote locally; only a hosted + // gateway, which cannot inspect client disk, may trust the normalized header. + return isHostedMode() ? supplied : (getGitRemote(projectPath) ?? undefined); +} + async function adoptByFingerprint(input: { req: GatewayRequest; headers: Record; projectPath: string; + gitRemote?: string; known: { headerName: string; sessionId: string } | null; msgCount: number; + requestGeneration?: number; config: GatewayConfig; credentialFingerprint: string; -}): Promise<{ sessionID: string; isNew: false; tier: 3 } | null> { +}): Promise<{ + sessionID: string; + isNew: false; + tier: 3; + provisionalIdentity: true; + provisionalKey?: string; + adoptionFingerprint: string; + expectedUnowned: boolean; +} | null> { const { req, headers, projectPath, + gitRemote, known, msgCount, + requestGeneration, config, credentialFingerprint, } = input; if (!projectPath) return null; const cred = extractAuth(req.rawHeaders); + // Restart adoption grants access to an existing session, so an upstream URL + // alone is not sufficient proof of ownership. + if (!cred) return null; + const authenticatedFingerprint = usesRemoteSessionBinding(config) + ? credentialTenantFingerprint(cred) + : authFingerprint(cred); + if (credentialFingerprint !== authenticatedFingerprint) return null; const fingerprintInput = req.messages.map((m) => ({ role: m.role, content: m.content, @@ -4546,23 +5604,26 @@ async function adoptByFingerprint(input: { ? { tenantFingerprint: credentialFingerprint } : { authSuffix: cred ? authFingerprint(cred) : "" }, ); + if (requestGeneration !== undefined) { + assertCurrentPipelineGeneration(req.signal, requestGeneration); + } const reqIsSubagent = !!headers["x-parent-session-id"]; - const candidates = findSessionStatesByFingerprint(fingerprint).filter( - (candidate) => - !remoteBinding || - loadSessionTracking(candidate.session_id)?.credentialFingerprint === - credentialFingerprint, - ); - if (cred && !remoteBinding) { - // v78 added the credential suffix to conversation fingerprints. Legacy - // candidates have the old unsuffixed fingerprint and no persisted owner; - // they still require the same project-scoped multi-message overlap below. - const legacyFingerprint = await fingerprintMessages(fingerprintInput); + const candidates = findSessionStatesByFingerprint(fingerprint, { + credentialFingerprint, + }).map((candidate) => ({ ...candidate, credentialBound: true })); + // v78 added the credential suffix to conversation fingerprints. Legacy + // candidates have the old unsuffixed fingerprint and no persisted owner; + // they still require the same multi-message overlap policy below. + const legacyFingerprint = await fingerprintMessages(fingerprintInput); + if (requestGeneration !== undefined) { + assertCurrentPipelineGeneration(req.signal, requestGeneration); + } + if (!remoteBinding) { candidates.push( ...findSessionStatesByFingerprint(legacyFingerprint, { legacyUnownedOnly: true, - }), + }).map((candidate) => ({ ...candidate, credentialBound: false })), ); } const eligibleCandidates = candidates.filter( @@ -4589,9 +5650,17 @@ async function adoptByFingerprint(input: { } if (probeMessages.length < ADOPT_MIN_OVERLAP) return null; - const pid = ensureProject(projectPath); + const incomingProjectId = resolveProjectByRemoteOrPath( + gitRemote, + projectPath, + ); const minOverlap = Math.max(ADOPT_MIN_OVERLAP, Math.ceil(probedUsers * 0.5)); - let best: { sid: string; overlap: number; countDiff: number } | null = null; + let best: { + sid: string; + overlap: number; + countDiff: number; + credentialBound: boolean; + } | null = null; // Source IDs include the candidate session, so valid rows cannot give two // sessions the same positive overlap set. Keep the tie guard as defense in // depth for a corrupt/imported database rather than selecting by row order. @@ -4602,6 +5671,32 @@ async function adoptByFingerprint(input: { if (msgCount - c.message_count < -MESSAGE_COUNT_PROXIMITY_THRESHOLD) { continue; } + // A confident or legacy-unowned candidate remains scoped to the incoming + // project. Only a credential-bound provisional binding may prove continuity + // against its current bucket before the incoming project has been created. + const usesPersistedProvisionalProject = + c.credentialBound && c.project_path_provisional === 1 && !!c.project_path; + const candidateProjectId = c.project_path + ? resolveProjectByRemoteOrPath(undefined, c.project_path) + : null; + if ( + !usesPersistedProvisionalProject && + (!incomingProjectId || candidateProjectId !== incomingProjectId) + ) { + continue; + } + // Derive message IDs from the persisted canonical path. A new clone path + // can resolve to the same project by git remote without being an alias yet; + // deriving IDs from that unregistered path would create a second project + // and make genuine transcript overlap impossible to observe. + const overlapProjectPath = + c.project_path && (usesPersistedProvisionalProject || candidateProjectId) + ? c.project_path + : projectPath; + const overlapProjectId = usesPersistedProvisionalProject + ? candidateProjectId + : incomingProjectId; + if (!overlapProjectId) continue; const probeIDs = probeMessages.map(({ index, message }) => { const sourceID = deterministicID( c.session_id, @@ -4610,7 +5705,7 @@ async function adoptByFingerprint(input: { message.content, ); return temporal.storedMessageId({ - projectPath, + projectPath: overlapProjectPath, sessionID: c.session_id, sourceID, legacySourceID: legacyDeterministicID( @@ -4620,7 +5715,11 @@ async function adoptByFingerprint(input: { ), }); }); - const overlap = countMatchingTemporalIds(pid, c.session_id, probeIDs); + const overlap = countMatchingTemporalIds( + overlapProjectId, + c.session_id, + probeIDs, + ); if (overlap < minOverlap) continue; const countDiff = Math.abs(msgCount - c.message_count); if ( @@ -4628,7 +5727,12 @@ async function adoptByFingerprint(input: { overlap > best.overlap || (overlap === best.overlap && countDiff < best.countDiff) ) { - best = { sid: c.session_id, overlap, countDiff }; + best = { + sid: c.session_id, + overlap, + countDiff, + credentialBound: c.credentialBound, + }; ambiguousBest = false; } else if (overlap === best.overlap && countDiff === best.countDiff) { ambiguousBest = true; @@ -4636,32 +5740,65 @@ async function adoptByFingerprint(input: { } if (!best || ambiguousBest) return null; - // When a known header is present, rebind it → adopted sid so future turns - // identify via the Tier 1 fast path (and stop re-confirming overlap). - if (known) { - headerSessionIndex.set( - sessionIndexKey(credentialFingerprint, known.headerName, known.sessionId), - best.sid, - ); - saveSessionTracking(best.sid, { - headerSessionId: known.sessionId, - headerName: known.headerName, - credentialFingerprint, - }); - } + // Keep the adopted header provisional until a successful response confirms + // it in postResponse. This preserves retry continuity without authorizing + // sensitive routes after a failed/aborted adoption turn. log.info( `adopted prior session ${best.sid.slice(0, 16)} for resumed conversation ` + `(overlap=${best.overlap}/${probedUsers}` + `${known ? `, header=${known.headerName}` : ""})`, ); - return { sessionID: best.sid, isNew: false, tier: 3 }; + if (known) { + const provisionalKey = sessionIndexKey( + credentialFingerprint, + known.headerName, + known.sessionId, + ); + setProvisionalHeaderMapping( + provisionalKey, + best.sid, + false, + fingerprint, + !best.credentialBound, + ); + return { + sessionID: best.sid, + isNew: false, + tier: 3, + provisionalIdentity: true, + provisionalKey, + adoptionFingerprint: fingerprint, + expectedUnowned: !best.credentialBound, + }; + } + return { + sessionID: best.sid, + isNew: false, + tier: 3, + provisionalIdentity: true, + adoptionFingerprint: fingerprint, + expectedUnowned: !best.credentialBound, + }; } +type IdentifiedSession = { + sessionID: string; + isNew: boolean; + tier: 1 | 2 | 2.5 | 3; + provisionalIdentity?: boolean; + provisionalKey?: string; + guardProject?: boolean; + adoptionFingerprint?: string; + expectedUnowned?: boolean; +}; + async function identifySession( req: GatewayRequest, projectPath: string, + projectPathSource: ProjectPathResult["source"] | undefined, + requestGeneration: number | undefined, config: GatewayConfig, -): Promise<{ sessionID: string; isNew: boolean; tier: 1 | 2 | 2.5 | 3 }> { +): Promise { const headers = req.rawHeaders; const credentialFingerprint = requestCredentialFingerprint(headers, config); @@ -4685,15 +5822,53 @@ async function identifySession( known.headerName, known.sessionId, ); + hydrateHeaderSessionIndex(config); + if (ambiguousHeaderSessionKeys.has(indexKey)) { + throw new Error("ambiguous persisted session header"); + } let existingSid = headerSessionIndex.get(indexKey); + let provisionalIdentity = false; + let guardProject = false; + let adoptionFingerprint: string | undefined; + let expectedUnowned = false; if (!existingSid) { - restoreHeaderSessionMappings(config); - existingSid = headerSessionIndex.get(indexKey); + const provisional = getProvisionalHeaderEntry(indexKey); + existingSid = provisional?.sessionID; + provisionalIdentity = provisional !== null; + guardProject = + existingSid !== undefined && + provisionalMappingGuardsProject(indexKey, existingSid); + adoptionFingerprint = provisional?.adoptionFingerprint; + expectedUnowned = provisional?.expectedUnowned === true; } if (existingSid) { + if ( + provisionalIdentity && + hasConflictingConfirmedHeader(req, existingSid, indexKey, config) + ) { + throw new Error("ambiguous session headers"); + } + if (provisionalIdentity) { + setProvisionalHeaderMapping( + indexKey, + existingSid, + guardProject, + adoptionFingerprint, + expectedUnowned, + ); + } // Session may only exist in DB (after gateway restart) — that's fine, // getOrCreateSession() will hydrate it from the session_state table. - return { sessionID: existingSid, isNew: false, tier: 1 }; + return { + sessionID: existingSid, + isNew: false, + tier: 1, + provisionalIdentity, + ...(provisionalIdentity ? { provisionalKey: indexKey } : {}), + ...(guardProject ? { guardProject: true } : {}), + ...(adoptionFingerprint ? { adoptionFingerprint } : {}), + ...(expectedUnowned ? { expectedUnowned: true } : {}), + }; } // --- Tier 1a: Cross-header migration --- @@ -4701,6 +5876,7 @@ async function identifySession( // x-lore-session-id), but the request also contains a lower-priority // known header that IS already indexed (e.g. x-session-affinity from // before the upgrade). Re-index under the new header and resume. + let fallbackMatch: { sessionID: string; headerName: string } | undefined; for (const fallbackName of KNOWN_SESSION_HEADERS) { if (fallbackName === known.headerName) continue; // skip the primary const fallbackValue = headers[fallbackName]; @@ -4712,149 +5888,77 @@ async function identifySession( ); const fallbackSid = headerSessionIndex.get(fallbackKey); if (fallbackSid) { - // Migrate: index under the new (higher-priority) header. - headerSessionIndex.set(indexKey, fallbackSid); - saveSessionTracking(fallbackSid, { - headerSessionId: known.sessionId, - headerName: known.headerName, - credentialFingerprint, - }); - // Update in-memory state if present. - const inMemory = sessions.get(fallbackSid); - if (inMemory) { - inMemory.headerSessionId = known.sessionId; - inMemory.headerName = known.headerName; - inMemory.credentialFingerprint = credentialFingerprint; + if (fallbackMatch && fallbackMatch.sessionID !== fallbackSid) { + throw new Error("ambiguous session headers"); } - log.info( - `session ${fallbackSid.slice(0, 16)}: migrated from ${fallbackName} to ${known.headerName}`, - ); - return { sessionID: fallbackSid, isNew: false, tier: 1 }; + fallbackMatch = { sessionID: fallbackSid, headerName: fallbackName }; } } - - // --- Tier 1b: Header value rotation detection --- - // Only for headers whose values may change on a client restart while the - // logical session continues (e.g. OpenCode's x-session-affinity nanoid). - // Headers like x-claude-code-session-id mint a fresh value per - // *conversation* — a new value is always a genuinely new session, and - // merging would collapse distinct conversations (and their projects) into - // one, causing cross-project contamination on remote/multi-client gateways. - const predecessor = !isRotationEligible(known.headerName) - ? null - : findRotationPredecessor( - credentialFingerprint, - known.headerName, - known.sessionId, - headerSessionIndex, - (sid) => { - // Session may be in memory or only in DB (after gateway restart). - const inMemory = sessions.get(sid); - if (inMemory) { - return { - sid, - isSubagent: !!inMemory.isSubagent, - lastActiveAt: inMemory.lastRequestTime, - }; - } - // Lightweight DB check for recency and subagent status. - const persisted = loadSessionTracking(sid); - if (!persisted) return null; // orphaned index entry - return { - sid, - isSubagent: persisted.isSubagent, - // lastTurnAt=0 means gradient never ran yet — session is new, - // treat as recently active (not infinitely stale). - lastActiveAt: - persisted.lastTurnAt > 0 ? persisted.lastTurnAt : Date.now(), - }; - }, + if (fallbackMatch) { + const incomingProject = + projectPathSource === "header" || projectPathSource === "inferred" + ? projectPath + : undefined; + const existing = loadSessionTracking(fallbackMatch.sessionID); + const conflictsWithConfidentProject = + !!incomingProject && + !!existing?.projectPath && + existing.projectPathProvisional === false && + existing.projectPath !== incomingProject; + if (!conflictsWithConfidentProject) { + setProvisionalHeaderMapping(indexKey, fallbackMatch.sessionID, true); + log.info( + `session ${fallbackMatch.sessionID.slice(0, 16)}: provisional migration from ${fallbackMatch.headerName} to ${known.headerName}`, ); - - // Fix 2 (defense in depth): even for a rotation-eligible header, never - // re-home a session onto a DIFFERENT confident project. If the incoming - // request carries an explicit X-Lore-Project that disagrees with the - // predecessor's bound project, this is not a benign restart — treat it as - // a genuinely new session to avoid cross-project contamination. - if (predecessor) { - const incomingProject = extractProjectHeader(headers); - if (incomingProject) { - const predTracking = loadSessionTracking(predecessor.sid); - const predProject = predTracking?.projectPath; - const predConfident = - !!predProject && predTracking?.projectPathProvisional === false; - if (predConfident && predProject !== incomingProject) { - log.warn( - `session rotation refused (${known.headerName}): incoming project ` + - `${incomingProject} differs from predecessor ${predecessor.sid.slice(0, 16)} ` + - `project ${predProject} — creating a new session instead of merging.`, - ); - const sessionID = generateSessionID(); - headerSessionIndex.set(indexKey, sessionID); - saveSessionTracking(sessionID, { - headerSessionId: known.sessionId, - headerName: known.headerName, - credentialFingerprint, - }); - // The old predecessor's index entry is intentionally preserved — the - // old session is still valid and may receive requests with its nanoid. - // It will age out via ROTATION_MAX_AGE_MS naturally. If another new - // nanoid arrives later, the old entry creates an ambiguity (multiple - // predecessors) → findRotationPredecessor returns null → new session. - return { sessionID, isNew: true, tier: 1 }; - } - } - } - - if (predecessor) { - // Resume the old session with the new header value. - const oldKey = sessionIndexKey( - credentialFingerprint, - known.headerName, - predecessor.oldHeaderValue, - ); - headerSessionIndex.delete(oldKey); - headerSessionIndex.set(indexKey, predecessor.sid); - - // Update in-memory state if present. - const inMemory = sessions.get(predecessor.sid); - if (inMemory) { - inMemory.headerSessionId = known.sessionId; - inMemory.headerName = known.headerName; - inMemory.credentialFingerprint = credentialFingerprint; + return { + sessionID: fallbackMatch.sessionID, + isNew: false, + tier: 1, + provisionalIdentity: true, + provisionalKey: indexKey, + guardProject: true, + }; } - - // Persist the new header mapping immediately. - saveSessionTracking(predecessor.sid, { - headerSessionId: known.sessionId, - headerName: known.headerName, - credentialFingerprint, - }); - - log.info( - `session ${predecessor.sid.slice(0, 16)}: resumed via ${known.headerName} value rotation`, + log.warn( + `session migration refused (${fallbackMatch.headerName}): incoming project ` + + `${incomingProject} differs from session ${fallbackMatch.sessionID.slice(0, 16)} ` + + `project ${existing.projectPath} - creating a new session instead of merging.`, ); - return { sessionID: predecessor.sid, isNew: false, tier: 1 }; } - // --- Tier 1 → 3b: restart-proof adoption --- - // The known header value is new and rotation found no predecessor. Before - // minting a fresh session, try to adopt a prior session for this same - // conversation (resumed after a restart under a new x-lore-session-id) via - // its persisted fingerprint + content-hash overlap. (issue #796) + // --- Tier 1 → 3b: overlap-proven restart adoption --- + // A new known-header value is never continuity proof by itself. Before + // minting a fresh session, adopt only when the persisted fingerprint and + // project-scoped leading-user-message overlap prove that this is the same + // conversation. Successful publication below still revokes an old value + // for rotation-eligible headers such as x-session-affinity. (issue #796) const adopted = await adoptByFingerprint({ req, headers, projectPath, + gitRemote: trustedAdoptionRemote(projectPath, headers), known, msgCount: req.messages.length, + requestGeneration, config, credentialFingerprint, }); if (adopted) return adopted; - // Genuinely new session — no predecessor or ambiguous concurrent sessions. + // If a lower-priority confirmed identity existed but project validation + // rejected the migration, keep this replacement provisional until success. + // A completely new header can retain the normal eager session bootstrap. const sessionID = generateSessionID(); + if (fallbackMatch) { + setProvisionalHeaderMapping(indexKey, sessionID); + return { + sessionID, + isNew: true, + tier: 1, + provisionalIdentity: true, + provisionalKey: indexKey, + }; + } headerSessionIndex.set(indexKey, sessionID); saveSessionTracking(sessionID, { headerSessionId: known.sessionId, @@ -4865,15 +5969,20 @@ async function identifySession( } // --- Tier 2: Learned headers --- - // Check if any existing session has a promoted header that matches - // a header value in the current request. - for (const [sid, state] of sessions) { - if (!state.headerSessionId || !state.headerName) continue; - if ((state.credentialFingerprint ?? "") !== credentialFingerprint) continue; - const currentValue = headers[state.headerName]; - if (currentValue && currentValue === state.headerSessionId) { - return { sessionID: sid, isNew: false, tier: 2 }; - } + // Resolve through the shared index so multiple headers identifying different + // sessions fail closed instead of selecting insertion order. + const indexedResolution = resolveIndexedSession(req, config, true); + if (indexedResolution.kind === "ambiguous") { + throw new Error("ambiguous session headers"); + } + if (indexedResolution.kind === "match") { + return { + sessionID: indexedResolution.sessionID, + isNew: false, + tier: 2, + provisionalIdentity: indexedResolution.provisional, + provisionalKey: indexedResolution.provisionalKey, + }; } // --- Tier 2.5: Context markers (injected by Hermes plugin pre_llm_call) --- @@ -4909,53 +6018,79 @@ async function identifySession( ? { tenantFingerprint: credentialFingerprint } : { authSuffix: cred ? authFingerprint(cred) : "" }, ); + if (requestGeneration !== undefined) { + assertCurrentPipelineGeneration(req.signal, requestGeneration); + } const msgCount = req.messages.length; // Find the best matching session: same fingerprint + closest message count let bestMatch: { sid: string; countDiff: number } | null = null; + let ambiguousBestMatch = false; - for (const [sid, state] of sessions) { - if ((state.credentialFingerprint ?? "") !== credentialFingerprint) continue; - if (state.fingerprint !== fingerprint) continue; + if (cred) { + for (const [sid, state] of sessions) { + if (state.credentialFingerprint !== credentialFingerprint) continue; + if (state.fingerprint !== fingerprint) continue; + if ( + (projectPathSource === "header" || projectPathSource === "inferred") && + state.projectPathProvisional === false && + state.projectPath !== projectPath + ) { + continue; + } - const diff = msgCount - state.messageCount; + const diff = msgCount - state.messageCount; - // Normal session: count grows by 2–6 per turn. - // Fork: count drops significantly (parent at 600, fork at 300). - // Reject if the count dropped too far (likely a fork). - if (diff < -MESSAGE_COUNT_PROXIMITY_THRESHOLD) continue; + // Normal session: count grows by 2–6 per turn. + // Fork: count drops significantly (parent at 600, fork at 300). + // Reject if the count dropped too far (likely a fork). + if (diff < -MESSAGE_COUNT_PROXIMITY_THRESHOLD) continue; - const absDiff = Math.abs(diff); - if (!bestMatch || absDiff < bestMatch.countDiff) { - bestMatch = { sid, countDiff: absDiff }; + const absDiff = Math.abs(diff); + if (!bestMatch || absDiff < bestMatch.countDiff) { + bestMatch = { sid, countDiff: absDiff }; + ambiguousBestMatch = false; + } else if (absDiff === bestMatch.countDiff) { + ambiguousBestMatch = true; + } } } + if (ambiguousBestMatch) bestMatch = null; if (bestMatch) { // Run header learning on the matched session (Tier 2 bootstrap). const state = sessions.get(bestMatch.sid); if (state && !state.headerSessionId) { - const result = learnHeaders(state.candidateHeaders, headers); - state.candidateHeaders = result.updatedCandidates; + const candidateSnapshot = state.candidateHeaders + ? new Map( + Array.from(state.candidateHeaders, ([name, candidate]) => [ + name, + { ...candidate }, + ]), + ) + : undefined; + const result = learnHeaders(candidateSnapshot, headers, { + commitGlobal: false, + }); if (result.promoted) { - state.headerSessionId = result.promoted.value; - state.headerName = result.promoted.name; - // Index the promoted header for future Tier 2 lookups. + // Preserve retry continuity in memory, but do not authorize the learned + // header until a successful response confirms it in postResponse. const indexKey = sessionIndexKey( credentialFingerprint, result.promoted.name, result.promoted.value, ); - headerSessionIndex.set(indexKey, bestMatch.sid); - // Persist immediately — rare event, critical for post-restart correlation - saveSessionTracking(bestMatch.sid, { - headerSessionId: result.promoted.value, - headerName: result.promoted.name, - credentialFingerprint, - }); + setProvisionalHeaderMapping(indexKey, bestMatch.sid); log.info( - `session ${bestMatch.sid.slice(0, 16)}: promoted header ${result.promoted.name} for Tier 2 identification`, + `session ${bestMatch.sid.slice(0, 16)}: provisional header promotion ${result.promoted.name}`, ); + return { + sessionID: bestMatch.sid, + isNew: false, + tier: 3, + provisionalIdentity: true, + provisionalKey: indexKey, + }; } } return { sessionID: bestMatch.sid, isNew: false, tier: 3 }; @@ -4971,8 +6106,10 @@ async function identifySession( req, headers, projectPath, + gitRemote: trustedAdoptionRemote(projectPath, headers), known: null, msgCount, + requestGeneration, config, credentialFingerprint, }); @@ -5578,6 +6715,8 @@ export function buildStreamingResponse( sessionState: SessionState; cacheOptions: AnthropicCacheOptions; upstreamRoute?: ResolvedRequestUpstreamRoute; + /** Suppress recall-result retention for amnesia/no-store turns. */ + noStore?: boolean; /** True iff the inbound CLIENT speaks Anthropic SSE. Controls whether the * recall marker is emitted as its own Anthropic SSE message envelope * (split) or as an inline synthetic text content block (which the @@ -5926,28 +7065,30 @@ export function buildStreamingResponse( ]; }, ); - addRecallStoreEntry( - recallContext.sessionState.recallStore, - storeKey, - { - toolUseId: recallBlock.id, - anchorId, - anchorContextId, - input, - position, - result, - ...(companionToolUses.length > 0 - ? { companionToolUses } - : {}), - }, - ); - // Persist the store (v46) so the marker still expands byte-identically - // after a gateway restart instead of leaking raw marker text upstream. - saveSessionTracking(recallContext.sessionState.sessionID, { - recallStore: serializeRecallStore( + if (!recallContext.noStore) { + addRecallStoreEntry( recallContext.sessionState.recallStore, - ), - }); + storeKey, + { + toolUseId: recallBlock.id, + anchorId, + anchorContextId, + input, + position, + result, + ...(companionToolUses.length > 0 + ? { companionToolUses } + : {}), + }, + ); + // Persist the store (v46) so the marker still expands byte-identically + // after a gateway restart instead of leaking raw marker text upstream. + saveSessionTracking(recallContext.sessionState.sessionID, { + recallStore: serializeRecallStore( + recallContext.sessionState.recallStore, + ), + }); + } // Emit marker — split into its own SSE message envelope for Anthropic-native // clients (so the marker renders as a DISTINCT assistant message in @@ -6396,7 +7537,11 @@ export function buildStreamingResponse( export function streamResponsesRecallAware( upstreamResponse: Response, opts: { - onComplete: (response: GatewayResponse) => void; + onComplete: (response: GatewayResponse, successful: boolean) => void; + onTransactionReady?: (transaction: { + commit: () => void; + rollback: () => void; + }) => void; sessionID?: string; maxRecallDepth?: number; maxDeferredBytes?: number; @@ -6404,6 +7549,7 @@ export function streamResponsesRecallAware( maxRetainedStateBytes?: number; maxStreamBytes?: number; maxSSEFrames?: number; + validation?: "public" | "codex"; /** Caller abort combined with the stream's client-disconnect controller. */ signal?: AbortSignal; /** @@ -6544,7 +7690,11 @@ export function streamResponsesRecallAware( } }; let transactionBaseline: ResponsesAccState | undefined; + let transactionProviderUsage: GatewayUsage = { ...ZERO_USAGE }; const transactionRollbacks: Array<() => void> = []; + let deferredTransaction: + | { commit: () => void; rollback: () => void } + | undefined; const restoreTransactionBaseline = (): void => { if (!transactionBaseline) return; state.id = transactionBaseline.id; @@ -6744,6 +7894,8 @@ export function streamResponsesRecallAware( if ( !item || typeof item.type !== "string" || + !isSupportedResponsesOutputItemType(item.type) || + !isValidResponsesOutputItemStatus(item.type, item.status, "added") || typeof item.id !== "string" || item.id.length === 0 || (item.type === "function_call" && @@ -6861,12 +8013,7 @@ export function streamResponsesRecallAware( const item = parsed.item as Record | undefined; if ( event === "response.output_item.done" && - (!item || - item.type !== declared?.type || - item.id !== declared?.id || - item.call_id !== declared?.call_id || - item.name !== declared?.name || - (declared?.type === "message" && item.role !== declared.role)) + (!item || !declared || !responsesDoneItemMatchesAdded(item, declared)) ) { throw new Error( `Responses output_item.done changed item identity for index ${outputIndex}`, @@ -7378,7 +8525,11 @@ export function streamResponsesRecallAware( if (!response || typeof response.id !== "string" || !response.id) { throw new Error("response.created missing response identity"); } - if (response.status !== undefined && response.status !== "in_progress") { + if ( + response.status !== undefined && + response.status !== "in_progress" && + !(opts.validation === "codex" && response.status === "queued") + ) { throw new Error("response.created has invalid status"); } if ( @@ -7431,7 +8582,9 @@ export function streamResponsesRecallAware( throw new Error("Responses terminal event has nonterminal status"); } if ( - (event === "response.completed" && status !== "completed") || + (event === "response.completed" && + status !== "completed" && + !(opts.validation === "codex" && status === "incomplete")) || (event === "response.incomplete" && status !== "incomplete") || (event === "response.failed" && status !== "failed" && @@ -7439,6 +8592,27 @@ export function streamResponsesRecallAware( ) { throw new Error("Responses terminal event contradicts response status"); } + if (status === "incomplete") { + const details = response?.incomplete_details; + if ( + details !== undefined && + details !== null && + (typeof details !== "object" || Array.isArray(details)) + ) { + throw new Error("malformed Responses terminal event"); + } + const reason = + details && typeof details === "object" && !Array.isArray(details) + ? (details as Record).reason + : undefined; + if ( + reason !== undefined && + reason !== "max_output_tokens" && + reason !== "content_filter" + ) { + throw new Error("malformed Responses terminal event"); + } + } lifecycle.terminal = true; } }; @@ -7475,90 +8649,6 @@ export function streamResponsesRecallAware( } if (changed) acc.rawItems.set(outputIndex, { ...raw, summary }); }; - const terminalTextPartsMatch = ( - actual: unknown, - streamed: unknown, - ): boolean => { - if (!Array.isArray(actual) || !Array.isArray(streamed)) return false; - if (actual.length !== streamed.length) return false; - return streamed.every((streamedPart, index) => { - const actualPart = actual[index]; - if ( - !streamedPart || - typeof streamedPart !== "object" || - Array.isArray(streamedPart) || - !actualPart || - typeof actualPart !== "object" || - Array.isArray(actualPart) - ) { - return false; - } - const streamedRecord = streamedPart as Record; - const actualRecord = actualPart as Record; - if (actualRecord.type !== streamedRecord.type) return false; - if ( - typeof streamedRecord.type === "string" && - ["output_text", "reasoning_text", "summary_text", "refusal"].includes( - streamedRecord.type, - ) - ) { - return ( - partValue(streamedRecord.type, actualRecord, "terminal content") === - partValue(streamedRecord.type, streamedRecord, "streamed content") - ); - } - return isDeepStrictEqual(actualRecord, streamedRecord); - }); - }; - const terminalItemMatches = ( - actual: Record, - streamed: Record, - ): boolean => { - if (actual.type !== streamed.type || actual.id !== streamed.id) - return false; - if (actual.status !== undefined && typeof actual.status !== "string") { - return false; - } - if ( - streamed.status !== undefined && - actual.status !== undefined && - actual.status !== streamed.status - ) { - return false; - } - if (actual.type === "function_call") { - return ( - actual.call_id === streamed.call_id && - actual.name === streamed.name && - actual.arguments === streamed.arguments - ); - } - if (actual.type === "message") { - return ( - actual.role === streamed.role && - terminalTextPartsMatch(actual.content, streamed.content) - ); - } - if (actual.type === "reasoning") { - for (const field of ["summary", "content"]) { - if (streamed[field] !== undefined) { - if (!terminalTextPartsMatch(actual[field], streamed[field])) { - return false; - } - } - } - if ( - streamed.encrypted_content !== undefined && - !isDeepStrictEqual(actual.encrypted_content, streamed.encrypted_content) - ) { - return false; - } - return true; - } - const { status: _actualStatus, ...actualSemantic } = actual; - const { status: _streamedStatus, ...streamedSemantic } = streamed; - return isDeepStrictEqual(actualSemantic, streamedSemantic); - }; const assertTerminalReasoningMatchesLifecycle = ( lifecycle: OutputLifecycle, actual: Record, @@ -7603,7 +8693,12 @@ export function streamResponsesRecallAware( if (acc.id && response.id !== acc.id) { throw new Error("Responses terminal event changed response identity"); } - if (response.output === undefined) return; + if (response.output === undefined) { + if (opts.validation === "public" && response.status === "completed") { + throw new Error("Responses terminal output must be an array"); + } + return; + } if (!Array.isArray(response.output)) { throw new Error("Responses terminal output must be an array"); } @@ -7617,6 +8712,12 @@ export function streamResponsesRecallAware( return item as Record; }); const expected = [...acc.rawItems.entries()].sort(([a], [b]) => a - b); + if ( + opts.validation === "public" && + actualOutput.length !== expected.length + ) { + throw new Error("Responses terminal output changed streamed item"); + } let expectedIndex = 0; for (const actual of actualOutput) { const isReference = actual.type === "item_reference"; @@ -7639,8 +8740,11 @@ export function streamResponsesRecallAware( if (matchIndex < 0) { throw new Error("Responses terminal output changed streamed item"); } + if (opts.validation === "public" && matchIndex !== expectedIndex) { + throw new Error("Responses terminal output changed streamed item"); + } const [outputIndex, streamed] = expected[matchIndex]; - if (!isReference && !terminalItemMatches(actual, streamed)) { + if (!isReference && !responsesTerminalItemMatches(actual, streamed)) { throw new Error("Responses terminal output changed streamed item"); } if (!isReference && actual.type === "reasoning") { @@ -7825,11 +8929,11 @@ export function streamResponsesRecallAware( return encoder.encode(output); }; - const finish = (resp: GatewayResponse): boolean => { + const finish = (resp: GatewayResponse, successful: boolean): boolean => { if (completionAttempted) return completed; completionAttempted = true; try { - opts.onComplete(resp); + opts.onComplete(resp, successful); completed = true; return true; } catch (err) { @@ -8144,17 +9248,21 @@ export function streamResponsesRecallAware( } signal.throwIfAborted(); }; - const safeEnqueue = async (chunk: Uint8Array): Promise => { + const safeEnqueue = async ( + chunk: Uint8Array, + afterEnqueue?: () => void, + ): Promise => { if (cancelled) return false; await waitForDemand(); if (cancelled) return false; try { controller.enqueue(sequenceChunk(chunk)); - return true; } catch { cancelled = true; return false; } + afterEnqueue?.(); + return true; }; const safeClose = (): void => { cleanupAbort(); @@ -8375,6 +9483,7 @@ export function streamResponsesRecallAware( ); } // No recall — forward the terminal event verbatim. + const finalResponse = finalizeResponsesAcc(state); if ( !(await safeEnqueue( encoder.encode( @@ -8385,13 +9494,19 @@ export function streamResponsesRecallAware( : JSON.stringify(terminalParsed), ), ), + () => { + terminalDelivered = true; + finish( + finalResponse, + state.terminalEvent === "response.completed", + ); + }, )) ) break; cancelAndReleaseReader(reader, signal.reason); principalReader = null; clearKeepalive(); - finish(finalizeResponsesAcc(state)); safeClose(); return; } @@ -8422,6 +9537,7 @@ export function streamResponsesRecallAware( items: new Map(state.items), rawItems: new Map(state.rawItems), }; + transactionProviderUsage = { ...ZERO_USAGE }; const pendingCommits: Array<() => void> = []; const transactionalEvents: Uint8Array[] = []; let transactionalBytes = 0; @@ -8804,6 +9920,11 @@ export function streamResponsesRecallAware( } mergeUsage(state.usage, contState.usage); }; + assertUsageMergeable( + transactionProviderUsage, + contState.usage, + ); + mergeUsage(transactionProviderUsage, contState.usage); if (continuationFailed) { throw new Error( "recall follow-up returned response.failed", @@ -8907,6 +10028,9 @@ export function streamResponsesRecallAware( for (const chunk of heldContinuationEvents) { queueTransactional(chunk); } + for (const index of contRecallIndices) { + recallIndices.add(shiftedOutputIndex(index, contIndex)); + } mergeContinuation(); if (!nextRecall || !nextExecuted || contOtherTool) { state.stopReason = contState.stopReason; @@ -8963,20 +10087,56 @@ export function streamResponsesRecallAware( } } if ( - !(await safeEnqueue(encoder.encode(buildTerminal(visibleResp)))) + !(await safeEnqueue( + encoder.encode(buildTerminal(visibleResp)), + () => { + terminalDelivered = true; + const successful = + state.terminalEvent === "response.completed"; + let transactionSettled = false; + const transaction = { + commit: () => { + if (transactionSettled) return; + try { + for (const commit of pendingCommits) commit(); + transactionSettled = true; + pendingCommits.length = 0; + transactionRollbacks.length = 0; + transactionBaseline = undefined; + } catch (error) { + pendingCommits.length = 0; + transaction.rollback(); + throw error; + } + }, + rollback: () => { + if (transactionSettled) return; + transactionSettled = true; + pendingCommits.length = 0; + rollbackTransaction(); + }, + }; + deferredTransaction = transaction; + if (successful) opts.onTransactionReady?.(transaction); + if (!finish(visibleResp, successful)) { + transaction.rollback(); + throw new Error( + "recall onComplete failed after delivery", + ); + } + if (successful) { + if (!opts.onTransactionReady) transaction.commit(); + } else { + transaction.rollback(); + } + }, + )) ) { throw new Error( "client disconnected while delivering recall terminal", ); } - terminalDelivered = true; if (cancelled) throw signal.reason; - if (!finish(visibleResp)) { - throw new Error("recall onComplete failed after delivery"); - } - for (const commit of pendingCommits.splice(0)) commit(); - transactionRollbacks.length = 0; - transactionBaseline = undefined; cancelAndReleaseReader(reader, signal.reason); principalReader = null; safeClose(); @@ -9034,6 +10194,30 @@ export function streamResponsesRecallAware( err, ); } + const failedResponse = finalizeResponsesAcc(state); + try { + assertUsageMergeable( + failedResponse.usage ?? ZERO_USAGE, + transactionProviderUsage, + ); + failedResponse.usage ??= { ...ZERO_USAGE }; + mergeUsage(failedResponse.usage, transactionProviderUsage); + } catch (usageError) { + log.error( + "failed to merge recall continuation usage for accounting:", + usageError, + ); + } + transactionProviderUsage = { ...ZERO_USAGE }; + failedResponse.content = failedResponse.content.filter( + (block) => + (block.type !== "tool_use" || block.name !== RECALL_TOOL_NAME) && + (block.type !== "text" || !parseRecallAnchor(block.text)), + ); + failedResponse.rawOutputItems = failedResponse.rawOutputItems?.filter( + (item) => + item.type !== "function_call" || item.name !== RECALL_TOOL_NAME, + ); await safeEnqueue( encoder.encode( formatResponsesEvent( @@ -9057,18 +10241,8 @@ export function streamResponsesRecallAware( }), ), ), + () => finish(failedResponse, false), ); - const failedResponse = finalizeResponsesAcc(state); - failedResponse.content = failedResponse.content.filter( - (block) => - (block.type !== "tool_use" || block.name !== RECALL_TOOL_NAME) && - (block.type !== "text" || !parseRecallAnchor(block.text)), - ); - failedResponse.rawOutputItems = failedResponse.rawOutputItems?.filter( - (item) => - item.type !== "function_call" || item.name !== RECALL_TOOL_NAME, - ); - finish(failedResponse); safeClose(); } })().catch((error) => { @@ -9092,7 +10266,8 @@ export function streamResponsesRecallAware( resumeDemand = undefined; cancelled = true; cleanupAbort(); - rollbackTransaction(); + if (deferredTransaction) deferredTransaction.rollback(); + else rollbackTransaction(); abortController.abort( new DOMException("Responses client disconnected", "AbortError"), ); @@ -9221,6 +10396,7 @@ export async function accumulateNonStreamResponse( | "gemini" = "anthropic", codex = false, signal?: AbortSignal, + requireValidCompletion = false, ): Promise { // Some providers (the ChatGPT/Copilot/Codex backend, DeepSeek) return an SSE // stream even when stream: false was sent — sometimes WITHOUT the @@ -9254,6 +10430,7 @@ export async function accumulateNonStreamResponse( signal, validation: codex ? "codex" : "public", stopAtTerminal: true, + requireCompletedTerminal: true, }); case "gemini": return accumulateGeminiSSEStream(sse, { @@ -9271,18 +10448,262 @@ export async function accumulateNonStreamResponse( } } - const json = JSON.parse(body) as Record; - switch (protocol) { - case "openai": - return accumulateOpenAINonStreamJSON(json); - case "openai-responses": - return accumulateResponsesNonStreamJSON(json); - case "gemini": - return parseGeminiResponseJSON(json); - default: - // Anthropic (incl. Bedrock via bedrock-mantle, which returns the native - // Anthropic non-streaming JSON shape). - return accumulateAnthropicNonStreamJSON(json); + const json = JSON.parse(body) as Record; + if (protocol === "openai-responses") { + const { response, status } = parseResponsesNonStreamEnvelope(json); + if (status !== "completed") { + throw new ResponsesTerminalError(response, status); + } + return response; + } + if (requireValidCompletion) { + assertValidNonStreamCompletion(json, protocol); + } + switch (protocol) { + case "openai": + return accumulateOpenAINonStreamJSON(json); + case "gemini": + return parseGeminiResponseJSON(json); + default: + // Anthropic (incl. Bedrock via bedrock-mantle, which returns the native + // Anthropic non-streaming JSON shape). + return accumulateAnthropicNonStreamJSON(json); + } +} + +function parseResponsesNonStreamEnvelope(json: Record): { + response: GatewayResponse; + status: string; +} { + const response = accumulateResponsesNonStreamJSON(json); + const status = typeof json.status === "string" ? json.status : "unknown"; + if (status === "completed" || status === "incomplete") { + assertValidNonStreamCompletion(json, "openai-responses"); + } + return { response, status }; +} + +async function preserveIncompleteResponsesTerminal( + operation: Promise, +): Promise { + try { + return await operation; + } catch (error) { + if ( + error instanceof ResponsesTerminalError && + error.status === "incomplete" + ) { + return error.response; + } + throw error; + } +} + +function assertValidNonStreamCompletion( + json: Record, + protocol: "anthropic" | "openai" | "openai-responses" | "vertex" | "gemini", +): void { + if (json.error !== undefined && json.error !== null) { + throw new Error("upstream response contained an error"); + } + + if (protocol === "openai") { + const choices = json.choices; + const first = Array.isArray(choices) ? choices[0] : undefined; + if ( + !first || + typeof first !== "object" || + Array.isArray(first) || + !(first as Record).message || + typeof (first as Record).finish_reason !== "string" + ) { + throw new Error("upstream OpenAI request did not complete"); + } + return; + } + + if (protocol === "openai-responses") { + const status = json.status; + if ( + (status !== "completed" && status !== "incomplete") || + typeof json.id !== "string" || + typeof json.model !== "string" || + !Array.isArray(json.output) || + !json.usage || + typeof json.usage !== "object" || + Array.isArray(json.usage) + ) { + throw new Error("upstream Responses request did not complete"); + } + const seenItemIDs = new Set(); + for (const rawItem of json.output) { + if (!rawItem || typeof rawItem !== "object" || Array.isArray(rawItem)) { + throw new Error("upstream Responses request did not complete"); + } + const item = rawItem as Record; + if ( + typeof item.type !== "string" || + !item.type || + !isSupportedResponsesOutputItemType(item.type) || + typeof item.id !== "string" || + !item.id || + seenItemIDs.has(item.id) + ) { + throw new Error("upstream Responses request did not complete"); + } + seenItemIDs.add(item.id); + if (item.type === "message") { + const validItemStatus = + item.status === "completed" || + (status === "incomplete" && item.status === "incomplete"); + if ( + item.role !== "assistant" || + !validItemStatus || + !Array.isArray(item.content) + ) { + throw new Error("upstream Responses request did not complete"); + } + for (const rawPart of item.content) { + if ( + !rawPart || + typeof rawPart !== "object" || + Array.isArray(rawPart) + ) { + throw new Error("upstream Responses request did not complete"); + } + const part = rawPart as Record; + if ( + (part.type === "output_text" && typeof part.text !== "string") || + (part.type === "refusal" && typeof part.refusal !== "string") || + (part.type !== "output_text" && part.type !== "refusal") + ) { + throw new Error("upstream Responses request did not complete"); + } + } + } else if (item.type === "function_call") { + const validItemStatus = + item.status === "completed" || + item.status === "failed" || + (status === "incomplete" && item.status === "incomplete"); + if ( + typeof item.call_id !== "string" || + !item.call_id || + typeof item.name !== "string" || + !item.name || + typeof item.arguments !== "string" || + !validItemStatus + ) { + throw new Error("upstream Responses request did not complete"); + } + } else if (item.type === "reasoning") { + const validItemStatus = + item.status === undefined || + item.status === "completed" || + (status === "incomplete" && item.status === "incomplete"); + if (!validItemStatus) { + throw new Error("upstream Responses request did not complete"); + } + for (const [field, partType] of [ + ["summary", "summary_text"], + ["content", "reasoning_text"], + ] as const) { + const parts = item[field]; + if (parts === undefined) continue; + if (!Array.isArray(parts)) { + throw new Error("upstream Responses request did not complete"); + } + for (const rawPart of parts) { + if ( + !rawPart || + typeof rawPart !== "object" || + Array.isArray(rawPart) || + (rawPart as Record).type !== partType || + typeof (rawPart as Record).text !== "string" + ) { + throw new Error("upstream Responses request did not complete"); + } + } + } + if ( + item.encrypted_content !== undefined && + item.encrypted_content !== null && + typeof item.encrypted_content !== "string" + ) { + throw new Error("upstream Responses request did not complete"); + } + } else if (item.type === "item_reference") { + // A standalone non-stream response has no streamed item lifecycle to + // resolve this reference against; accepting it would silently erase + // provider output during normalization. + throw new Error("upstream Responses request did not complete"); + } else { + if ( + !isValidResponsesOutputItemStatus(item.type, item.status, "terminal") + ) { + throw new Error("upstream Responses request did not complete"); + } + } + } + if (status === "incomplete") { + const details = json.incomplete_details; + if ( + details !== undefined && + details !== null && + (typeof details !== "object" || + Array.isArray(details) || + typeof (details as Record).reason !== "string") + ) { + throw new Error("upstream Responses request did not complete"); + } + const reason = + details && typeof details === "object" && !Array.isArray(details) + ? (details as Record).reason + : undefined; + if ( + reason !== undefined && + reason !== "max_output_tokens" && + reason !== "content_filter" + ) { + throw new Error("upstream Responses request did not complete"); + } + } + return; + } + + if (protocol === "gemini") { + const candidates = json.candidates; + const first = Array.isArray(candidates) ? candidates[0] : undefined; + const promptFeedback = json.promptFeedback; + const blockReason = + promptFeedback && + typeof promptFeedback === "object" && + !Array.isArray(promptFeedback) + ? (promptFeedback as Record).blockReason + : undefined; + if ( + (!first || + typeof first !== "object" || + Array.isArray(first) || + typeof (first as Record).finishReason !== "string") && + typeof blockReason !== "string" + ) { + throw new Error("upstream Gemini request did not complete"); + } + return; + } + + if ( + json.type !== "message" || + json.role !== "assistant" || + typeof json.id !== "string" || + typeof json.model !== "string" || + !Array.isArray(json.content) || + typeof json.stop_reason !== "string" || + !json.usage || + typeof json.usage !== "object" || + Array.isArray(json.usage) + ) { + throw new Error("upstream Anthropic request did not complete"); } } @@ -9503,7 +10924,14 @@ export function accumulateResponsesNonStreamJSON( // Map Responses API status to gateway stop reason const status = json.status as string | undefined; let stopReason = "end_turn"; - if (status === "incomplete") stopReason = "max_tokens"; + if (status === "incomplete") { + const details = json.incomplete_details; + const reason = + details && typeof details === "object" && !Array.isArray(details) + ? (details as Record).reason + : undefined; + stopReason = reason === "content_filter" ? "content_filter" : "max_tokens"; + } if (content.some((b) => b.type === "tool_use") && stopReason === "end_turn") { stopReason = "tool_use"; } @@ -9792,6 +11220,7 @@ export function recordCacheTurnUsage( requestBody?: string, /** Active gen_ai.chat span to enrich with divergence diagnostics. */ genAiSpan?: Sentry.Span, + endSpan?: () => void, ): CacheBustCause | undefined { // Capture the idle-resume flag up front: it is consumed (set false) inside // the block below but is still needed afterwards by recordCacheUsage so a @@ -9868,7 +11297,8 @@ export function recordCacheTurnUsage( // session-state bookkeeping that never touches the span, and ending the span // first means a throw in recordCacheUsage can't leak an unfinished span. if (genAiSpan) { - genAiSpan.end(); + if (endSpan) endSpan(); + else genAiSpan.end(); } // --- Consecutive bust tracking for tier-based decisions --- @@ -9937,12 +11367,11 @@ export function storeTurnTemporal(input: { return; } - // Resolve (and, if needed, lazily backfill/merge) the project OUTSIDE the - // savepoint. `ensureProject` can reach `mergeProjectInternal`'s raw - // `BEGIN IMMEDIATE` on the NULL-git_remote backfill-with-conflict path, which - // would nest inside the savepoint's transaction and throw "cannot start a - // transaction within a transaction". Warming it here makes the `ensureProject` - // calls inside temporal.store / recordToolCalls cheap cache hits. (#1084.) + // Resolve (and, if needed, lazily backfill/merge) the project before the + // temporal savepoint. mergeProjectInternal is nested-savepoint safe, so this + // whole operation may itself run inside a larger transaction. Warming here + // makes the ensureProject calls inside temporal.store / recordToolCalls cheap + // cache hits. (#1084.) ensureProject(projectPath); withSavepoint("post_response_temporal", () => { @@ -10015,6 +11444,34 @@ export function storeTurnTemporal(input: { }); } +function accountConversationUsage( + usage: GatewayUsage, + model: string, + sessionID: string, + resolvedConversationTTL: "5m" | "1h" | undefined, +): AnthropicUsage { + const usageForSentry: AnthropicUsage = { + input_tokens: usage.inputTokens, + output_tokens: usage.outputTokens, + cache_read_input_tokens: usage.cacheReadInputTokens, + cache_creation_input_tokens: usage.cacheCreationInputTokens, + }; + setSentryCacheContext(usage); + emitCostMetric( + model, + usageForSentry, + "conversation", + resolvedConversationTTL, + ); + recordConversationCost( + sessionID, + model, + usageForSentry, + resolvedConversationTTL, + ); + return usageForSentry; +} + /** * Run after a successful response: calibrate, store temporal messages, * and schedule background work (distillation, curation). @@ -10028,13 +11485,19 @@ function postResponseForTenant( requestBody?: string, /** Active gen_ai.chat span to finalize with usage attributes. */ genAiSpan?: Sentry.Span, -): void { + /** Storage policy captured when this turn resolved its session. */ + suppressTemporalStorage = false, + endSpan?: () => void, +): boolean { + postResponseStartObserver?.(); const { sessionID, projectPath } = sessionState; // Guard: resp.usage can be undefined at runtime for vLLM / partial responses. const usage = resp.usage ?? ZERO_USAGE; try { + confirmKnownSessionHeader(req, sessionState, config); + // --- Calibrate overhead from real token counts --- const actualInput = (usage.inputTokens ?? 0) + @@ -10043,23 +11506,10 @@ function postResponseForTenant( calibrate(actualInput, sessionID, getLastTransformedCount(sessionID)); // --- Sentry cache context + cost metric --- - setSentryCacheContext(usage); - const usageForSentry: AnthropicUsage = { - input_tokens: usage.inputTokens, - output_tokens: usage.outputTokens, - cache_read_input_tokens: usage.cacheReadInputTokens, - cache_creation_input_tokens: usage.cacheCreationInputTokens, - }; - emitCostMetric( + const usageForSentry = accountConversationUsage( + usage, resp.model, - usageForSentry, - "conversation", - sessionState.resolvedConversationTTL, - ); - recordConversationCost( sessionID, - resp.model, - usageForSentry, sessionState.resolvedConversationTTL, ); if (genAiSpan) { @@ -10073,13 +11523,19 @@ function postResponseForTenant( // the whole pipeline. The seam also enriches and ENDS genAiSpan (before its // own recordCacheUsage call) so the extraction is ordering-identical to the // original inlined block. See issue #928. + if (suppressTemporalStorage) { + sessionState.cacheAnalytics.lastRequestBody = null; + sessionState.cacheAnalytics.lastNormalizedBody = null; + sessionState.cacheAnalytics.lastRequestBodyLength = 0; + } recordCacheTurnUsage( sessionState, usage, resp.model, projectPath, - requestBody, + suppressTemporalStorage ? undefined : requestBody, genAiSpan, + endSpan, ); // Admin credentials are authorized at dispatch time and never retained in // session snapshots. The idle warmer still receives gateway-global extras, @@ -10111,8 +11567,7 @@ function postResponseForTenant( // Note: tool-call outcomes for a tool_use seeded during a no-store turn are // intentionally dropped — the seed row never exists, so the later // tool_result UPDATE is a harmless no-op (no phantom 'pending' rows leak). - const noStore = - sessionState.amnesia || req.rawHeaders["x-lore-no-store"] === "true"; + const noStore = suppressTemporalStorage; // Persist (and tool-trace) this turn's messages, batched into one savepoint. // Extracted seam — see storeTurnTemporal (#1084). @@ -10300,14 +11755,81 @@ function postResponseForTenant( } // --- Schedule background work (fire-and-forget) --- + saveSessionTracking(sessionID, { + messageCount: sessionState.messageCount, + turnsSinceCuration: sessionState.turnsSinceCuration, + consecutiveTextOnlyTurns: sessionState.consecutiveTextOnlyTurns, + projectPath: sessionState.projectPath || null, + projectPathProvisional: sessionState.projectPathProvisional === true, + ...(sessionState.compactionAnomalyPending + ? { compactionAnomalyPending: true } + : {}), + }); + if (!sessionState.headerSessionId) { + const result = learnHeaders( + sessionState.candidateHeaders, + req.rawHeaders, + ); + sessionState.candidateHeaders = result.updatedCandidates; + } if (!noStore) { scheduleBackgroundWork(sessionState, config); } + return true; } catch (e) { log.error("post-response processing failed:", e); + return false; + } finally { + endSpan?.(); + } +} + +/** Record validated provider usage without publishing successful-turn state. */ +function accountUnsuccessfulResponse( + resp: GatewayResponse, + sessionID: string, + resolvedConversationTTL: "5m" | "1h" | undefined, + genAiSpan: Sentry.Span | undefined, + endSpan: () => void, + markDirty?: () => void, +): void { + const usage = resp.usage ?? ZERO_USAGE; + const hasUsage = Object.values(usage).some( + (tokens) => typeof tokens === "number" && tokens > 0, + ); + try { + if (hasUsage) { + markDirty?.(); + const usageForSentry = accountConversationUsage( + usage, + resp.model, + sessionID, + resolvedConversationTTL, + ); + if (genAiSpan) { + setGenAiUsageAttributes(genAiSpan, usageForSentry, resp.model); + } + } + } finally { + genAiSpan?.setStatus({ + code: 2, + message: "upstream response did not complete", + }); + endSpan(); } } +function conversationTTLForAccounting( + sessionID: string, +): "5m" | "1h" | undefined { + const liveTTL = sessions.get(sessionID)?.resolvedConversationTTL; + if (liveTTL === "5m" || liveTTL === "1h") return liveTTL; + const persistedTTL = loadSessionTracking(sessionID)?.resolvedConversationTTL; + return persistedTTL === "5m" || persistedTTL === "1h" + ? persistedTTL + : undefined; +} + function postResponse( req: GatewayRequest, resp: GatewayResponse, @@ -10315,8 +11837,10 @@ function postResponse( config: GatewayConfig, requestBody?: string, genAiSpan?: Sentry.Span, -): void { - withTenant(sessionState.storageTenantId ?? "", () => + suppressTemporalStorage = false, + endSpan?: () => void, +): boolean { + return withTenant(sessionState.storageTenantId ?? "", () => postResponseForTenant( req, resp, @@ -10324,6 +11848,8 @@ function postResponse( config, requestBody, genAiSpan, + suppressTemporalStorage, + endSpan, ), ); } @@ -10348,6 +11874,10 @@ function scheduleBackgroundWorkForTenant( config: GatewayConfig, ): void { const { sessionID, projectPath } = sessionState; + const signal = AbortSignal.any([ + pipelineGenerationAbort.signal, + sessionLifecycleSignal(sessionID), + ]); // Skip background work when the session's auth credential is stale and no // fresh fallback is available — worker LLM calls would just 401. @@ -10435,6 +11965,7 @@ function scheduleBackgroundWorkForTenant( force: true, urgent: true, callType: "direct", + signal, // Never run meta-distillation while the conversation cache is warm. // Meta archives gen-0 rows and creates a gen-1 row, rewriting the // synthetic distilled prefix at messages[0/1] on the next turn. That @@ -10480,6 +12011,7 @@ function scheduleBackgroundWorkForTenant( skipMeta: true, callType: batchQueueEnabled ? "batch" : "direct", workerHealth: makeWorkerHealth(sessionID, "lore-distill"), + signal, // #627 Phase 1: stamp the session's gitHead on every distilled row. metadata: buildSessionMetadata(sessionState.gitHead), }), @@ -10562,6 +12094,7 @@ function scheduleBackgroundWorkForTenant( sessionID, model, workerHealth: makeWorkerHealth(sessionID, "lore-curator"), + signal, // #627 Phase 1: stamp the session's gitHead on curator entries. metadata: buildSessionMetadata(sessionState.gitHead), }), @@ -10572,6 +12105,7 @@ function scheduleBackgroundWorkForTenant( ) .then((result) => { if (!result) return; // skipped by circuit breaker + signal.throwIfAborted(); sessionState.turnsSinceCuration = 0; saveSessionTracking(sessionID, { turnsSinceCuration: 0 }); if ( @@ -10639,6 +12173,7 @@ export async function generateCompactionSummary(opts: { previousSummary?: string; sessionUpstream?: { providerID?: string; modelID?: string }; signal?: AbortSignal; + trackOperation?: (operation: Promise) => void; }): Promise { const { projectPath, sessionID, config, previousSummary, sessionUpstream } = opts; @@ -10654,35 +12189,39 @@ export async function generateCompactionSummary(opts: { if (temporal.undistilledCount(projectPath, sessionID) > 0) { const llm = getLLMClient(config); const model = getWorkerModel(sessionUpstream); - await promiseAgainstAbort( - () => - distillation.run({ - llm, - projectPath, - sessionID, - model, - force: true, - urgent: true, - callType: "direct", - signal: opts.signal, - workerHealth: makeWorkerHealth(sessionID, "lore-distill"), - // #627 Phase 1: stamp the session's gitHead on urgent-compaction rows. - // Compaction is invoked via HTTP intercept or /v1/compact, so we look up - // the session by ID rather than threading state through the call. - metadata: buildSessionMetadata(sessions.get(sessionID)?.gitHead), - }), - opts.signal, - ); + await promiseAgainstAbort(() => { + const operation = distillation.run({ + llm, + projectPath, + sessionID, + model, + force: true, + urgent: true, + callType: "direct", + signal: opts.signal, + workerHealth: makeWorkerHealth(sessionID, "lore-distill"), + // #627 Phase 1: stamp the session's gitHead on urgent-compaction rows. + // Compaction is invoked via HTTP intercept or /v1/compact, so we look up + // the session by ID rather than threading state through the call. + metadata: buildSessionMetadata(sessions.get(sessionID)?.gitHead), + }); + opts.trackOperation?.(operation); + return operation; + }, opts.signal); } // 2. Load distillation summaries + long-term knowledge. const distillations = distillation.loadForSession(projectPath, sessionID); const cfg = loreConfig(); const entries = cfg.knowledge.enabled - ? await promiseAgainstAbort( - () => ltm.forProjectOffloaded(projectPath, cfg.crossProject), - opts.signal, - ) + ? await promiseAgainstAbort(() => { + const operation = ltm.forProjectOffloaded( + projectPath, + cfg.crossProject, + ); + opts.trackOperation?.(operation); + return operation; + }, opts.signal) : []; opts.signal?.throwIfAborted(); const knowledge = entries.length @@ -10720,27 +12259,62 @@ export async function generateCompactionSummary(opts: { async function handleCompactionInner( req: GatewayRequest, config: GatewayConfig, + requestGeneration: number, + trackOperation: (operation: Promise) => void, + claimSession: (sessionID: string) => Promise, ): Promise { if (!req.rawHeaders["x-lore-project"]) { const markerProject = extractProjectMarker(req.messages); if (markerProject) req.rawHeaders["x-lore-project"] = markerProject; } const pathResult = getProjectPath(req.system, req.rawHeaders); - - const { sessionID } = await identifySession(req, pathResult.path, config); - stripContextMarkers(req.messages); - const sessionState = getOrCreateSession( - sessionID, + const credential = extractAuth(req.rawHeaders); + if (!credential) { + return errorResponse(401, "A provider credential is required"); + } + const sessionState = resolveAuthenticatedDirectSession( + req, pathResult.path, - pathResult.source, - requestCredentialFingerprint(req.rawHeaders, config) ?? "", - config, - ); - const projectPath = resolveSessionProjectPath( - pathResult, - sessionState, config, + false, ); + if ( + !sessionState || + (!sessionState.lastUpstream && + !streamingPostResponseFinalizers.has(sessionState.sessionID)) + ) { + return errorResponse(404, "No authenticated session found"); + } + if ( + sessionState.projectPathProvisional === true || + (pathResult.source !== "cwd" && + sessionState.projectPath !== pathResult.path) + ) { + return errorResponse( + 403, + "Project path does not match the authenticated session", + ); + } + const sessionID = sessionState.sessionID; + const authorizedProjectPath = sessionState.projectPath; + await claimSession(sessionID); + if (!confirmedIndexedIdentityResolvesTo(req, sessionID, config)) { + return errorResponse(404, "No authenticated session found"); + } + await awaitStreamingPostResponse(sessionID, req.signal); + assertCurrentPipelineGeneration(req.signal, requestGeneration); + if (!confirmedIndexedIdentityResolvesTo(req, sessionID, config)) { + return errorResponse(404, "No authenticated session found"); + } + if (!isConfidentlyBoundToProject(sessionState, authorizedProjectPath)) { + return errorResponse( + 403, + "Project path does not match the authenticated session", + ); + } + stripContextMarkers(req.messages); + const projectPath = sessionState.projectPath; + setSessionAuth(sessionID, credential, sessionState.lastUpstream?.providerID); // NOTE: the project binding is NOT persisted here — compaction never changes // the binding, and the preceding normal turn already persisted it. A restart // between the last normal turn and a compaction-only turn rehydrates the @@ -10749,7 +12323,14 @@ async function handleCompactionInner( // Initialize the project AFTER path correction so we never create a row for // the gateway's cwd / an unattributed bucket from a path-less probe request. - await initIfNeeded(projectPath, config, pathResult.gitRemote); + await initIfNeeded( + projectPath, + config, + pathResult.gitRemote, + req.signal, + requestGeneration, + ); + assertCurrentPipelineGeneration(req.signal, requestGeneration); setSentryLightContext({ model: req.model, projectPath }); log.info(`compaction intercepted for session ${sessionID.slice(0, 16)}`); @@ -10769,7 +12350,9 @@ async function handleCompactionInner( previousSummary: extractPreviousSummary(req), sessionUpstream: sessionState.lastUpstream, signal: req.signal, + trackOperation, }); + trackOperation(summaryPromise); if (req.stream) { // Open the SSE stream immediately and emit keep-alive `ping`s while the @@ -10821,9 +12404,25 @@ async function handleCompactionInner( const summary = await summaryPromise; if (summary == null) { log.warn( - `compaction summary empty for session ${sessionID.slice(0, 16)} — falling back to upstream`, + `compaction summary empty for session ${sessionID.slice(0, 16)} — using authenticated upstream`, + ); + const trustedUpstream = extractUpstreamUrlHeader({ + "x-lore-upstream-url": sessionState.lastUpstream?.url ?? "", + }); + if (!trustedUpstream) { + return errorResponse(502, "No trusted upstream destination"); + } + const fallbackHeaders = { ...req.rawHeaders }; + fallbackHeaders["x-lore-upstream-url"] = trustedUpstream; + if (sessionState.lastUpstream?.providerID) { + fallbackHeaders["x-lore-provider"] = sessionState.lastUpstream.providerID; + } else { + delete fallbackHeaders["x-lore-provider"]; + } + return await handlePassthrough( + { ...req, rawHeaders: fallbackHeaders }, + config, ); - return await handlePassthrough(req, config); } const resp = buildCompactionResponse(sessionID, summary, req.model); return nonStreamHttpResponse( @@ -10838,24 +12437,140 @@ async function handleCompactionInner( async function handleCompaction( req: GatewayRequest, config: GatewayConfig, + requestGeneration: number, + trackOperation: (operation: Promise) => void, + claimSession: (sessionID: string) => Promise, ): Promise { const abortScope = createForegroundAbortScope(req.signal); try { - const response = await handleCompactionInner( - { ...req, signal: abortScope.signal }, - config, + const run = (signal: AbortSignal) => { + if ( + pipelineResetInProgress || + requestGeneration !== streamingPostResponseGeneration + ) { + return Promise.resolve( + errorResponse(503, "Gateway pipeline generation changed"), + ); + } + return handleCompactionInner( + { ...req, signal }, + config, + requestGeneration, + trackOperation, + claimSession, + ); + }; + const response = + req.stream && req.protocol === "openai-responses" + ? earlyFlushStreamingResponse( + run, + req.model, + abortScope.signal, + trackOperation, + ) + : await run(abortScope.signal); + return wrapBodyWithCleanup( + response, + abortScope.dispose, + abortScope.signal, + (reason) => + abortScope.abort( + reason ?? new DOMException("response cancelled", "AbortError"), + ), ); - return wrapBodyWithCleanup(response, abortScope.dispose, abortScope.signal); } catch (error) { abortScope.dispose(); throw error; } } -// --------------------------------------------------------------------------- -// Case 1b: Explicit compaction endpoint (POST /v1/compact) -// --------------------------------------------------------------------------- - +// --------------------------------------------------------------------------- +// Case 1b: Explicit compaction endpoint (POST /v1/compact) +// --------------------------------------------------------------------------- + +function directCompactionFailureResponse( + route: string, + error: unknown, +): Response { + log.error(`${route} error:`, error); + const unavailable = + error instanceof StreamingPostResponseWaitCapacityError || + error instanceof PipelineCapacityError; + const aborted = + error instanceof DOMException && + (error.name === "AbortError" || error.name === "TimeoutError"); + return new Response( + JSON.stringify({ + error: "compaction_failed", + message: unavailable + ? "Compaction temporarily unavailable" + : "Compaction failed", + }), + { + status: unavailable ? 503 : aborted ? 502 : 500, + headers: { "content-type": "application/json" }, + }, + ); +} + +function preflightDirectCompactionSession( + req: Request, + config: GatewayConfig, +): Response | null { + const rawHeaders: Record = {}; + req.headers.forEach((value, key) => { + rawHeaders[key] = value; + }); + if (hasConflictingAuthHeaders(rawHeaders)) { + return errorResponse( + 400, + "Conflicting authentication headers: send either x-api-key or Authorization, not both", + ); + } + if (!extractAuth(rawHeaders)) { + return new Response( + JSON.stringify({ + error: "unauthorized", + message: "A provider credential is required", + }), + { status: 401, headers: { "content-type": "application/json" } }, + ); + } + const minimalReq: GatewayRequest = { + protocol: "anthropic", + system: "", + messages: [], + tools: [], + model: "", + maxTokens: 0, + stream: false, + metadata: {}, + rawHeaders, + }; + const sessionID = findIndexedKnownSessionID(minimalReq, config); + if (!sessionID || (loadSessionTracking(sessionID)?.messageCount ?? 0) === 0) { + return new Response( + JSON.stringify({ + error: "session_not_found", + message: "No authenticated session found for the given headers", + }), + { status: 404, headers: { "content-type": "application/json" } }, + ); + } + return null; +} + +function directRequestCredentialFingerprint( + req: Request, + config: GatewayConfig, +): string { + const rawHeaders: Record = {}; + req.headers.forEach((value, key) => { + rawHeaders[key] = value; + }); + return requestCredentialFingerprint(rawHeaders, config) ?? ""; +} + /** * Cancel-when-fits decision for the explicit `/v1/compact` endpoint. * @@ -10955,6 +12670,9 @@ async function handleCompactEndpointInner( req: Request, config: GatewayConfig, signal: AbortSignal, + requestGeneration: number, + trackOperation: (operation: Promise) => void, + claimSession: (sessionID: string) => Promise, rawHeaders: Record, ): Promise { if (hasConflictingAuthHeaders(rawHeaders)) { @@ -10989,6 +12707,7 @@ async function handleCompactEndpointInner( // Decode any Content-Encoding (e.g. zstd) before JSON-parsing. body = JSON.parse(await decodeRequestBody(req, signal)) as typeof body; } catch { + signal.throwIfAborted(); return new Response( JSON.stringify({ error: "invalid_request", @@ -11025,18 +12744,15 @@ async function handleCompactEndpointInner( stream: false, metadata: {}, rawHeaders, + signal, }; - const { sessionID, isNew } = await identifySession( + const state = resolveAuthenticatedDirectSession( minimalReq, projectPath, config, ); - - if (isNew) { - // No prior session found — the caller's session header didn't match any - // existing session. This typically means no conversation turns have gone - // through the gateway yet, so there's nothing to compact. + if (!state) { return new Response( JSON.stringify({ error: "session_not_found", @@ -11048,26 +12764,36 @@ async function handleCompactEndpointInner( ); } - let state: SessionState; - try { - state = getOrCreateSession( - sessionID, - projectPath, - "header", - requestCredentialFingerprint(rawHeaders, config) ?? "", - config, + if (!isConfidentlyBoundToProject(state, projectPath)) { + return new Response( + JSON.stringify({ + error: "project_mismatch", + message: "project_path does not match the authenticated session", + }), + { status: 403, headers: { "content-type": "application/json" } }, + ); + } + const sessionID = state.sessionID; + await claimSession(sessionID); + if (!confirmedIndexedIdentityResolvesTo(minimalReq, sessionID, config)) { + return new Response( + JSON.stringify({ + error: "session_not_found", + message: "No authenticated session found for the given headers", + }), + { status: 404, headers: { "content-type": "application/json" } }, + ); + } + await awaitStreamingPostResponse(sessionID, signal); + assertCurrentPipelineGeneration(signal, requestGeneration); + if (!confirmedIndexedIdentityResolvesTo(minimalReq, sessionID, config)) { + return new Response( + JSON.stringify({ + error: "session_not_found", + message: "No authenticated session found for the given headers", + }), + { status: 404, headers: { "content-type": "application/json" } }, ); - } catch (error) { - if (error instanceof SessionTenantMismatchError) { - return new Response( - JSON.stringify({ - error: "session_not_found", - message: "No session found for the authenticated storage tenant", - }), - { status: 404, headers: { "content-type": "application/json" } }, - ); - } - throw error; } if ( state.projectPathProvisional === true || @@ -11083,7 +12809,14 @@ async function handleCompactEndpointInner( } setSessionAuth(sessionID, credential, state.lastUpstream?.providerID); - await initIfNeeded(state.projectPath, config, gitRemote); + await initIfNeeded( + state.projectPath, + config, + gitRemote, + signal, + requestGeneration, + ); + assertCurrentPipelineGeneration(signal, requestGeneration); // Cancel-when-fits policy. The gateway is the authoritative source for // "does this session's raw context fit in the layer-0 budget?" — the plugin @@ -11135,7 +12868,9 @@ async function handleCompactEndpointInner( : undefined, sessionUpstream: state?.lastUpstream, signal, + trackOperation, }); + assertCurrentPipelineGeneration(signal, requestGeneration); if (summary == null) { log.warn( @@ -11162,10 +12897,12 @@ async function handleCompactEndpointInner( headers: { "content-type": "application/json" }, }); } catch (err) { - const msg = err instanceof Error ? err.message : "Compaction failed"; log.error("compact endpoint error:", err); return new Response( - JSON.stringify({ error: "compaction_failed", message: msg }), + JSON.stringify({ + error: "compaction_failed", + message: "Compaction failed", + }), { status: 500, headers: { "content-type": "application/json" } }, ); } @@ -11177,13 +12914,30 @@ export async function handleCompactEndpoint( ): Promise { const rawHeaders = requestHeaders(req.headers); return withRequestStorageTenant(rawHeaders, config, async () => { + if (pipelineResetInProgress) { + return errorResponse(503, "Gateway pipeline is resetting"); + } + const preflight = preflightDirectCompactionSession(req, config); + if (preflight) return preflight; + streamingPostResponsesAccepting = true; + const requestGeneration = streamingPostResponseGeneration; const abortScope = createForegroundAbortScope(req.signal); try { - const response = await handleCompactEndpointInner( - req, - config, + const response = await runActivePipelineRequest( abortScope.signal, - rawHeaders, + (signal, trackOperation, claimSession) => + handleCompactEndpointInner( + req, + config, + signal, + requestGeneration, + trackOperation, + claimSession, + rawHeaders, + ), + undefined, + undefined, + directRequestCredentialFingerprint(req, config), ); return wrapBodyWithCleanup( response, @@ -11192,7 +12946,7 @@ export async function handleCompactEndpoint( ); } catch (error) { abortScope.dispose(); - throw error; + return directCompactionFailureResponse("compact endpoint", error); } }); } @@ -11218,6 +12972,9 @@ async function handleResponsesCompactEndpointInner( req: Request, config: GatewayConfig, signal: AbortSignal, + requestGeneration: number, + trackOperation: (operation: Promise) => void, + claimSession: (sessionID: string) => Promise, rawHeaders: Record, ): Promise { if (hasConflictingAuthHeaders(rawHeaders)) { @@ -11231,7 +12988,7 @@ async function handleResponsesCompactEndpointInner( ); } const credential = extractAuth(rawHeaders); - if (usesRemoteSessionBinding(config) && !credential) { + if (!credential) { return new Response( JSON.stringify({ error: "unauthorized", @@ -11248,6 +13005,7 @@ async function handleResponsesCompactEndpointInner( try { bodyText = await decodeRequestBody(req, signal); } catch { + signal.throwIfAborted(); return new Response( JSON.stringify({ error: "invalid_request", @@ -11275,130 +13033,154 @@ async function handleResponsesCompactEndpointInner( let gatewayReq: GatewayRequest; try { gatewayReq = parseOpenAIResponsesRequest(body, rawHeaders); + gatewayReq.signal = signal; } catch { - // If parsing fails, still attempt passthrough — the upstream may accept it. - log.warn( - "responses/compact: failed to parse request body — falling back to upstream", - ); - return await passthroughResponsesCompact( - bodyText, - rawHeaders, - config, - signal, + return new Response( + JSON.stringify({ + error: "invalid_request", + message: "Invalid Responses compaction body", + }), + { status: 400, headers: { "content-type": "application/json" } }, ); } const pathResult = getProjectPath(gatewayReq.system, rawHeaders); const gitRemote = extractGitRemoteHeader(rawHeaders); - - const { sessionID, isNew } = await identifySession( + const state = resolveAuthenticatedDirectSession( gatewayReq, pathResult.path, config, ); - - // If no prior session, skip Lore compaction and passthrough to upstream. - if (!isNew) { - let state: SessionState; - try { - state = getOrCreateSession( - sessionID, - pathResult.path, - pathResult.source, - requestCredentialFingerprint(rawHeaders, config) ?? "", + if (!state) { + if (!extractKnownSessionHeader(rawHeaders)) { + return await passthroughResponsesCompact( + bodyText, + rawHeaders, config, - ); - } catch (error) { - if (error instanceof SessionTenantMismatchError) { - return new Response( - JSON.stringify({ - error: "session_not_found", - message: "No session found for the authenticated storage tenant", - }), - { status: 404, headers: { "content-type": "application/json" } }, - ); - } - throw error; - } - if ( - usesRemoteSessionBinding(config) && - (state.projectPathProvisional === true || - state.projectPath !== pathResult.path) - ) { - return new Response( - JSON.stringify({ - error: "project_mismatch", - message: "project path does not match the authenticated session", - }), - { status: 403, headers: { "content-type": "application/json" } }, + signal, + undefined, + gatewayReq, ); } - const compactProjectPath = usesRemoteSessionBinding(config) - ? state.projectPath - : pathResult.path; - await initIfNeeded(compactProjectPath, config, gitRemote); - - log.info( - `responses/compact: generating Lore summary for session ${sessionID.slice(0, 16)}`, + return new Response( + JSON.stringify({ + error: "session_not_found", + message: "No authenticated session found for the given headers", + }), + { status: 404, headers: { "content-type": "application/json" } }, + ); + } + if (!isConfidentlyBoundToProject(state, pathResult.path)) { + return new Response( + JSON.stringify({ + error: "project_mismatch", + message: "project path does not match the authenticated session", + }), + { status: 403, headers: { "content-type": "application/json" } }, + ); + } + const sessionID = state.sessionID; + await claimSession(sessionID); + if (!confirmedIndexedIdentityResolvesTo(gatewayReq, sessionID, config)) { + return new Response( + JSON.stringify({ + error: "session_not_found", + message: "No authenticated session found for the given headers", + }), + { status: 404, headers: { "content-type": "application/json" } }, + ); + } + await awaitStreamingPostResponse(sessionID, signal); + assertCurrentPipelineGeneration(signal, requestGeneration); + if (!confirmedIndexedIdentityResolvesTo(gatewayReq, sessionID, config)) { + return new Response( + JSON.stringify({ + error: "session_not_found", + message: "No authenticated session found for the given headers", + }), + { status: 404, headers: { "content-type": "application/json" } }, + ); + } + if ( + state.projectPathProvisional === true || + state.projectPath !== pathResult.path + ) { + return new Response( + JSON.stringify({ + error: "project_mismatch", + message: "project path does not match the authenticated session", + }), + { status: 403, headers: { "content-type": "application/json" } }, ); + } + setSessionAuth(sessionID, credential, state.lastUpstream?.providerID); - try { - const summary = await generateCompactionSummary({ - projectPath: compactProjectPath, - sessionID, - config, - signal, - }); + await initIfNeeded( + state.projectPath, + config, + gitRemote, + signal, + requestGeneration, + ); + assertCurrentPipelineGeneration(signal, requestGeneration); - if (summary != null) { - // Clear cached warmup body — post-compaction messages will differ. - const sessionState = sessions.get(sessionID); - if (sessionState) { - sessionState.cacheAnalytics.lastRequestBody = null; - } + log.info( + `responses/compact: generating Lore summary for session ${sessionID.slice(0, 16)}`, + ); - // Return in Codex's expected format: { output: ResponseItem[] } - // Must include id, status, and annotations to match the - // CompactHistoryResponse { output: Vec } struct. - return new Response( - JSON.stringify({ - output: [ - { - type: "message", - id: `msg_lore_compact_${crypto.randomUUID().replace(/-/g, "").slice(0, 12)}`, - role: "assistant", - status: "completed", - content: [ - { type: "output_text", text: summary, annotations: [] }, - ], - }, - ], - }), - { status: 200, headers: { "content-type": "application/json" } }, - ); - } + try { + const summary = await generateCompactionSummary({ + projectPath: state.projectPath, + sessionID, + config, + sessionUpstream: state.lastUpstream, + signal, + trackOperation, + }); + assertCurrentPipelineGeneration(signal, requestGeneration); - log.warn( - `responses/compact: Lore summary generation failed for session ${sessionID.slice(0, 16)} — falling back to upstream`, - ); - } catch (err) { - log.warn( - "responses/compact: Lore compaction error, falling back to upstream:", - err, + if (summary != null) { + state.cacheAnalytics.lastRequestBody = null; + + // Return in Codex's expected format: { output: ResponseItem[] } + // Must include id, status, and annotations to match the + // CompactHistoryResponse { output: Vec } struct. + return new Response( + JSON.stringify({ + output: [ + { + type: "message", + id: `msg_lore_compact_${crypto.randomUUID().replace(/-/g, "").slice(0, 12)}`, + role: "assistant", + status: "completed", + content: [ + { type: "output_text", text: summary, annotations: [] }, + ], + }, + ], + }), + { status: 200, headers: { "content-type": "application/json" } }, ); } - } else { - log.info( - "responses/compact: no prior session found — falling back to upstream", + + log.warn( + `responses/compact: Lore summary generation failed for session ${sessionID.slice(0, 16)} — falling back to upstream`, + ); + } catch (err) { + signal.throwIfAborted(); + log.warn( + "responses/compact: Lore compaction error, falling back to upstream:", + err, ); } - // Fallback: passthrough to upstream OpenAI /v1/responses/compact + // Fallback only to the destination previously authenticated by a normal turn. return await passthroughResponsesCompact( bodyText, rawHeaders, config, signal, + state.lastUpstream?.url || null, gatewayReq, ); } @@ -11409,13 +13191,28 @@ export async function handleResponsesCompactEndpoint( ): Promise { const rawHeaders = requestHeaders(req.headers); return withRequestStorageTenant(rawHeaders, config, async () => { + if (pipelineResetInProgress) { + return errorResponse(503, "Gateway pipeline is resetting"); + } + streamingPostResponsesAccepting = true; + const requestGeneration = streamingPostResponseGeneration; const abortScope = createForegroundAbortScope(req.signal); try { - const response = await handleResponsesCompactEndpointInner( - req, - config, + const response = await runActivePipelineRequest( abortScope.signal, - rawHeaders, + (signal, trackOperation, claimSession) => + handleResponsesCompactEndpointInner( + req, + config, + signal, + requestGeneration, + trackOperation, + claimSession, + rawHeaders, + ), + undefined, + undefined, + directRequestCredentialFingerprint(req, config), ); return wrapBodyWithCleanup( response, @@ -11424,7 +13221,10 @@ export async function handleResponsesCompactEndpoint( ); } catch (error) { abortScope.dispose(); - throw error; + return directCompactionFailureResponse( + "responses/compact endpoint", + error, + ); } }); } @@ -11437,6 +13237,7 @@ export async function passthroughResponsesCompact( rawHeaders: Record, config: GatewayConfig, callerSignal?: AbortSignal, + trustedUpstreamBase?: string | null, parsedRequest?: GatewayRequest, ): Promise { const abortScope = createForegroundAbortScope(callerSignal); @@ -11452,7 +13253,28 @@ export async function passthroughResponsesCompact( // Responses request. If parsing failed, an explicit validated URL override is // the only safe custom route; an explicit provider without a compatible URL // fails closed because model routing is unavailable. - const headerUpstream = extractUpstreamUrlHeader(rawHeaders); + const trustedUpstream = + trustedUpstreamBase === undefined + ? undefined + : trustedUpstreamBase + ? extractUpstreamUrlHeader({ + "x-lore-upstream-url": trustedUpstreamBase, + }) + : undefined; + if (trustedUpstreamBase !== undefined && !trustedUpstream) { + abortScope.dispose(); + return new Response( + JSON.stringify({ + error: "compaction_failed", + message: "No trusted upstream destination", + }), + { status: 502, headers: { "content-type": "application/json" } }, + ); + } + const headerUpstream = + trustedUpstreamBase === undefined + ? extractUpstreamUrlHeader(rawHeaders) + : undefined; if (headerUpstream && !extractAuth(rawHeaders)) { abortScope.dispose(); return new Response( @@ -11464,7 +13286,7 @@ export async function passthroughResponsesCompact( ); } let route: ResolvedRequestUpstreamRoute | undefined; - if (parsedRequest) { + if (parsedRequest && trustedUpstreamBase === undefined) { try { route = resolveRequestUpstreamRoute(parsedRequest, config); } catch (error) { @@ -11482,7 +13304,11 @@ export async function passthroughResponsesCompact( const fallbackProviderID = route ? route.providerID : extractProviderHeader(rawHeaders); - if (rawHeaders["x-lore-provider"] && !fallbackProviderID) { + if ( + trustedUpstreamBase === undefined && + rawHeaders["x-lore-provider"] && + !fallbackProviderID + ) { abortScope.dispose(); return new Response( JSON.stringify({ @@ -11492,7 +13318,11 @@ export async function passthroughResponsesCompact( { status: 502, headers: { "content-type": "application/json" } }, ); } - if (rawHeaders["x-lore-upstream-url"] && !headerUpstream) { + if ( + trustedUpstreamBase === undefined && + rawHeaders["x-lore-upstream-url"] && + !headerUpstream + ) { abortScope.dispose(); return new Response( JSON.stringify({ @@ -11552,14 +13382,16 @@ export async function passthroughResponsesCompact( ); } const effectiveUpstreamBase = + trustedUpstream ?? route?.effectiveUpstreamBase ?? headerUpstream ?? fallbackProviderRoute?.url ?? config.upstreamOpenAI; - const effectiveProtocol = - route?.effectiveProtocol ?? - fallbackProviderRoute?.protocol ?? - "openai-responses"; + const effectiveProtocol = trustedUpstream + ? "openai-responses" + : (route?.effectiveProtocol ?? + fallbackProviderRoute?.protocol ?? + "openai-responses"); if (effectiveProtocol !== "openai-responses") { abortScope.dispose(); return new Response( @@ -11631,9 +13463,9 @@ export async function passthroughResponsesCompact( abortScope.signal, ); return wrapBodyWithCleanup(upstream, abortScope.dispose, abortScope.signal); - } catch { + } catch (err) { abortScope.dispose(); - log.error("responses/compact upstream passthrough failed"); + log.error("responses/compact upstream passthrough error:", err); return new Response( JSON.stringify({ error: "compaction_failed", @@ -11685,22 +13517,25 @@ export async function completeBudgetThrottleDelay( export function createForegroundAbortScope(caller?: AbortSignal): { signal: AbortSignal; + abort: (reason?: unknown) => void; dispose: () => void; } { const controller = new AbortController(); activeForegroundAbortControllers.add(controller); - const onCallerAbort = () => controller.abort(caller?.reason); + const abort = (reason?: unknown) => { + if (!controller.signal.aborted) controller.abort(reason); + }; + const onCallerAbort = () => abort(caller?.reason); caller?.addEventListener("abort", onCallerAbort, { once: true }); if (caller?.aborted) onCallerAbort(); const timer = setTimeout( () => - controller.abort( - new DOMException("foreground request timed out", "TimeoutError"), - ), + abort(new DOMException("foreground request timed out", "TimeoutError")), FOREGROUND_REQUEST_TIMEOUT_MS, ); return { signal: controller.signal, + abort, dispose: () => { activeForegroundAbortControllers.delete(controller); clearTimeout(timer); @@ -11713,6 +13548,7 @@ export function wrapBodyWithCleanup( response: Response, cleanup: () => void, signal?: AbortSignal, + onCancel?: (reason?: unknown) => void, ): Response { if (!response.body) { cleanup(); @@ -11745,7 +13581,9 @@ export function wrapBodyWithCleanup( }, async pull(controller) { try { - const { done, value } = await readStreamChunk(reader, { signal }); + const { done, value } = signal + ? await readStreamChunk(reader, { signal }) + : await reader.read(); if (done) { finish(); try { @@ -11764,6 +13602,7 @@ export function wrapBodyWithCleanup( } }, cancel(reason) { + onCancel?.(reason); finish(); cancelAndReleaseReader(reader, reason); }, @@ -11782,6 +13621,150 @@ export function wrapBodyWithCleanup( }); } +async function claimPipelineSession( + active: ActivePipelineRequest, + sessionID: string, + signal: AbortSignal, +): Promise { + if (active.sessionIDs.has(sessionID)) return; + signal.throwIfAborted(); + if (pipelineSessionHasCapacity(sessionID)) { + active.sessionIDs.add(sessionID); + return; + } + if ( + pendingSessionClaims.has(sessionID) || + pendingSessionClaims.size >= MAX_PENDING_SESSION_CLAIMS || + pendingSessionClaimsForAdmissionKey(active.admissionKey) >= + MAX_ACTIVE_PIPELINE_REQUESTS_PER_ADMISSION_KEY + ) { + throw new PipelineCapacityError("session request queue full"); + } + + activePipelineRequests.delete(active); + return new Promise((resolve, reject) => { + let claim: PendingSessionClaim; + const onAbort = () => { + if (pendingSessionClaims.get(sessionID) !== claim) return; + pendingSessionClaims.delete(sessionID); + signal.removeEventListener("abort", onAbort); + reject(signal.reason); + pumpPendingSessionClaims(); + }; + claim = { active, sessionID, signal, resolve, reject, onAbort }; + pendingSessionClaims.set(sessionID, claim); + signal.addEventListener("abort", onAbort, { once: true }); + if (signal.aborted) onAbort(); + else pumpPendingSessionClaims(); + }); +} + +async function runActivePipelineRequest( + callerSignal: AbortSignal | undefined, + operation: ( + signal: AbortSignal, + trackOperation: (operation: Promise) => void, + claimSession: (sessionID: string) => Promise, + ) => Promise, + onResponseBodySettled?: () => void, + onResponseBodyCancelled?: () => void, + admissionKey = "", +): Promise { + if ( + detachedPipelineRequests.size + + activePipelineRequests.size + + pendingSessionClaims.size >= + maxDetachedPipelineRequests || + activePipelineRequests.size + streamingPostResponsePending >= + maxActivePipelineRequests || + activePipelineRequestsForAdmissionKey(admissionKey) + + (streamingPostResponsePendingByAdmissionKey.get(admissionKey) ?? 0) >= + MAX_ACTIVE_PIPELINE_REQUESTS_PER_ADMISSION_KEY + ) { + return errorResponse(503, "Gateway is busy"); + } + const lifecycle = new AbortController(); + const signal = callerSignal + ? AbortSignal.any([callerSignal, lifecycle.signal]) + : lifecycle.signal; + let settle: (() => void) | undefined; + const settled = new Promise((resolve) => { + settle = resolve; + }); + const pendingOperations = new Set>(); + let finished = false; + let bodySettled = false; + let responseReturned = false; + let responseCancelled = false; + const markResponseCancelled = (): void => { + if (responseCancelled) return; + responseCancelled = true; + onResponseBodyCancelled?.(); + }; + function onAbort(): void { + if (responseReturned) { + if (callerSignal?.aborted) markResponseCancelled(); + settleResponse(); + } + } + const trackOperation = (operation: Promise): void => { + const tracked = operation.then( + () => {}, + () => {}, + ); + pendingOperations.add(tracked); + void tracked.finally(() => pendingOperations.delete(tracked)); + }; + async function finish(): Promise { + if (finished) return; + finished = true; + signal.removeEventListener("abort", onAbort); + while (pendingOperations.size > 0) { + await Promise.all(pendingOperations); + } + activePipelineRequests.delete(active); + detachedPipelineRequests.delete(active); + pumpPendingSessionClaims(); + settle?.(); + } + function settleResponse(): void { + if (bodySettled) return; + bodySettled = true; + onResponseBodySettled?.(); + void finish(); + } + const active: ActivePipelineRequest = { + admissionKey, + abort: (reason) => { + lifecycle.abort(reason); + if (responseReturned) settleResponse(); + }, + settled, + sessionIDs: new Set(), + }; + activePipelineRequests.add(active); + signal.addEventListener("abort", onAbort, { once: true }); + + try { + const response = await operation(signal, trackOperation, (sessionID) => + claimPipelineSession(active, sessionID, signal), + ); + responseReturned = true; + if (signal.aborted) { + if (callerSignal?.aborted) markResponseCancelled(); + settleResponse(); + } + return wrapBodyWithCleanup(response, settleResponse, undefined, () => { + markResponseCancelled(); + settleResponse(); + }); + } catch (error) { + onResponseBodySettled?.(); + void finish(); + throw error; + } +} + export function validatedMetaStream( response: Response, protocol: "anthropic" | "openai" | "openai-responses" | "gemini", @@ -11994,6 +13977,11 @@ async function handlePassthrough( undefined, abortScope.signal, ); + if (wireProtocol === "openai-responses") { + parseResponsesNonStreamEnvelope( + JSON.parse(body) as Record, + ); + } return new Response(body, { status: upstreamResponse.status, headers: { "content-type": "application/json" }, @@ -12034,31 +14022,33 @@ async function handlePassthrough( } } // Other cross-protocol streaming combos: accumulate + re-emit - const resp = + const resp = await preserveIncompleteResponsesTerminal( wireProtocol === "openai" - ? await accumulateOpenAISSEStream(upstreamResponse, { + ? accumulateOpenAISSEStream(upstreamResponse, { signal: abortScope.signal, strict: true, stopAtTerminal: true, consumeUntilDone: true, }) : wireProtocol === "openai-responses" - ? await accumulateResponsesSSEStream(upstreamResponse, { + ? accumulateResponsesSSEStream(upstreamResponse, { signal: abortScope.signal, validation: req.codex === true ? "codex" : "public", stopAtTerminal: true, + requireCompletedTerminal: true, }) : wireProtocol === "gemini" - ? await accumulateGeminiSSEStream(upstreamResponse, { + ? accumulateGeminiSSEStream(upstreamResponse, { signal: abortScope.signal, strict: true, stopAtTerminal: true, }) - : await accumulateSSEResponse(upstreamResponse, { + : accumulateSSEResponse(upstreamResponse, { signal: abortScope.signal, strict: true, stopAtTerminal: true, - }); + }), + ); return nonStreamHttpResponse( resp, req.protocol, @@ -12069,11 +14059,13 @@ async function handlePassthrough( } // Non-streaming cross-protocol: accumulate + re-emit - const resp = await accumulateNonStreamResponse( - upstreamResponse, - wireProtocol, - req.codex === true, - abortScope.signal, + const resp = await preserveIncompleteResponsesTerminal( + accumulateNonStreamResponse( + upstreamResponse, + wireProtocol, + req.codex === true, + abortScope.signal, + ), ); return nonStreamHttpResponse( resp, @@ -12084,6 +14076,393 @@ async function handlePassthrough( ); } +/** + * Validate a provisional session identity without touching session-owned state. + * The full Lore turn runs only on a later retry after this successful response + * confirms the presented header. Failed and incomplete attempts leave the + * adopted session, project rows, auth registries, and gradient state untouched. + */ +async function handleProvisionalConversationTurn( + req: GatewayRequest, + config: GatewayConfig, + identified: IdentifiedSession, + pathResult: ProjectPathResult, + requestOrder: number, + requestGeneration: number, + downstreamSettled: Promise, + downstreamWasCancelled: () => boolean, +): Promise { + // Resolve and validate route intent once, but keep it private until the + // provisional identity is confirmed by a complete response and client EOF. + const requestUpstream = prepareRequestUpstream(req, config); + const abortScope = createForegroundAbortScope(req.signal); + let forwarded: UpstreamResult; + try { + forwarded = await forwardToUpstream( + req, + config, + undefined, + undefined, + abortScope.signal, + requestUpstream.route, + ); + } catch (error) { + abortScope.dispose(); + throw error; + } + const upstreamResponse = wrapBodyWithCleanup( + forwarded.response, + abortScope.dispose, + abortScope.signal, + ); + if (!upstreamResponse.ok) { + return preserveUpstreamErrorResponse(upstreamResponse, abortScope.signal); + } + + let accumulated: GatewayResponse; + try { + accumulated = req.stream + ? forwarded.effectiveProtocol === "openai-responses" + ? await accumulateResponsesSSEStream(upstreamResponse, { + signal: abortScope.signal, + validation: req.codex ? "codex" : "public", + stopAtTerminal: true, + requireCompletedTerminal: true, + }) + : forwarded.effectiveProtocol === "openai" + ? await accumulateOpenAISSEStream(upstreamResponse, { + signal: abortScope.signal, + strict: true, + stopAtTerminal: true, + consumeUntilDone: true, + }) + : forwarded.effectiveProtocol === "gemini" + ? await accumulateGeminiSSEStream(upstreamResponse, { + signal: abortScope.signal, + strict: true, + stopAtTerminal: true, + }) + : await accumulateSSEResponse(upstreamResponse, { + signal: abortScope.signal, + strict: true, + stopAtTerminal: true, + }) + : await accumulateNonStreamResponse( + upstreamResponse, + forwarded.effectiveProtocol, + req.codex === true, + abortScope.signal, + true, + ); + } catch (error) { + abortScope.dispose(); + if (!(error instanceof ResponsesTerminalError)) throw error; + scheduleStreamingPostResponse( + identified.sessionID, + requestGeneration, + async () => { + await downstreamSettled; + await new Promise((resolve) => setImmediate(resolve)); + const pause = provisionalFinalizerPauseForTest; + if (pause) { + pause.onWait(); + await pause.pause; + } + if (requestGeneration !== streamingPostResponseGeneration) return; + if ( + identified.guardProject && + conflictsWithConfidentSessionProject(identified.sessionID, pathResult) + ) { + dropOwnedProvisionalKey( + identified.provisionalKey, + identified.sessionID, + ); + return; + } + accountUnsuccessfulResponse( + error.response, + identified.sessionID, + conversationTTLForAccounting(identified.sessionID), + undefined, + () => {}, + () => { + const state = sessions.get(identified.sessionID); + if (state) state._dirty = true; + }, + ); + }, + () => {}, + true, + requestCredentialFingerprint(req.rawHeaders, config) ?? undefined, + ); + if (error.status === "incomplete" && !hasRecallToolUse(error.response)) { + return nonStreamHttpResponse( + error.response, + req.protocol, + req.stream, + undefined, + requestEnablesLongContext(req), + ); + } + return errorResponse(502, "Gateway request failed"); + } + if ( + forwarded.effectiveProtocol === "gemini" && + !["end_turn", "max_tokens", "tool_use"].includes(accumulated.stopReason) + ) { + throw new Error("upstream Gemini request did not complete"); + } + abortScope.dispose(); + const response = nonStreamHttpResponse( + accumulated, + req.protocol, + req.stream, + undefined, + requestEnablesLongContext(req), + ); + + const commit = async (): Promise => { + if ( + requestGeneration !== streamingPostResponseGeneration || + req.signal?.aborted || + downstreamWasCancelled() + ) { + return false; + } + if ( + identified.provisionalKey && + !provisionalKeyOwned(identified.provisionalKey, identified.sessionID) + ) { + return false; + } + const credential = extractAuth(req.rawHeaders); + const persisted = loadSessionTracking(identified.sessionID); + const liveState = sessions.get(identified.sessionID); + let restoredUpstream: + | ReturnType + | undefined; + if (!liveState && persisted?.lastUpstream) { + try { + restoredUpstream = deserializeUpstreamState( + persisted.lastUpstream, + config, + ); + } catch { + log.warn( + `corrupt last upstream for session ${identified.sessionID.slice(0, 16)}, ignoring`, + ); + } + } + const upstreamState: MutableUpstreamState = { + lastUpstream: liveState?.lastUpstream ?? restoredUpstream?.lastUpstream, + upstreamByProvider: new Map( + liveState?.upstreamByProvider ?? restoredUpstream?.upstreamByProvider, + ), + _upstreamRequestOrder: liveState?._upstreamRequestOrder, + _upstreamRequestOrderByProvider: + liveState?._upstreamRequestOrderByProvider + ? new Map(liveState._upstreamRequestOrderByProvider) + : undefined, + }; + const upstreamUpdate = applyRequestUpstream( + upstreamState, + requestUpstream.snapshot, + requestOrder, + config, + ); + // Reconstruct the binding exactly as the full pipeline does, but keep it + // private until this successful provisional turn is durably committed. + // This is what self-heals rows written to a cwd/unattributed bucket before + // publishing the newly adopted header and confident path. + postResponseStartObserver?.(); + const noStore = + persisted?.amnesia === true || + req.rawHeaders["x-lore-no-store"] === "true"; + const loreMessages = gatewayMessagesToLore( + req.messages, + identified.sessionID, + ); + const credentialFingerprint = + requestCredentialFingerprint(req.rawHeaders, config) ?? ""; + const known = knownSessionHeaderForRequest( + req, + identified.sessionID, + config, + ); + let projectPath = pathResult.path; + let projectPathProvisional = pathResult.source === "cwd"; + withSavepoint("commit_provisional_turn", () => { + if ( + identified.expectedUnowned && + !legacyAdoptionTargetIsUnowned(identified.sessionID) + ) { + dropOwnedProvisionalKey( + identified.provisionalKey, + identified.sessionID, + ); + throw new Error("legacy session owner changed during adoption"); + } + if ( + identified.guardProject && + conflictsWithConfidentSessionProject(identified.sessionID, pathResult) + ) { + dropOwnedProvisionalKey( + identified.provisionalKey, + identified.sessionID, + ); + throw new Error("session project changed during provisional migration"); + } + // Project creation/reattribution belongs to the same transaction as the + // turn, tracking, route, and header confirmation. A local write failure + // must leave the provisional project and identity wholly unchanged. + const pathState = { + sessionID: identified.sessionID, + projectPath: persisted?.projectPath ?? pathResult.path, + projectPathProvisional: persisted?.projectPath + ? persisted.projectPathProvisional + : pathResult.source === "cwd", + gitRemote: pathResult.gitRemote, + } as Partial as SessionState; + projectPath = resolveSessionProjectPath(pathResult, pathState, config); + projectPathProvisional = pathState.projectPathProvisional === true; + if ( + projectPathProvisional && + (pathResult.source === "header" || pathResult.source === "inferred") + ) { + throw new Error("provisional project re-attribution failed"); + } + ensureProject(projectPath, undefined, pathResult.gitRemote); + storeTurnTemporal({ + loreMessages, + assistantContentBlocks: accumulated.content, + usage: accumulated.usage ?? ZERO_USAGE, + model: accumulated.model, + projectPath, + sessionID: identified.sessionID, + noStore, + }); + saveSessionTracking(identified.sessionID, { + messageCount: req.messages.length, + turnsSinceCuration: persisted?.turnsSinceCuration ?? 0, + consecutiveTextOnlyTurns: persisted?.consecutiveTextOnlyTurns ?? 0, + projectPath, + projectPathProvisional, + credentialFingerprint, + ...(identified.adoptionFingerprint + ? { fingerprint: identified.adoptionFingerprint } + : {}), + ...(upstreamUpdate.changed + ? { lastUpstream: serializeUpstreamState(upstreamState) } + : {}), + ...(known + ? { + headerSessionId: known.sessionId, + headerName: known.headerName, + } + : {}), + }); + }); + const state = getOrCreateSession( + identified.sessionID, + projectPath, + projectPathProvisional ? "cwd" : "header", + credentialFingerprint, + config, + ); + if (upstreamUpdate.changed) { + if (upstreamState.lastUpstream) { + state.lastUpstream = upstreamState.lastUpstream; + } else { + delete state.lastUpstream; + } + state.upstreamByProvider = upstreamState.upstreamByProvider; + if (upstreamState._upstreamRequestOrder !== undefined) { + state._upstreamRequestOrder = upstreamState._upstreamRequestOrder; + } else { + delete state._upstreamRequestOrder; + } + if (upstreamState._upstreamRequestOrderByProvider) { + state._upstreamRequestOrderByProvider = + upstreamState._upstreamRequestOrderByProvider; + } else { + delete state._upstreamRequestOrderByProvider; + } + if (upstreamUpdate.resetCache) { + state.cacheAnalytics.lastRequestBody = null; + } + } + if (known) publishKnownSessionHeader(known, state, credentialFingerprint); + else state.credentialFingerprint = credentialFingerprint; + if (identified.tier === 3) observeHeaderValues(req.rawHeaders); + state.projectPath = projectPath; + state.projectPathProvisional = projectPathProvisional; + if (identified.adoptionFingerprint) { + state.fingerprint = identified.adoptionFingerprint; + } + if (pathResult.gitRemote) state.gitRemote = pathResult.gitRemote; + state.messageCount = req.messages.length; + state._dirty = true; + if (credential) { + captureLegacyGlobalAuth(req, config, credential); + setSessionAuth( + state.sessionID, + credential, + extractProviderHeader(req.rawHeaders) || undefined, + ); + } + captureBillingPrefix(state.sessionID, req.system); + captureSessionHeaders(state.sessionID, req.rawHeaders); + return true; + }; + scheduleStreamingPostResponse( + identified.sessionID, + requestGeneration, + async () => { + await downstreamSettled; + await new Promise((resolve) => setImmediate(resolve)); + const pause = provisionalFinalizerPauseForTest; + if (pause) { + pause.onWait(); + await pause.pause; + } + if (requestGeneration !== streamingPostResponseGeneration) return; + if (downstreamWasCancelled()) { + accountUnsuccessfulResponse( + accumulated, + identified.sessionID, + conversationTTLForAccounting(identified.sessionID), + undefined, + () => {}, + ); + return; + } + if ( + identified.guardProject && + conflictsWithConfidentSessionProject(identified.sessionID, pathResult) + ) { + dropOwnedProvisionalKey( + identified.provisionalKey, + identified.sessionID, + ); + return; + } + if (!(await commit())) return; + accountConversationUsage( + accumulated.usage ?? ZERO_USAGE, + accumulated.model, + identified.sessionID, + conversationTTLForAccounting(identified.sessionID), + ); + const state = sessions.get(identified.sessionID); + if (state) state._dirty = true; + }, + () => {}, + true, + requestCredentialFingerprint(req.rawHeaders, config) ?? undefined, + ); + return response; +} + /** * Check whether the upstream prompt cache is likely still warm for this * session. Returns true when a warmup ping was successfully sent within @@ -12188,11 +14567,34 @@ export function mergeRecallUsage( return merged; } +function assertCurrentPipelineGeneration( + signal: AbortSignal | undefined, + requestGeneration: number, +): void { + signal?.throwIfAborted(); + if ( + pipelineResetInProgress || + requestGeneration !== streamingPostResponseGeneration + ) { + throw new DOMException("gateway pipeline generation changed", "AbortError"); + } +} + async function handleConversationTurn( req: GatewayRequest, config: GatewayConfig, requestOrder: number, + requestGeneration: number, + downstreamSettled: Promise, + downstreamWasCancelled: () => boolean, + claimSession: (sessionID: string) => Promise, ): Promise { + if ( + pipelineResetInProgress || + requestGeneration !== streamingPostResponseGeneration + ) { + return errorResponse(503, "Gateway pipeline generation changed"); + } // --- 1. Project path & init --- // Enrich headers with context markers injected by lore-hermes plugin. // This lets getProjectPath() pick up [lore:project=...] via the existing @@ -12205,21 +14607,74 @@ async function handleConversationTurn( // --- 2. Capture auth credentials for background workers --- const cred = extractAuth(req.rawHeaders); - const legacyGlobalProvider = cred - ? captureLegacyGlobalAuth(req, config, cred) - : undefined; // --- 3. Session identification --- - const { sessionID, isNew, tier } = await identifySession( - req, - pathResult.path, - config, - ); - - // Strip [lore:session-id=...] and [lore:project=...] context markers from - // user messages so they are not forwarded to the upstream LLM, stored in - // temporal storage, or visible to the model. + const admitted = await withIdentityAdmission(req, config, async () => { + const result = await identifySession( + req, + pathResult.path, + pathResult.source, + requestGeneration, + config, + ); + const claimed = result.isNew || result.provisionalIdentity === true; + if (claimed) await claimSession(result.sessionID); + const revalidateConfirmedIdentity = + !result.isNew && result.provisionalIdentity !== true && result.tier !== 3; + return { identified: result, claimed, revalidateConfirmedIdentity }; + }); + const { identified } = admitted; + const { sessionID, isNew, tier } = identified; + if (!admitted.claimed) await claimSession(sessionID); + if (identified.expectedUnowned && !legacyAdoptionTargetIsUnowned(sessionID)) { + dropOwnedProvisionalKey(identified.provisionalKey, sessionID); + return errorResponse(404, "No authenticated session found"); + } + if ( + admitted.revalidateConfirmedIdentity && + !confirmedIndexedIdentityResolvesTo(req, sessionID, config) + ) { + return errorResponse(404, "No authenticated session found"); + } + if ( + identified.guardProject && + conflictsWithConfidentSessionProject(sessionID, pathResult) + ) { + dropOwnedProvisionalKey(identified.provisionalKey, sessionID); + throw new Error("session project changed during provisional migration"); + } + await awaitStreamingPostResponse(sessionID, req.signal); + assertCurrentPipelineGeneration(req.signal, requestGeneration); + if ( + admitted.revalidateConfirmedIdentity && + !confirmedIndexedIdentityResolvesTo(req, sessionID, config) + ) { + return errorResponse(404, "No authenticated session found"); + } + // Marker-derived project/session data has already been copied into headers; + // strip it before either the provisional verifier or full pipeline forwards. stripContextMarkers(req.messages); + if (identified.provisionalIdentity) { + const preUpstreamPause = pipelinePreUpstreamPauseForTest; + if (preUpstreamPause) { + preUpstreamPause.onWait(); + await preUpstreamPause.pause; + assertCurrentPipelineGeneration(req.signal, requestGeneration); + } + return handleProvisionalConversationTurn( + req, + config, + identified, + pathResult, + requestOrder, + requestGeneration, + downstreamSettled, + downstreamWasCancelled, + ); + } + const legacyGlobalProvider = cred + ? captureLegacyGlobalAuth(req, config, cred) + : undefined; const sessionState = getOrCreateSession( sessionID, @@ -12228,13 +14683,22 @@ async function handleConversationTurn( requestCredentialFingerprint(req.rawHeaders, config) ?? "", config, ); + await beforeUpstreamCaptureForTest?.(req, sessionState); + const sessionSignal = sessionLifecycleSignal(sessionID); + const suppressTemporalStorage = + sessionState.amnesia || req.rawHeaders["x-lore-no-store"] === "true"; + const preUpstreamPause = pipelinePreUpstreamPauseForTest; + if (preUpstreamPause) { + preUpstreamPause.onWait(); + await preUpstreamPause.pause; + assertCurrentPipelineGeneration(req.signal, requestGeneration); + } let projectPath = resolveSessionProjectPath(pathResult, sessionState, config); // Routing and policy are request intent, not a property of a successful // response. Capture now so a failed policy-tightening request still governs // workers, and use the order assigned synchronously in handleRequest so an // older concurrent turn can never overwrite a newer one. - await beforeUpstreamCaptureForTest?.(req, sessionState); const requestUpstreamRoute = captureRequestUpstream( req, sessionState, @@ -12290,7 +14754,10 @@ async function handleConversationTurn( const refRes = await ltm.validateProjectReferences( projectPath, resolver, + Date.now(), + req.signal, ); + assertCurrentPipelineGeneration(req.signal, requestGeneration); if (refRes.penalized > 0) { log.info( `reference drift (remote): penalized ${refRes.penalized}/${refRes.checked} ` + @@ -12298,6 +14765,7 @@ async function handleConversationTurn( ); } } catch (e) { + assertCurrentPipelineGeneration(req.signal, requestGeneration); log.warn("synthetic reference-validation error (non-fatal):", e); } sessionState.refcheckInProbe = false; @@ -12331,7 +14799,14 @@ async function handleConversationTurn( // Initialize the project AFTER path correction so a path-less probe request // never creates a project row for the gateway's cwd or an unattributed // bucket (provider-agnostic: applies to every protocol/client). - await initIfNeeded(projectPath, config, pathResult.gitRemote); + await initIfNeeded( + projectPath, + config, + pathResult.gitRemote, + req.signal, + requestGeneration, + ); + assertCurrentPipelineGeneration(req.signal, requestGeneration); // Mark sub-agent sessions (x-parent-session-id present). // These get their own session but are flagged for cache warming exemption. @@ -12440,28 +14915,19 @@ async function handleConversationTurn( // Track fingerprint for future correlation if (isNew) { - const credentialFingerprint = - requestCredentialFingerprint(req.rawHeaders, config) ?? ""; - const fingerprint = await fingerprintMessages( - req.messages.map((m) => ({ role: m.role, content: m.content })), - usesRemoteSessionBinding(config) - ? { tenantFingerprint: credentialFingerprint } - : { authSuffix: cred ? authFingerprint(cred) : "" }, - ); - sessionState.fingerprint = fingerprint; - // Persist fingerprint immediately — rare event (new session only) - saveSessionTracking(sessionID, { fingerprint, credentialFingerprint }); - - // Seed header learning for new sessions (Tier 2 bootstrap). - // Even Tier 1 sessions don't need this, but it's harmless and - // avoids branching. For Tier 3 (fingerprinted) new sessions, - // this seeds the first round of candidate collection. - if (!sessionState.headerSessionId) { - const result = learnHeaders( - sessionState.candidateHeaders, - req.rawHeaders, + if (!suppressTemporalStorage) { + const credentialFingerprint = + requestCredentialFingerprint(req.rawHeaders, config) ?? ""; + const fingerprint = await fingerprintMessages( + req.messages.map((m) => ({ role: m.role, content: m.content })), + usesRemoteSessionBinding(config) + ? { tenantFingerprint: credentialFingerprint } + : { authSuffix: cred ? authFingerprint(cred) : "" }, ); - sessionState.candidateHeaders = result.updatedCandidates; + assertCurrentPipelineGeneration(req.signal, requestGeneration); + sessionState.fingerprint = fingerprint; + // Persist fingerprint immediately — rare event (new session only) + saveSessionTracking(sessionID, { fingerprint, credentialFingerprint }); } // Re-check knowledge files on new session start. The file watcher @@ -12622,6 +15088,7 @@ async function handleConversationTurn( // getModelEntrySync sites — worker selection, cost metrics — intentionally // keep using the sync fallback on the very first turn; they self-correct.) await ensureModelDataReady(); + assertCurrentPipelineGeneration(req.signal, requestGeneration); // Price the session model from the provider it is actually routed to (the // X-Lore-Provider header), not the flat last-write-wins entry — a bare id // published by several providers at different cache prices would otherwise @@ -12919,22 +15386,21 @@ async function handleConversationTurn( // catalog scan) independently, compounding the very latency that caused // the retries. Share one in-flight compute; the settled value lands in // stableLtmCache before the promise resolves, so re-reading is race-free. - const pending = stableLtmInFlight.get(sessionID); - if (pending) { - await pending; - stable = stableLtmCache.get(sessionID); - } - if (!stable) { - stable = await singleFlightStableLtm(sessionID, () => + stable = await singleFlightStableLtm( + sessionID, + (signal) => computeStableLtm( sessionID, projectPath, cfg, contextHint, prefBudget, + signal, + requestGeneration, ), - ); - } + req.signal, + ); + assertCurrentPipelineGeneration(req.signal, requestGeneration); } stableLtmText = stable?.formatted; @@ -13008,6 +15474,7 @@ async function handleConversationTurn( sessionID, contextBudget, { + signal: req.signal, excludeCategories: ["preference"], ...(contextHint ? { contextHint } : {}), ...(stickyIds.size ? { stickyIds } : {}), @@ -13017,6 +15484,7 @@ async function handleConversationTurn( overflowSink, }, ); + assertCurrentPipelineGeneration(req.signal, requestGeneration); freshContextEntries = contextEntries; freshContextOverflow = overflowSink.map((e) => ({ id: e.id, @@ -13226,6 +15694,7 @@ async function handleConversationTurn( // budget, not the system cache budget), so it adds nothing here. setLtmTokens(stable?.tokenCount ?? 0, sessionID); } catch (e) { + assertCurrentPipelineGeneration(req.signal, requestGeneration); log.error("LTM injection failed:", e); setLtmTokens(0, sessionID); } finally { @@ -13269,7 +15738,13 @@ async function handleConversationTurn( // snapshot transform() reads, so its loadDistillationsCached hits the cache // instead of the DB. On a pool timeout it's a no-op and transform() falls back // to the identical in-process load. - await prewarmDistillationSnapshot(projectPath, sessionID, loreMessages); + await prewarmDistillationSnapshot( + projectPath, + sessionID, + loreMessages, + req.signal, + ); + assertCurrentPipelineGeneration(req.signal, requestGeneration); const result = transform({ messages: loreMessages, projectPath, @@ -13330,6 +15805,7 @@ async function handleConversationTurn( sessionID, contextBudget, { + signal: req.signal, excludeCategories: ["preference"], ...(contextHint ? { contextHint } : {}), ...(stickyIds.size ? { stickyIds } : {}), @@ -13339,6 +15815,7 @@ async function handleConversationTurn( overflowSink, }, ); + assertCurrentPipelineGeneration(req.signal, requestGeneration); const contextOverflow = overflowSink.map((e) => ({ id: e.id, category: e.category, @@ -13519,6 +15996,7 @@ async function handleConversationTurn( } } } catch (e) { + assertCurrentPipelineGeneration(req.signal, requestGeneration); // On error, leave the step-6 LTM state intact (cache, pin, text) // so the turn proceeds with the pre-refresh knowledge rather than // an inconsistent state. The next turn will retry via step 6. @@ -13648,6 +16126,7 @@ async function handleConversationTurn( cfg.knowledge.referenceValidation ) { const peek = await ltm.peekProjectRefsOffloaded(projectPath); + assertCurrentPipelineGeneration(req.signal, requestGeneration); if (!peek.gated && peek.refs.length > 0) { block = buildCombinedResolveRefcheckBlock( target, @@ -13899,6 +16378,7 @@ async function handleConversationTurn( } } } + assertCurrentPipelineGeneration(req.signal, requestGeneration); // Start gen_ai.chat span before the upstream call so it captures real // wall-clock duration (including network latency and streaming time). @@ -13916,6 +16396,41 @@ async function handleConversationTurn( // NO gen_ai.input.messages — privacy (proxy for other people's projects) }, }); + let streamingFinalizerRegistered = false; + let genAiSpanEnded = false; + let recallPersistenceTransaction: + | { commit: () => void; rollback: () => void } + | undefined; + const rollbackRecallPersistence = (): void => { + recallPersistenceTransaction?.rollback(); + recallPersistenceTransaction = undefined; + }; + const endGenAiSpan = (): void => { + if (genAiSpanEnded) return; + genAiSpanEnded = true; + genAiSpan?.end(); + }; + const dropStreamingFinalizer = (): void => { + rollbackRecallPersistence(); + streamingFinalizerRegistered = true; + genAiSpan?.setStatus({ + code: 2, + message: "post-response finalizer dropped", + }); + endGenAiSpan(); + }; + const releaseForeground = (): void => { + if (!streamingFinalizerRegistered && !genAiSpanEnded) { + genAiSpan?.setStatus({ + code: 2, + message: req.stream + ? "stream cancelled before terminal response" + : "request ended before terminal response", + }); + endGenAiSpan(); + } + foregroundAbort.dispose(); + }; let upstreamResult: UpstreamResult; try { @@ -13928,7 +16443,7 @@ async function handleConversationTurn( requestUpstreamRoute, ); } catch (error) { - foregroundAbort.dispose(); + releaseForeground(); throw error; } const upstreamResponse = wrapBodyWithCleanup( @@ -13942,7 +16457,7 @@ async function handleConversationTurn( foregroundOwnershipTransferred = true; return wrapBodyWithCleanup( response, - foregroundAbort.dispose, + releaseForeground, foregroundAbort.signal, ); }; @@ -13950,7 +16465,7 @@ async function handleConversationTurn( try { return await operation; } catch (error) { - if (!foregroundOwnershipTransferred) foregroundAbort.dispose(); + if (!foregroundOwnershipTransferred) releaseForeground(); throw error; } }; @@ -14009,7 +16524,7 @@ async function handleConversationTurn( code: 2, message: `HTTP ${upstreamResponse.status}`, }); - genAiSpan.end(); + endGenAiSpan(); return finishForeground( new Response(errorBody, { status: upstreamResponse.status, @@ -14083,20 +16598,22 @@ async function handleConversationTurn( const side: "before" | "after" = index < position ? "before" : "after"; return [{ id: block.id, name: block.name, input: block.input, side }]; }); - addRecallStoreEntry(sessionState.recallStore, storeKey, { - toolUseId: recallBlock.id, - anchorId, - anchorContextId, - input, - position, - result, - ...(companionToolUses.length > 0 ? { companionToolUses } : {}), - }); - // Persist the store (v46) so the marker still expands byte-identically - // after a gateway restart instead of leaking raw marker text upstream. - saveSessionTracking(sessionState.sessionID, { - recallStore: serializeRecallStore(sessionState.recallStore), - }); + if (!suppressTemporalStorage) { + addRecallStoreEntry(sessionState.recallStore, storeKey, { + toolUseId: recallBlock.id, + anchorId, + anchorContextId, + input, + position, + result, + ...(companionToolUses.length > 0 ? { companionToolUses } : {}), + }); + // Persist the store (v46) so the marker still expands byte-identically + // after a gateway restart instead of leaking raw marker text upstream. + saveSessionTracking(sessionState.sessionID, { + recallStore: serializeRecallStore(sessionState.recallStore), + }); + } const markerText = buildAnchoredRecallMarker( input.query, @@ -14121,14 +16638,20 @@ async function handleConversationTurn( `recall (non-stream, mixed, depth=${recallDepth}): stored result for session ${sessionState.sessionID.slice(0, 16)}`, ); markerResp.usage = cumulativeUsage; - postResponse( - req, - markerResp, - sessionState, - config, - requestBody, - genAiSpan, - ); + if (req.stream) { + finishStreaming(markerResp); + } else { + postResponse( + req, + markerResp, + sessionState, + config, + requestBody, + genAiSpan, + suppressTemporalStorage, + endGenAiSpan, + ); + } return nonStreamHttpResponse( shouldInjectWarning ? injectContextWarning(markerResp, warningText) @@ -14177,6 +16700,7 @@ async function handleConversationTurn( signal, validation: currentModifiedReq.codex ? "codex" : "public", stopAtTerminal: true, + requireCompletedTerminal: true, }), }; let jsonFollowUp: Awaited>; @@ -14198,20 +16722,41 @@ async function handleConversationTurn( recallBlock, foregroundAbort.signal, ); - } catch { + } catch (fetchErr) { + if ( + foregroundAbort.signal.aborted || + (fetchErr instanceof Error && fetchErr.name === "AbortError") + ) { + throw fetchErr; + } + if (fetchErr instanceof ResponsesTerminalError) { + Object.assign( + cumulativeUsage, + mergeRecallUsage( + cumulativeUsage, + fetchErr.response.usage ?? ZERO_USAGE, + ), + ); + } log.error( `recall follow-up fetch failed (non-stream, depth=${recallDepth}) for session ${sessionState.sessionID.slice(0, 16)}`, ); // Fall back to response with marker (no continuation) markerResp.usage = cumulativeUsage; - postResponse( - req, - markerResp, - sessionState, - config, - requestBody, - genAiSpan, - ); + if (req.stream) { + finishStreaming(markerResp); + } else { + postResponse( + req, + markerResp, + sessionState, + config, + requestBody, + genAiSpan, + suppressTemporalStorage, + endGenAiSpan, + ); + } return nonStreamHttpResponse( shouldInjectWarning ? injectContextWarning(markerResp, warningText) @@ -14240,14 +16785,20 @@ async function handleConversationTurn( }); // Fall back to response with marker (no continuation) markerResp.usage = cumulativeUsage; - postResponse( - req, - markerResp, - sessionState, - config, - requestBody, - genAiSpan, - ); + if (req.stream) { + finishStreaming(markerResp); + } else { + postResponse( + req, + markerResp, + sessionState, + config, + requestBody, + genAiSpan, + suppressTemporalStorage, + endGenAiSpan, + ); + } return nonStreamHttpResponse( shouldInjectWarning ? injectContextWarning(markerResp, warningText) @@ -14292,14 +16843,20 @@ async function handleConversationTurn( }; } currentResp.usage = cumulativeUsage; - postResponse( - req, - currentResp, - sessionState, - config, - requestBody, - genAiSpan, - ); + if (req.stream) { + finishStreaming(currentResp); + } else { + postResponse( + req, + currentResp, + sessionState, + config, + requestBody, + genAiSpan, + suppressTemporalStorage, + endGenAiSpan, + ); + } // Telemetry: flag a completion we're about to hand back with NO usable // content (no text, no tool_use) — the "no response data" class // (github-copilot #1052 follow-up). Checked on the model's response, before @@ -14336,6 +16893,119 @@ async function handleConversationTurn( }; const finishWithRecall = async (resp: GatewayResponse): Promise => finishForeground(await awaitForeground(finalizeWithRecall(resp))); + function finishStreaming(resp: GatewayResponse): void { + if (streamingFinalizerRegistered) return; + streamingFinalizerRegistered = true; + scheduleStreamingPostResponse( + sessionState.sessionID, + requestGeneration, + async () => { + await downstreamSettled; + await new Promise((resolve) => setImmediate(resolve)); + if (requestGeneration !== streamingPostResponseGeneration) { + dropStreamingFinalizer(); + return; + } + if (sessionSignal.aborted) { + dropStreamingFinalizer(); + return; + } + if (downstreamWasCancelled()) { + rollbackRecallPersistence(); + accountUnsuccessfulResponse( + resp, + sessionState.sessionID, + sessionState.resolvedConversationTTL, + genAiSpan, + endGenAiSpan, + () => { + sessionState._dirty = true; + }, + ); + return; + } + try { + const postResponseFailed = new Error( + "Responses recall post-response persistence failed", + ); + try { + withTenant(sessionState.storageTenantId ?? "", () => + withSavepoint("responses_recall_post_response", () => { + const persisted = postResponseForTenant( + req, + resp, + sessionState, + config, + requestBody, + genAiSpan, + suppressTemporalStorage, + endGenAiSpan, + ); + if (!persisted) throw postResponseFailed; + recallPersistenceTransaction?.commit(); + }), + ); + recallPersistenceTransaction = undefined; + } catch (error) { + rollbackRecallPersistence(); + if (error !== postResponseFailed) throw error; + } + } catch (error) { + rollbackRecallPersistence(); + throw error; + } + }, + dropStreamingFinalizer, + true, + requestCredentialFingerprint(req.rawHeaders, config) ?? undefined, + ); + } + function finishUnsuccessfulStreaming(resp: GatewayResponse): void { + if (streamingFinalizerRegistered) return; + streamingFinalizerRegistered = true; + scheduleStreamingPostResponse( + sessionState.sessionID, + requestGeneration, + async () => { + await downstreamSettled; + await new Promise((resolve) => setImmediate(resolve)); + rollbackRecallPersistence(); + if ( + requestGeneration !== streamingPostResponseGeneration || + sessionSignal.aborted + ) { + dropStreamingFinalizer(); + return; + } + accountUnsuccessfulResponse( + resp, + sessionState.sessionID, + sessionState.resolvedConversationTTL, + genAiSpan, + endGenAiSpan, + () => { + sessionState._dirty = true; + }, + ); + }, + dropStreamingFinalizer, + true, + requestCredentialFingerprint(req.rawHeaders, config) ?? undefined, + ); + } + async function captureUnsuccessfulResponses( + operation: Promise, + ): Promise<{ response: GatewayResponse; successful: boolean } | undefined> { + try { + return { response: await operation, successful: true }; + } catch (error) { + if (!(error instanceof ResponsesTerminalError)) throw error; + finishUnsuccessfulStreaming(error.response); + return error.status === "incomplete" + ? { response: error.response, successful: false } + : undefined; + } + } if (req.stream && upstreamResponse.body) { // Non-Anthropic upstream streaming responses need their own accumulator @@ -14367,15 +17037,15 @@ async function handleConversationTurn( const responsesVisibleContent: GatewayContentBlock[] = []; return finishForeground( streamResponsesRecallAware(upstreamResponse, { - onComplete: (resp) => - postResponse( - req, - resp, - sessionState, - config, - requestBody, - genAiSpan, - ), + validation: req.codex ? "codex" : "public", + onComplete: (response, successful) => { + if (successful) finishStreaming(response); + else finishUnsuccessfulStreaming(response); + }, + onTransactionReady: (transaction) => { + rollbackRecallPersistence(); + recallPersistenceTransaction = transaction; + }, sessionID: sessionState.sessionID, maxRecallDepth: MAX_RECALL_DEPTH, signal: foregroundAbort.signal, @@ -14392,6 +17062,7 @@ async function handleConversationTurn( stableLtmText, pendingKnowledgeDelta, ); + const deferredTransferRecordings: Array<() => void> = []; const { result, input } = await withTenant( sessionState.storageTenantId ?? "", () => @@ -14407,6 +17078,7 @@ async function handleConversationTurn( getLLMClient(config), alreadyInLtm.size > 0 ? alreadyInLtm : undefined, signal, + (record) => deferredTransferRecordings.push(record), ), ); const recallBlock = acc.content[contentPosition]; @@ -14472,14 +17144,18 @@ async function handleConversationTurn( anchorText, resultText: result, commit: () => { + for (const record of deferredTransferRecordings) record(); + if (suppressTemporalStorage) return; addRecallStoreEntry( sessionState.recallStore, storeKey, storedRecall, ); persistStore(); + recallPersistenceCommitObserver?.(); }, rollback: () => { + if (suppressTemporalStorage) return; if (sessionState.recallStore.delete(storeKey)) persistStore(); }, @@ -14545,15 +17221,10 @@ async function handleConversationTurn( return finishForeground( streamResponsesPassthrough( upstreamResponse, - (resp) => - postResponse( - req, - resp, - sessionState, - config, - requestBody, - genAiSpan, - ), + (response, successful) => { + if (successful) finishStreaming(response); + else finishUnsuccessfulStreaming(response); + }, sessionState.sessionID, req.codex ? "codex" : "public", foregroundAbort.signal, @@ -14562,14 +17233,34 @@ async function handleConversationTurn( } // Warning to inject, or a non-Responses client: buffer the full // upstream, run recall interception, then re-emit. - const resp = await awaitForeground( - accumulateResponsesSSEStream(upstreamResponse, { - signal: foregroundAbort.signal, - validation: req.codex ? "codex" : "public", - stopAtTerminal: true, - }), + const captured = await awaitForeground( + captureUnsuccessfulResponses( + accumulateResponsesSSEStream(upstreamResponse, { + signal: foregroundAbort.signal, + validation: req.codex ? "codex" : "public", + stopAtTerminal: true, + requireCompletedTerminal: true, + }), + ), ); - return finishWithRecall(resp); + if (!captured) { + return finishForeground(errorResponse(502, "Gateway request failed")); + } + if (!captured.successful) { + if (hasRecallToolUse(captured.response)) { + return finishForeground(errorResponse(502, "Gateway request failed")); + } + return finishForeground( + nonStreamHttpResponse( + captured.response, + req.protocol, + req.stream, + undefined, + requestEnablesLongContext(req), + ), + ); + } + return finishWithRecall(captured.response); } if (effectiveProtocol === "openai") { @@ -14606,8 +17297,7 @@ async function handleConversationTurn( ); const anthropicSSE = buildStreamingResponse( upstreamResponse, - (resp) => - postResponse(req, resp, sessionState, config, requestBody, genAiSpan), + finishStreaming, hasRecallTool ? { clientMessages: recallClientMessages, @@ -14616,6 +17306,7 @@ async function handleConversationTurn( sessionState, cacheOptions, upstreamRoute: requestUpstreamRoute, + noStore: suppressTemporalStorage, clientSpeaksAnthropic: req.protocol === "anthropic", stableLtmText, ...(pendingKnowledgeDelta ? { pendingKnowledgeDelta } : {}), @@ -14657,15 +17348,34 @@ async function handleConversationTurn( } // Non-streaming: dispatch to correct accumulator based on upstream protocol. - const resp = await awaitForeground( - accumulateNonStreamResponse( - upstreamResponse, - effectiveProtocol, - modifiedReq.codex === true, - foregroundAbort.signal, + const captured = await awaitForeground( + captureUnsuccessfulResponses( + accumulateNonStreamResponse( + upstreamResponse, + effectiveProtocol, + modifiedReq.codex === true, + foregroundAbort.signal, + ), ), ); - return finishWithRecall(resp); + if (!captured) { + return finishForeground(errorResponse(502, "Gateway request failed")); + } + if (!captured.successful) { + if (hasRecallToolUse(captured.response)) { + return finishForeground(errorResponse(502, "Gateway request failed")); + } + return finishForeground( + nonStreamHttpResponse( + captured.response, + req.protocol, + req.stream, + undefined, + requestEnablesLongContext(req), + ), + ); + } + return finishWithRecall(captured.response); } // --------------------------------------------------------------------------- @@ -14977,15 +17687,60 @@ async function handleLoreSlashCommand( req: GatewayRequest, allSessions: Map, config: GatewayConfig, + claimSession: (sessionID: string) => Promise, ): Promise { const text = lastUserTextTrimmed(req); if (!text.toLowerCase().startsWith("/lore:")) return null; + let state = findLiveSessionState(req, config, allSessions); + const indexedSessionID = findIndexedSessionID(req, config); + if (!state && indexedSessionID) { + const pathResult = getProjectPath(req.system, req.rawHeaders); + state = getOrCreateSession( + indexedSessionID, + pathResult.path, + pathResult.source, + requestCredentialFingerprint(req.rawHeaders, config) ?? "", + config, + ); + } + const sessionID = indexedSessionID ?? state?.sessionID; + if (sessionID) { + await claimSession(sessionID); + if ( + indexedSessionID && + !confirmedIndexedIdentityResolvesTo(req, sessionID, config) + ) { + return slashResponse( + req, + "No authenticated active session found.", + `msg_lore_${Date.now()}`, + ); + } + await awaitStreamingPostResponse(sessionID, req.signal); + req.signal?.throwIfAborted(); + if ( + indexedSessionID && + !confirmedIndexedIdentityResolvesTo(req, sessionID, config) + ) { + return slashResponse( + req, + "No authenticated active session found.", + `msg_lore_${Date.now()}`, + ); + } + } + // Route to specific handlers const warmupResult = handleWarmupSlashCommand(req, allSessions, config); if (warmupResult) return warmupResult; - const curateResult = await handleCurateSlashCommand(req, allSessions, config); + const curateResult = await handleCurateSlashCommand( + req, + allSessions, + config, + claimSession, + ); if (curateResult) return curateResult; const amnesiaResult = handleAmnesiaSlashCommand(req, allSessions, config); @@ -15024,33 +17779,23 @@ function handleAmnesiaSlashCommand( const isOff = lower === "/lore:amnesia:off"; if (!isOn && !isOff) return null; - // Find the session - const known = extractKnownSessionHeader(req.rawHeaders); - let state: SessionState | undefined; - if (known) { - const credentialFingerprint = requestCredentialFingerprint( - req.rawHeaders, - config, - ); - if (credentialFingerprint !== null) { - const indexKey = sessionIndexKey( - credentialFingerprint, - known.headerName, - known.sessionId, - ); - const sid = headerSessionIndex.get(indexKey); - if (sid) state = allSessions.get(sid); - } - } + const state = findLiveSessionState(req, config, allSessions); - if (state) { - state.amnesia = isOn; - log.info( - `amnesia: ${lower} for session=${state.sessionID.slice(0, 16)} — ` + - `storage ${isOn ? "suppressed" : "resumed"}`, + if (!state) { + return slashResponse( + req, + "No active session found. Amnesia mode was not changed.", + `msg_lore_${Date.now()}`, ); } + state.amnesia = isOn; + saveSessionTracking(state.sessionID, { amnesia: isOn }); + log.info( + `amnesia: ${lower} for session=${state.sessionID.slice(0, 16)} — ` + + `storage ${isOn ? "suppressed" : "resumed"}`, + ); + const responseText = isOn ? "Amnesia mode on — memory storage suppressed. Recall still works." : "Amnesia mode off — memory storage resumed."; @@ -15091,6 +17836,8 @@ function handleWarmupSlashCommand( const isOn = lower === "/lore:warm:on"; if (!isStop && !isKeep && !isAuto && !isReset && !isOff && !isOn) return null; + const state = findLiveSessionState(req, config, allSessions); + if ( (isReset || isOff || isOn) && (config.remoteGateway || config.hostedMode) @@ -15101,8 +17848,23 @@ function handleWarmupSlashCommand( ); } - // Reset is a breaker-wide admin action — clear every tripped bucket and - // return immediately (it does not depend on resolving this session). + // Global controls require an authenticated, resolved session. Otherwise any + // network caller could persistently change warming for every tenant. + if ( + (isReset || isOff || isOn) && + (isHostedMode() || + !state || + !state.lastUpstream || + !extractAuth(req.rawHeaders)) + ) { + return slashResponse( + req, + "No authenticated active session found. Global cache warming was not changed.", + `msg_lore_${Date.now()}`, + ); + } + + // Reset is a breaker-wide admin action. if (isReset) { resetCircuitBreaker(); log.info( @@ -15115,8 +17877,7 @@ function handleWarmupSlashCommand( ); } - // on/off are GLOBAL admin actions (persisted KV override) — apply and return - // immediately, independent of this session. + // on/off are GLOBAL admin actions (persisted KV override). if (isOff || isOn) { setWarmingEnabled(isOn); log.info( @@ -15131,25 +17892,6 @@ function handleWarmupSlashCommand( ); } - // Find the session for this request (use the same header-based lookup) - const known = extractKnownSessionHeader(req.rawHeaders); - let state: SessionState | undefined; - if (known) { - const credentialFingerprint = requestCredentialFingerprint( - req.rawHeaders, - config, - ); - if (credentialFingerprint !== null) { - const indexKey = sessionIndexKey( - credentialFingerprint, - known.headerName, - known.sessionId, - ); - const sid = headerSessionIndex.get(indexKey); - if (sid) state = allSessions.get(sid); - } - } - // Update session warmup state if (state) { if (!state.warmup) { @@ -15203,56 +17945,52 @@ async function handleCurateSlashCommand( req: GatewayRequest, allSessions: Map, config: GatewayConfig, + claimSession: (sessionID: string) => Promise, ): Promise { const text = lastUserTextTrimmed(req); if (text.toLowerCase() !== "/lore:curate") return null; - // Find the session - const known = extractKnownSessionHeader(req.rawHeaders); - let state: SessionState | undefined; - let sessionID: string | undefined; - if (known) { - const credentialFingerprint = requestCredentialFingerprint( - req.rawHeaders, + const indexedSessionID = findIndexedSessionID(req, config); + const pathResult = getProjectPath(req.system, req.rawHeaders); + let state = findLiveSessionState(req, config, allSessions); + let sessionID = state?.sessionID; + + if (!state && indexedSessionID) { + state = getOrCreateSession( + indexedSessionID, + pathResult.path, + pathResult.source, + requestCredentialFingerprint(req.rawHeaders, config) ?? "", config, ); - if (credentialFingerprint !== null) { - const indexKey = sessionIndexKey( - credentialFingerprint, - known.headerName, - known.sessionId, - ); - const sid = headerSessionIndex.get(indexKey); - if (sid) { - state = allSessions.get(sid); - sessionID = sid; - } - } + sessionID = indexedSessionID; } - // Fall back to finding any recent session for this project - if (!sessionID) { - // Use the most recently active session - let latest: SessionState | undefined; - const credentialFingerprint = requestCredentialFingerprint( - req.rawHeaders, - config, + if (!sessionID || !state) { + return slashResponse( + req, + "No active session found for curation.", + "msg_lore_curate_none", ); - if (credentialFingerprint !== null) { - for (const s of allSessions.values()) { - if ((s.credentialFingerprint ?? "") !== credentialFingerprint) continue; - if (!latest || s.lastRequestTime > latest.lastRequestTime) { - latest = s; - } - } - } - if (latest) { - state = latest; - sessionID = latest.sessionID; - } } - if (!sessionID || !state) { + await claimSession(sessionID); + if ( + indexedSessionID && + !confirmedIndexedIdentityResolvesTo(req, sessionID, config) + ) { + return slashResponse( + req, + "No active session found for curation.", + "msg_lore_curate_none", + ); + } + await awaitStreamingPostResponse(sessionID, req.signal); + req.signal?.throwIfAborted(); + if ( + indexedSessionID && + !confirmedIndexedIdentityResolvesTo(req, sessionID, config) + ) { return slashResponse( req, "No active session found for curation.", @@ -15260,8 +17998,13 @@ async function handleCurateSlashCommand( ); } - const projectPath = state.projectPath; + const projectPath = resolveSessionProjectPath(pathResult, state, config); + saveSessionTracking(sessionID, { + projectPath: state.projectPath || null, + projectPathProvisional: state.projectPathProvisional === true, + }); const { distillation, curator } = await import("@loreai/core"); + req.signal?.throwIfAborted(); const llm = getLLMClient(config); const model = getWorkerModel(state.lastUpstream); @@ -15279,12 +18022,15 @@ async function handleCurateSlashCommand( skipMeta: true, urgent: true, callType: "direct", + signal: req.signal, workerHealth: makeWorkerHealth(sessionID, "lore-distill"), // #627 Phase 1: stamp the session's gitHead on slash-curate rows. metadata: buildSessionMetadata(state.gitHead), }); + req.signal?.throwIfAborted(); distilled = dResult.distilled; } catch (e) { + req.signal?.throwIfAborted(); log.error("/lore:curate distillation error:", e); } @@ -15298,14 +18044,17 @@ async function handleCurateSlashCommand( projectPath, sessionID, model, + signal: req.signal, workerHealth: makeWorkerHealth(sessionID, "lore-curator"), // #627 Phase 1: stamp the session's gitHead on slash-curate entries. metadata: buildSessionMetadata(state.gitHead), }); + req.signal?.throwIfAborted(); created = cResult.created; updated = cResult.updated; deleted = cResult.deleted; } catch (e) { + req.signal?.throwIfAborted(); log.error("/lore:curate curation error:", e); } @@ -15426,6 +18175,7 @@ export function earlyFlushStreamingResponse( run: (signal: AbortSignal) => Promise, modelId: string, signal?: AbortSignal, + trackOperation?: (operation: Promise) => void, ): Response { const encoder = new TextEncoder(); const keepalive = encoder.encode(`: lore preparing\n\n`); @@ -15460,6 +18210,7 @@ export function earlyFlushStreamingResponse( } let reader: ReadableStreamDefaultReader | undefined; + let responsePromise: Promise | undefined; let cancelled = false; let finished = false; @@ -15487,8 +18238,12 @@ export function earlyFlushStreamingResponse( if (cancelled || finished) return; try { if (!reader) { + if (!responsePromise) { + responsePromise = run(operationSignal); + trackOperation?.(responsePromise); + } const inner = await responseAgainstAbort( - () => run(operationSignal), + () => responsePromise as Promise, operationSignal, ); if (cancelled) { @@ -15501,17 +18256,12 @@ export function earlyFlushStreamingResponse( ) { // The inner response has no streamable SSE body (e.g. an error // Response with a plain string body). Surface as response.failed. - const status = inner.status; - const text = await readForegroundBody( - inner, - true, - undefined, - operationSignal, - ).catch(() => ""); + log.error( + `early-flush inner response was not SSE (status=${inner.status})`, + ); + void inner.body?.cancel(operationSignal.reason).catch(() => {}); if (!cancelled) { - controller.enqueue( - emitFailed(`${status}: ${text.slice(0, 500)}`), - ); + controller.enqueue(emitFailed("Gateway request failed")); finish(controller); } return; @@ -15525,10 +18275,9 @@ export function earlyFlushStreamingResponse( if (chunk.done) finish(controller); else controller.enqueue(chunk.value); } catch (err) { - const message = - err instanceof Error ? err.message : "unknown gateway error"; + log.error("early-flush stream failed:", err); if (!cancelled) { - controller.enqueue(emitFailed(message)); + controller.enqueue(emitFailed("Gateway request failed")); finish(controller); } } @@ -15560,6 +18309,48 @@ export function earlyFlushStreamingResponse( async function handleRequestForTenant( req: GatewayRequest, config: GatewayConfig, +): Promise { + if (!req?.rawHeaders) { + return errorResponse(400, "Malformed request: missing headers"); + } + if (pipelineResetInProgress) { + return errorResponse(503, "Gateway pipeline is resetting"); + } + streamingPostResponsesAccepting = true; + const requestGeneration = streamingPostResponseGeneration; + let resolveDownstreamSettled: (() => void) | undefined; + let downstreamCancelled = false; + const downstreamSettled = new Promise((resolve) => { + resolveDownstreamSettled = resolve; + }); + return runActivePipelineRequest( + req.signal, + (signal, trackOperation, claimSession) => + handleRequestInner( + { ...req, signal }, + config, + requestGeneration, + downstreamSettled, + () => downstreamCancelled, + trackOperation, + claimSession, + ), + () => resolveDownstreamSettled?.(), + () => { + downstreamCancelled = true; + }, + requestCredentialFingerprint(req.rawHeaders, config) ?? undefined, + ); +} + +async function handleRequestInner( + req: GatewayRequest, + config: GatewayConfig, + requestGeneration: number, + downstreamSettled: Promise, + downstreamWasCancelled: () => boolean, + trackOperation: (operation: Promise) => void, + claimSession: (sessionID: string) => Promise, ): Promise { const requestStartMs = Date.now(); const requestOrder = ++upstreamRequestOrder; @@ -15604,7 +18395,12 @@ async function handleRequestForTenant( // --- Case 0: Slash command interception (/lore:*) --- // All /lore:* commands are intercepted here and never forwarded upstream. - const slashResult = await handleLoreSlashCommand(req, sessions, config); + const slashResult = await handleLoreSlashCommand( + req, + sessions, + config, + claimSession, + ); if (slashResult) return slashResult; // --- Case 0.5: Claude Code side-channel → forward upstream untouched --- @@ -15644,7 +18440,13 @@ async function handleRequestForTenant( log.info( `compaction detected: ${reason} messages=${req.messages.length} tools=${req.tools.length}`, ); - return await handleCompaction(req, config); + return await handleCompaction( + req, + config, + requestGeneration, + trackOperation, + claimSession, + ); } // --- Case 2: Meta request (title gen, summary, categorization, etc.) → passthrough --- @@ -15666,15 +18468,30 @@ async function handleRequestForTenant( if (req.stream && req.protocol === "openai-responses") { return earlyFlushStreamingResponse( (signal) => - handleConversationTurn({ ...req, signal }, config, requestOrder), + handleConversationTurn( + { ...req, signal }, + config, + requestOrder, + requestGeneration, + downstreamSettled, + downstreamWasCancelled, + claimSession, + ), req.model, req.signal, + trackOperation, ); } - return await handleConversationTurn(req, config, requestOrder); + return await handleConversationTurn( + req, + config, + requestOrder, + requestGeneration, + downstreamSettled, + downstreamWasCancelled, + claimSession, + ); } catch (err) { - const message = - err instanceof Error ? err.message : "Unknown gateway error"; // Client disconnect / abort is benign — downgrade from error to info. const isAbort = err instanceof DOMException && err.name === "AbortError"; if (isAbort) { @@ -15687,7 +18504,7 @@ async function handleRequestForTenant( } else { log.error("pipeline request failed"); } - return errorResponse(502, message); + return errorResponse(502, "Gateway request failed"); } } diff --git a/packages/gateway/src/recall.ts b/packages/gateway/src/recall.ts index 418198a5..b2bcb1dd 100644 --- a/packages/gateway/src/recall.ts +++ b/packages/gateway/src/recall.ts @@ -928,6 +928,7 @@ export async function executeRecall( llm?: LLMClient, alreadyInLtmIds?: ReadonlySet, signal?: AbortSignal, + deferTransferRecording?: (record: () => void) => void, ): Promise<{ result: string; input: { query: string; scope?: RecallScope; id?: string }; @@ -952,6 +953,7 @@ export async function executeRecall( searchConfig: cfg.search, // Genuine agent recall — record cross-project transfer metrics (#506). recordTransfers: true, + deferTransferRecording, ...(alreadyInLtmIds ? { alreadyInLtmIds } : {}), }); signal?.throwIfAborted(); diff --git a/packages/gateway/src/session.ts b/packages/gateway/src/session.ts index de969711..863f39bd 100644 --- a/packages/gateway/src/session.ts +++ b/packages/gateway/src/session.ts @@ -412,6 +412,7 @@ export function collectCandidateHeaders( export function learnHeaders( candidates: Map | undefined, rawHeaders: Record, + options: { commitGlobal?: boolean } = {}, ): { updatedCandidates: Map; promoted: { name: string; value: string } | null; @@ -435,12 +436,14 @@ export function learnHeaders( } // Update global cross-session tracking - let globalSet = globalHeaderValues.get(name); - if (!globalSet) { - globalSet = new Set(); - globalHeaderValues.set(name, globalSet); + if (options.commitGlobal !== false) { + let globalSet = globalHeaderValues.get(name); + if (!globalSet) { + globalSet = new Set(); + globalHeaderValues.set(name, globalSet); + } + globalSet.add(value); } - globalSet.add(value); } // Remove candidates that disappeared from this request @@ -460,7 +463,9 @@ export function learnHeaders( if (!isSessionHeaderName(name)) continue; const globalSet = globalHeaderValues.get(name); - if (globalSet && globalSet.size > 1) { + const distinctValues = new Set(globalSet); + distinctValues.add(candidate.value); + if (distinctValues.size > 1) { promoted = { name, value: candidate.value }; break; } @@ -472,7 +477,9 @@ export function learnHeaders( if (candidate.seenCount < LEARNING_THRESHOLD) continue; const globalSet = globalHeaderValues.get(name); - if (globalSet && globalSet.size > 1) { + const distinctValues = new Set(globalSet); + distinctValues.add(candidate.value); + if (distinctValues.size > 1) { promoted = { name, value: candidate.value }; break; } @@ -482,6 +489,19 @@ export function learnHeaders( return { updatedCandidates: currentCandidates, promoted }; } +/** Commit candidate-header observations after a provisional turn succeeds. */ +export function observeHeaderValues(rawHeaders: Record): void { + const incoming = collectCandidateHeaders(rawHeaders); + for (const [name, value] of incoming) { + let globalSet = globalHeaderValues.get(name); + if (!globalSet) { + globalSet = new Set(); + globalHeaderValues.set(name, globalSet); + } + globalSet.add(value); + } +} + // --------------------------------------------------------------------------- // Tier 1b: Header value rotation detection // --------------------------------------------------------------------------- diff --git a/packages/gateway/src/stream/openai-responses.ts b/packages/gateway/src/stream/openai-responses.ts index 36af7832..df073518 100644 --- a/packages/gateway/src/stream/openai-responses.ts +++ b/packages/gateway/src/stream/openai-responses.ts @@ -15,6 +15,7 @@ * underlying SSE wire format is the same. */ import { asString, log } from "@loreai/core"; +import { isDeepStrictEqual } from "node:util"; import { ZERO_USAGE, type GatewayContentBlock, @@ -89,6 +90,344 @@ export interface ResponsesAccState { >; } +export const SUPPORTED_RESPONSES_OUTPUT_ITEM_TYPES = [ + "message", + "function_call", + "function_call_output", + "reasoning", + "item_reference", + "web_search_call", + "file_search_call", + "computer_call", + "computer_call_output", + "computer_tool_call", + "computer_tool_call_output", + "code_interpreter_call", + "image_generation_call", + "local_shell_call", + "local_shell_call_output", + "shell_call", + "shell_call_output", + "mcp_call", + "mcp_list_tools", + "mcp_approval_request", + "mcp_approval_response", + "custom_tool_call", + "custom_tool_call_output", + "apply_patch_call", + "apply_patch_call_output", + "program", + "program_output", + "tool_search_call", + "tool_search_output", + "additional_tools", + "compaction", +] as const; +const RESPONSES_OUTPUT_ITEM_TYPES = new Set( + SUPPORTED_RESPONSES_OUTPUT_ITEM_TYPES, +); +type OutputItemStatusPhase = "added" | "done" | "terminal"; +type OutputItemStatusMatrix = Partial< + Record> +>; +const IN_PROGRESS = new Set(["in_progress"]); +const COMPLETED = new Set(["completed"]); +const COMPLETED_OR_INCOMPLETE = new Set(["completed", "incomplete"]); +const COMPLETED_OR_FAILED = new Set(["completed", "failed"]); +const OUTPUT_ITEM_STATUSES_BY_TYPE: Record = { + message: { + added: IN_PROGRESS, + done: COMPLETED_OR_INCOMPLETE, + terminal: COMPLETED_OR_INCOMPLETE, + }, + function_call: { + added: IN_PROGRESS, + done: new Set(["completed", "incomplete", "failed"]), + terminal: new Set(["completed", "incomplete", "failed"]), + }, + reasoning: { + added: IN_PROGRESS, + done: COMPLETED_OR_INCOMPLETE, + terminal: COMPLETED_OR_INCOMPLETE, + }, + web_search_call: { + added: new Set(["in_progress", "searching"]), + done: COMPLETED_OR_FAILED, + terminal: COMPLETED_OR_FAILED, + }, + file_search_call: { + added: new Set(["in_progress", "searching"]), + done: new Set(["completed", "incomplete", "failed"]), + terminal: new Set(["completed", "incomplete", "failed"]), + }, + tool_search_call: { + added: IN_PROGRESS, + done: COMPLETED_OR_INCOMPLETE, + terminal: COMPLETED_OR_INCOMPLETE, + }, + computer_call: { + added: IN_PROGRESS, + done: COMPLETED_OR_INCOMPLETE, + terminal: COMPLETED_OR_INCOMPLETE, + }, + computer_tool_call: { + added: IN_PROGRESS, + done: COMPLETED_OR_INCOMPLETE, + terminal: COMPLETED_OR_INCOMPLETE, + }, + code_interpreter_call: { + added: new Set(["in_progress", "interpreting"]), + done: new Set(["completed", "incomplete", "failed"]), + terminal: new Set(["completed", "incomplete", "failed"]), + }, + image_generation_call: { + added: new Set(["in_progress", "generating"]), + done: COMPLETED_OR_FAILED, + terminal: COMPLETED_OR_FAILED, + }, + local_shell_call: { + added: IN_PROGRESS, + done: COMPLETED_OR_INCOMPLETE, + terminal: COMPLETED_OR_INCOMPLETE, + }, + shell_call: { + added: IN_PROGRESS, + done: COMPLETED_OR_INCOMPLETE, + terminal: COMPLETED_OR_INCOMPLETE, + }, + mcp_call: { + added: new Set(["in_progress", "calling"]), + done: new Set(["completed", "incomplete", "failed"]), + terminal: new Set(["completed", "incomplete", "failed"]), + }, + custom_tool_call: { + added: IN_PROGRESS, + done: COMPLETED_OR_INCOMPLETE, + terminal: COMPLETED_OR_INCOMPLETE, + }, + apply_patch_call: { + added: IN_PROGRESS, + done: COMPLETED, + terminal: COMPLETED, + }, + apply_patch_call_output: { + added: COMPLETED_OR_FAILED, + done: COMPLETED_OR_FAILED, + terminal: COMPLETED_OR_FAILED, + }, +}; +for (const type of [ + "function_call_output", + "computer_call_output", + "computer_tool_call_output", + "local_shell_call_output", + "shell_call_output", + "custom_tool_call_output", + "program_output", + "tool_search_output", +]) { + OUTPUT_ITEM_STATUSES_BY_TYPE[type] = { + added: COMPLETED, + done: COMPLETED, + terminal: COMPLETED, + }; +} + +export function isSupportedResponsesOutputItemType( + type: unknown, +): type is string { + return typeof type === "string" && RESPONSES_OUTPUT_ITEM_TYPES.has(type); +} + +export function isValidResponsesOutputItemStatus( + type: unknown, + status: unknown, + phase: OutputItemStatusPhase, +): boolean { + if (status === undefined) return true; + if (typeof type !== "string" || typeof status !== "string") return false; + return OUTPUT_ITEM_STATUSES_BY_TYPE[type]?.[phase]?.has(status) ?? false; +} + +function recordExtends( + actual: Record, + streamed: Record, + ignored: ReadonlySet = new Set(), +): boolean { + return Object.entries(streamed).every( + ([field, value]) => + ignored.has(field) || terminalValueExtends(actual[field], value), + ); +} + +function terminalValueExtends(actual: unknown, streamed: unknown): boolean { + if (streamed === undefined) return true; + if (Array.isArray(streamed)) { + if (!Array.isArray(actual) || actual.length !== streamed.length) { + return false; + } + return streamed.every((value, index) => + terminalValueExtends(actual[index], value), + ); + } + if (isRecord(streamed)) { + if (!isRecord(actual)) return false; + return Object.entries(streamed).every(([field, value]) => + terminalValueExtends(actual[field], value), + ); + } + return isDeepStrictEqual(actual, streamed); +} + +function sparseValueExtends(actual: unknown, established: unknown): boolean { + if (established === undefined || established === null) return true; + if (typeof established === "string" && established.length === 0) { + return typeof actual === "string"; + } + if (Array.isArray(established)) { + if (!Array.isArray(actual) || actual.length < established.length) { + return false; + } + return established.every((value, index) => + sparseValueExtends(actual[index], value), + ); + } + if (isRecord(established)) { + if (!isRecord(actual)) return false; + return Object.entries(established).every(([field, value]) => + sparseValueExtends(actual[field], value), + ); + } + return isDeepStrictEqual(actual, established); +} + +function sparseTextPartsExtend(actual: unknown, established: unknown): boolean { + if (!Array.isArray(established)) return established === undefined; + if (!Array.isArray(actual) || actual.length < established.length) + return false; + return established.every((rawPart, index) => { + const actualPart = actual[index]; + if (!isRecord(rawPart) || !isRecord(actualPart)) return false; + return Object.entries(rawPart).every(([field, value]) => { + if ( + (field === "text" || field === "refusal") && + typeof value === "string" + ) { + return ( + typeof actualPart[field] === "string" && + actualPart[field].startsWith(value) + ); + } + return sparseValueExtends(actualPart[field], value); + }); + }); +} + +function terminalTextPartsMatch(actual: unknown, streamed: unknown): boolean { + if (!Array.isArray(actual) || !Array.isArray(streamed)) return false; + if (actual.length !== streamed.length) return false; + return streamed.every((streamedPart, index) => { + const actualPart = actual[index]; + if (!isRecord(streamedPart) || !isRecord(actualPart)) return false; + return recordExtends(actualPart, streamedPart); + }); +} + +export function responsesDoneItemMatchesAdded( + done: Record, + added: Record, +): boolean { + if ( + done.type !== added.type || + !isValidResponsesOutputItemStatus(done.type, done.status, "done") || + !isValidResponsesOutputItemStatus(added.type, added.status, "added") + ) { + return false; + } + const ignored = new Set(["status"]); + if (added.type === "function_call") { + if ( + typeof done.arguments !== "string" || + (added.arguments !== undefined && + (typeof added.arguments !== "string" || + !done.arguments.startsWith(added.arguments))) + ) { + return false; + } + ignored.add("arguments"); + } + if (added.type === "message") { + if (!sparseTextPartsExtend(done.content, added.content)) return false; + ignored.add("content"); + } + if (added.type === "reasoning") { + if (!sparseTextPartsExtend(done.summary, added.summary)) return false; + if (!sparseTextPartsExtend(done.content, added.content)) return false; + ignored.add("summary"); + ignored.add("content"); + } + return Object.entries(added).every( + ([field, value]) => + ignored.has(field) || sparseValueExtends(done[field], value), + ); +} + +export function responsesTerminalItemMatches( + actual: Record, + streamed: Record, +): boolean { + if (actual.type !== streamed.type || actual.id !== streamed.id) return false; + if ( + !isValidResponsesOutputItemStatus(actual.type, actual.status, "terminal") || + !isValidResponsesOutputItemStatus(streamed.type, streamed.status, "done") + ) { + return false; + } + if ( + streamed.status !== undefined && + actual.status !== undefined && + actual.status !== streamed.status + ) { + return false; + } + if (actual.type === "function_call") { + return recordExtends(actual, streamed, new Set(["status"])); + } + if (actual.type === "message") { + return ( + recordExtends(actual, streamed, new Set(["status", "content"])) && + terminalTextPartsMatch(actual.content, streamed.content) + ); + } + if (actual.type === "reasoning") { + for (const field of ["summary", "content"] as const) { + if ( + streamed[field] !== undefined && + !terminalTextPartsMatch(actual[field], streamed[field]) + ) { + return false; + } + } + return recordExtends( + actual, + streamed, + new Set(["status", "summary", "content"]), + ); + } + return recordExtends(actual, streamed, new Set(["status"])); +} + +/** Validated unsuccessful terminal, carrying usage for accounting-only paths. */ +export class ResponsesTerminalError extends Error { + constructor( + readonly response: GatewayResponse, + readonly status: string, + ) { + super(`upstream Responses request ended with status ${status}`); + this.name = "ResponsesTerminalError"; + } +} + export function makeResponsesAccState(): ResponsesAccState { return { id: "", @@ -518,6 +857,13 @@ function validatePublicResponsesEvent( typeof addedItem !== "object" || Array.isArray(addedItem) || typeof addedItem.type !== "string" || + !isSupportedResponsesOutputItemType(addedItem.type) || + (addedItem.id !== undefined && typeof addedItem.id !== "string") || + !isValidResponsesOutputItemStatus( + addedItem.type, + addedItem.status, + "added", + ) || (addedItem.type === "message" && typeof addedItem.id !== "string") || (addedItem.type === "message" && addedItem.status !== undefined && @@ -557,6 +903,13 @@ function validatePublicResponsesEvent( typeof doneItem !== "object" || Array.isArray(doneItem) || typeof doneItem.type !== "string" || + !isSupportedResponsesOutputItemType(doneItem.type) || + (doneItem.id !== undefined && typeof doneItem.id !== "string") || + !isValidResponsesOutputItemStatus( + doneItem.type, + doneItem.status, + "done", + ) || (doneItem.type === "message" && typeof doneItem.id !== "string") || (doneItem.type === "message" && doneItem.status !== undefined && @@ -571,11 +924,7 @@ function validatePublicResponsesEvent( doneItem.name.length === 0 || typeof doneItem.arguments !== "string")) || !addedItem || - doneItem.type !== addedItem.type || - (typeof addedItem.id === "string" && doneItem.id !== addedItem.id) || - (addedItem.type === "function_call" && - (doneItem.call_id !== addedItem.call_id || - doneItem.name !== addedItem.name)) || + !responsesDoneItemMatchesAdded(doneItem, addedItem) || (typeof doneItem.id === "string" && state.itemIndexById.get(doneItem.id) !== outputIndex) || (typeof doneItem.call_id === "string" && @@ -792,16 +1141,30 @@ function validateCodexOutputItem( event: "response.output_item.added" | "response.output_item.done", item: unknown, ): asserts item is Record { - if (!isRecord(item) || typeof item.type !== "string") { + if ( + !isRecord(item) || + typeof item.type !== "string" || + !isSupportedResponsesOutputItemType(item.type) + ) { malformedResponsesEvent(); } - for (const field of ["id", "status"] as const) { - if (item[field] !== undefined && typeof item[field] !== "string") { - malformedResponsesEvent(); - } + if (item.id !== undefined && typeof item.id !== "string") { + malformedResponsesEvent(); + } + if ( + !isValidResponsesOutputItemStatus( + item.type, + item.status, + event === "response.output_item.added" ? "added" : "done", + ) + ) { + malformedResponsesEvent(); } if ( event === "response.output_item.added" && + (item.type === "message" || + item.type === "function_call" || + item.type === "reasoning") && item.status !== undefined && item.status !== "in_progress" ) { @@ -809,12 +1172,23 @@ function validateCodexOutputItem( } if ( event === "response.output_item.done" && + (item.type === "message" || item.type === "reasoning") && item.status !== undefined && item.status !== "completed" && item.status !== "incomplete" ) { malformedResponsesEvent(); } + if ( + event === "response.output_item.done" && + item.type === "function_call" && + item.status !== undefined && + item.status !== "completed" && + item.status !== "incomplete" && + item.status !== "failed" + ) { + malformedResponsesEvent(); + } if (item.type === "message" && item.content !== undefined) { if (!Array.isArray(item.content)) malformedResponsesEvent(); for (const part of item.content) { @@ -1069,6 +1443,9 @@ function normalizeCodexItemEvent( } if (event === "response.output_item.done") { reconcileCodexDoneItem(state, outputIndex, item); + if (existing && !responsesDoneItemMatchesAdded(item, existing)) { + malformedResponsesEvent(); + } } else { if (itemId) { bindResponsesIdentity(state.itemIndexById, itemId, outputIndex); @@ -1206,6 +1583,7 @@ function validatedTerminalStatus( event: "response.completed" | "response.done" | "response.incomplete", parsed: Record, validation: ResponsesValidationMode, + requireKnownIncompleteReason = false, ): string { const terminal = parsed.response as Record | undefined; if (!terminal || typeof terminal !== "object" || Array.isArray(terminal)) { @@ -1250,7 +1628,7 @@ function validatedTerminalStatus( : status === "completed" || status === "incomplete"; if (!valid) throw new Error("Responses terminal event/status mismatch"); if (status === "incomplete") { - if (validation === "public") { + if (validation === "public" || requireKnownIncompleteReason) { if ( details?.reason !== undefined && details.reason !== "max_output_tokens" && @@ -1297,6 +1675,9 @@ export async function accumulateResponsesSSEStream( onValidatedEvent?: (event: string, data: string) => void | Promise; /** Passthrough clients must receive a provider's valid failure terminal. */ allowFailureTerminal?: boolean; + /** Buffered callers that run successful-turn side effects must reject + * incomplete terminals rather than treating a parsed body as completion. */ + requireCompletedTerminal?: boolean; /** Internal state injection used by validated true passthrough. */ state?: ResponsesAccState; onReader?: (reader: ReadableStreamDefaultReader) => void; @@ -1364,7 +1745,8 @@ export async function accumulateResponsesSSEStream( if ( opts.validation && event === "response.failed" && - !opts.allowFailureTerminal + !opts.allowFailureTerminal && + !opts.requireCompletedTerminal ) { throw new Error("response.failed terminal"); } @@ -1597,7 +1979,8 @@ export async function accumulateResponsesSSEStream( event === "response.completed" || event === "response.done" || event === "response.incomplete" || - (opts.allowFailureTerminal && event === "response.failed") + ((opts.allowFailureTerminal || opts.requireCompletedTerminal) && + event === "response.failed") ) { const terminal = parsed.response as Record | undefined; if (opts.validation && terminal?.output !== undefined) { @@ -1625,20 +2008,20 @@ export async function accumulateResponsesSSEStream( throw new Error("malformed Responses terminal event"); } snapshotIndices.add(outputIndex); - for (const field of [ - "call_id", - "name", - "arguments", - "content", - ] as const) { + if (snapshot.type === "item_reference") { if ( - snapshot[field] !== undefined && - JSON.stringify(snapshot[field]) !== - JSON.stringify(accumulated[field]) + Object.keys(snapshot).some( + (key) => key !== "type" && key !== "id", + ) ) { throw new Error("malformed Responses terminal event"); } + continue; + } + if (!responsesTerminalItemMatches(snapshot, accumulated)) { + throw new Error("malformed Responses terminal event"); } + state.rawItems.set(outputIndex, { ...accumulated, ...snapshot }); } if ( snapshotIndices.size !== doneItems.size || @@ -1650,7 +2033,12 @@ export async function accumulateResponsesSSEStream( terminalStatus = opts.validation ? event === "response.failed" ? "failed" - : validatedTerminalStatus(event, parsed, opts.validation) + : validatedTerminalStatus( + event, + parsed, + opts.validation, + opts.requireCompletedTerminal, + ) : typeof terminal?.status === "string" ? terminal.status : null; @@ -1699,6 +2087,16 @@ export async function accumulateResponsesSSEStream( if (opts.validation && !terminalStatus) { throw new Error("missing terminal response status"); } + if ( + opts.validation && + opts.requireCompletedTerminal && + terminalStatus !== "completed" + ) { + throw new ResponsesTerminalError( + finalizeResponsesAcc(state), + terminalStatus ?? "unknown", + ); + } return finalizeResponsesAcc(state); } @@ -1736,13 +2134,13 @@ export function formatResponsesEvent(event: string, data: string): string { * injected tool_use never leaks to the client. When the recall tool is present * the caller keeps the buffered `accumulateResponsesSSEStream` path. * - * `onComplete` is invoked exactly once with the accumulated response when the - * upstream stream ends, mirroring the Anthropic `buildStreamingResponse` - * contract so `postResponse` (cost/calibration/temporal) runs identically. + * `onComplete` is invoked exactly once with the accumulated response and a + * success flag. Failed/incomplete/malformed streams must not enter successful + * turn persistence or session-identity confirmation. */ export function streamResponsesPassthrough( upstreamResponse: Response, - onComplete: (response: GatewayResponse) => void, + onComplete: (response: GatewayResponse, successful: boolean) => void, sessionID?: string, validation: ResponsesValidationMode = "public", signal?: AbortSignal, @@ -1855,11 +2253,11 @@ export function streamResponsesPassthrough( keepaliveTimer = null; }; - const finish = (): void => { + const finish = (successful: boolean): void => { if (completed) return; completed = true; try { - onComplete(finalizeResponsesAcc(state)); + onComplete(finalizeResponsesAcc(state), successful); } catch (err) { log.error("openai-responses passthrough onComplete error:", err); } @@ -1905,6 +2303,7 @@ export function streamResponsesPassthrough( event === "response.failed" ) { terminalForwarded = true; + finish(state.terminalEvent === "response.completed"); } }, }, @@ -1912,7 +2311,10 @@ export function streamResponsesPassthrough( clearKeepalive(); if (!completed) { completed = true; - onComplete(accumulated); + onComplete( + accumulated, + state.terminalEvent === "response.completed", + ); } safeClose(); } catch (err) { @@ -1961,17 +2363,14 @@ export function streamResponsesPassthrough( usage: null, error: { type: "server_error", - message: - err instanceof Error - ? err.message - : "upstream stream error", + message: "Upstream response stream failed", }, }, }), ), ), ); - finish(); + finish(false); safeClose(); } }; @@ -2491,6 +2890,10 @@ export function translateAnthropicStreamToResponses( } const finalStatus = mapStatusFromStopReason(resp.stopReason); + const terminalEvent = + finalStatus === "incomplete" + ? "response.incomplete" + : "response.completed"; const ru = resp.usage ?? ZERO_USAGE; const inclusiveInputTokens = safeTokenSum( @@ -2521,14 +2924,24 @@ export function translateAnthropicStreamToResponses( await safeEnqueue( encoder.encode( - emit("response.completed", { - type: "response.completed", + emit(terminalEvent, { + type: terminalEvent, response: { id: respId, object: "response", created_at: created, model: resp.model, status: finalStatus, + ...(finalStatus === "incomplete" + ? { + incomplete_details: { + reason: + resp.stopReason === "content_filter" + ? "content_filter" + : "max_output_tokens", + }, + } + : {}), output: finalOutput, usage: usageData, }, @@ -2570,10 +2983,7 @@ export function translateAnthropicStreamToResponses( usage: null, error: { type: "server_error", - message: - err instanceof Error - ? err.message - : "upstream stream error", + message: "Upstream response stream failed", }, }, }), @@ -2652,6 +3062,7 @@ export function mapStatusFromStopReason(reason: string): string { return "completed"; case "max_tokens": case "length": + case "content_filter": return "incomplete"; default: return "completed"; diff --git a/packages/gateway/src/translate/openai-responses.ts b/packages/gateway/src/translate/openai-responses.ts index 3edb82a4..4cfdc311 100644 --- a/packages/gateway/src/translate/openai-responses.ts +++ b/packages/gateway/src/translate/openai-responses.ts @@ -722,49 +722,57 @@ function buildOpenAIResponsesNonStreamResponse( resp: GatewayResponse, ): Response { const usage = resp.usage ?? ZERO_USAGE; - const output: Array> = []; + const output: Array> = resp.rawOutputItems + ? [...resp.rawOutputItems] + : []; let textContent = ""; const functionCalls: Array> = []; - for (const block of resp.content) { - if (block.type === "text") { - textContent += block.text; - } else if (block.type === "tool_use") { - functionCalls.push({ - type: "function_call", - id: `fc_${block.id}`, - call_id: block.id, - name: block.name, - arguments: JSON.stringify(block.input), + if (!resp.rawOutputItems) { + for (const block of resp.content) { + if (block.type === "text") { + textContent += block.text; + } else if (block.type === "tool_use") { + functionCalls.push({ + type: "function_call", + id: `fc_${block.id}`, + call_id: block.id, + name: block.name, + arguments: JSON.stringify(block.input), + status: "completed", + }); + } + } + + if (textContent) { + output.push({ + type: "message", + id: `msg_${resp.id}`, + role: "assistant", status: "completed", + content: [ + { + type: "output_text", + text: textContent, + annotations: [], + }, + ], }); } - } - if (textContent) { - output.push({ - type: "message", - id: `msg_${resp.id}`, - role: "assistant", - status: "completed", - content: [ - { - type: "output_text", - text: textContent, - annotations: [], - }, - ], - }); + output.push(...functionCalls); } - output.push(...functionCalls); - + const status = mapStopReasonToStatus(resp.stopReason); const response = { id: resp.id.startsWith("resp_") ? resp.id : `resp_${resp.id}`, object: "response", created_at: Math.floor(Date.now() / 1000), model: resp.model, - status: mapStopReasonToStatus(resp.stopReason), + status, + ...(status === "incomplete" + ? { incomplete_details: incompleteDetails(resp.stopReason) } + : {}), output, usage: responsesUsage(usage), }; @@ -783,6 +791,7 @@ function mapStopReasonToStatus(reason: string): string { return "completed"; case "max_tokens": case "length": + case "content_filter": return "incomplete"; case "tool_use": return "completed"; @@ -791,6 +800,13 @@ function mapStopReasonToStatus(reason: string): string { } } +function incompleteDetails(stopReason: string): { reason: string } { + return { + reason: + stopReason === "content_filter" ? "content_filter" : "max_output_tokens", + }; +} + function buildOpenAIResponsesStreamResponse(resp: GatewayResponse): Response { const usage = resp.usage ?? ZERO_USAGE; const encoder = new TextEncoder(); @@ -975,15 +991,20 @@ function buildOpenAIResponsesStreamResponse(resp: GatewayResponse): Response { } } - // response.completed - emit("response.completed", { - type: "response.completed", + const status = mapStopReasonToStatus(resp.stopReason); + const terminalEvent = + status === "incomplete" ? "response.incomplete" : "response.completed"; + emit(terminalEvent, { + type: terminalEvent, response: { id: respId, object: "response", created_at: created, model: resp.model, - status: mapStopReasonToStatus(resp.stopReason), + status, + ...(status === "incomplete" + ? { incomplete_details: incompleteDetails(resp.stopReason) } + : {}), output: resp.content .map((block, i) => { if (block.type === "text") { diff --git a/packages/gateway/src/worker-model.ts b/packages/gateway/src/worker-model.ts index ba76063a..e3174a05 100644 --- a/packages/gateway/src/worker-model.ts +++ b/packages/gateway/src/worker-model.ts @@ -48,6 +48,7 @@ let cachedProviderModels: Map | null = null; let cachedProviderRoutes: Map | null = null; let cachedModelDataAt = 0; let inflightFetch: Promise> | null = null; +let modelDataGeneration = 0; const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour /** @@ -349,7 +350,9 @@ export function fetchModelData(): Promise> { // Deduplicate concurrent calls: return the in-flight promise if one exists if (inflightFetch) return inflightFetch; - inflightFetch = (async () => { + const generation = modelDataGeneration; + let request!: Promise>; + request = (async () => { try { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 10_000); @@ -437,6 +440,7 @@ export function fetchModelData(): Promise> { } } + if (generation !== modelDataGeneration) return modelData; cachedProviderRoutes = providerRoutes; cachedProviderModels = providerModelsIndex; cachedModelData = modelData; @@ -453,11 +457,12 @@ export function fetchModelData(): Promise> { log.warn("models.dev API request failed"); return cachedModelData ?? new Map(); } finally { - inflightFetch = null; + if (inflightFetch === request) inflightFetch = null; } })(); - return inflightFetch; + inflightFetch = request; + return request; } /** @@ -601,6 +606,7 @@ export async function ensureModelDataReady(timeoutMs = 2_000): Promise { /** Clear cached data (for testing). */ export function clearModelDataCache(): void { + modelDataGeneration++; cachedModelData = null; cachedModelDataByProvider = null; cachedProviderModels = null; @@ -626,6 +632,7 @@ export function _setModelDataForTest( byProvider?: Record, providerModelsIndex?: Record, ): void { + modelDataGeneration++; cachedModelData = new Map(Object.entries(entries)); cachedModelDataByProvider = byProvider ? new Map(Object.entries(byProvider)) diff --git a/packages/gateway/test/agents.test.ts b/packages/gateway/test/agents.test.ts index b27b52c9..bc5f8534 100644 --- a/packages/gateway/test/agents.test.ts +++ b/packages/gateway/test/agents.test.ts @@ -1,5 +1,13 @@ import { describe, test, expect, beforeEach, afterEach } from "vitest"; -import { AGENTS, captureUserUpstream } from "../src/cli/agents"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + AGENTS, + captureUserEnvCredential, + captureUserUpstream, +} from "../src/cli/agents"; // --------------------------------------------------------------------------- // Claude Code agent @@ -35,15 +43,25 @@ describe("Claude Code agent envVars", () => { }); test("both X-Lore-Project and X-Lore-Git-Remote coexist when git remote is available", () => { - // Use the actual repo cwd so safeRemote() finds a real git remote. - const env = claude.envVars("http://127.0.0.1:3207", process.cwd()); - const headers = env.ANTHROPIC_CUSTOM_HEADERS ?? ""; - expect(headers).toContain("X-Lore-Project:"); - expect(headers).toContain("X-Lore-Git-Remote:"); - // Project header should appear first (injected before git remote). - const projectIdx = headers.indexOf("X-Lore-Project:"); - const remoteIdx = headers.indexOf("X-Lore-Git-Remote:"); - expect(projectIdx).toBeLessThan(remoteIdx); + const cwd = mkdtempSync(join(tmpdir(), "lore-agent-remote-")); + try { + execFileSync("git", ["init"], { cwd, stdio: "ignore" }); + execFileSync( + "git", + ["remote", "add", "origin", "git@github.com:test/repo.git"], + { cwd, stdio: "ignore" }, + ); + const env = claude.envVars("http://127.0.0.1:3207", cwd); + const headers = env.ANTHROPIC_CUSTOM_HEADERS ?? ""; + expect(headers).toContain(`X-Lore-Project: ${cwd}`); + expect(headers).toContain("X-Lore-Git-Remote: github.com/test/repo"); + // Project header should appear first (injected before git remote). + const projectIdx = headers.indexOf("X-Lore-Project:"); + const remoteIdx = headers.indexOf("X-Lore-Git-Remote:"); + expect(projectIdx).toBeLessThan(remoteIdx); + } finally { + rmSync(cwd, { recursive: true, force: true }); + } }); test("preserves user-set ANTHROPIC_CUSTOM_HEADERS", () => { @@ -392,6 +410,9 @@ describe("captureUserUpstream", () => { "http://localhost:8080", "http://127.0.0.5:1234", "http://[::1]:3000", + "http://[::ffff:127.0.0.1]:3207", + "http://[0:0:0:0:0:ffff:7f00:5]:3207", + "http://[::ffff:0.0.0.0]:3207", ]) { expect( captureUserUpstream(claude, GATEWAY, { ANTHROPIC_BASE_URL: url }), @@ -399,19 +420,28 @@ describe("captureUserUpstream", () => { } }); - test("ignores a non-URL / unparseable value", () => { - const captured = captureUserUpstream(claude, GATEWAY, { - ANTHROPIC_BASE_URL: "not a url", - }); - expect(captured).toBe(null); + test("does not reject an IPv4-mapped public address", () => { + expect( + captureUserUpstream(claude, GATEWAY, { + ANTHROPIC_BASE_URL: "https://[::ffff:192.0.2.1]/v1", + }), + ).toMatchObject({ wireProtocol: "anthropic" }); }); - test("ignores an empty / whitespace value", () => { - expect( + test("rejects a non-URL / unparseable value", () => { + expect(() => + captureUserUpstream(claude, GATEWAY, { + ANTHROPIC_BASE_URL: "not a url", + }), + ).toThrow(/unsafe or invalid upstream URL/); + }); + + test("rejects an explicitly configured whitespace value", () => { + expect(() => captureUserUpstream(claude, GATEWAY, { ANTHROPIC_BASE_URL: " ", }), - ).toBe(null); + ).toThrow(/unsafe or invalid upstream URL/); }); test("returns null for an agent with no adoptable base-URL var (opencode)", () => { @@ -431,4 +461,13 @@ describe("captureUserUpstream", () => { wireProtocol: "gemini", }); }); + + test("does not detach an env credential from an unsafe configured upstream", () => { + expect( + captureUserEnvCredential(claude, { + ANTHROPIC_AUTH_TOKEN: "private-proxy-token", + ANTHROPIC_BASE_URL: "https://proxy.example/v1?api_key=secret", + }), + ).toBeNull(); + }); }); diff --git a/packages/gateway/test/api.test.ts b/packages/gateway/test/api.test.ts index af73f327..3870ff2f 100644 --- a/packages/gateway/test/api.test.ts +++ b/packages/gateway/test/api.test.ts @@ -7,6 +7,10 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { unlinkSync, existsSync } from "node:fs"; import { zstdCompressSync } from "node:zlib"; +import { + loopbackRequest, + type LoopbackRequestInit, +} from "./helpers/loopback-request"; // --------------------------------------------------------------------------- // Test-scoped server setup @@ -64,13 +68,13 @@ afterAll(async () => { // Helpers // --------------------------------------------------------------------------- -function api(path: string, init?: RequestInit): Promise { - return fetch(`${baseURL}${path}`, init); +function api(path: string, init?: LoopbackRequestInit): Promise { + return loopbackRequest(`${baseURL}${path}`, init); } async function apiJSON( path: string, - init?: RequestInit, + init?: LoopbackRequestInit, ): Promise { const res = await api(path, init); return res.json() as Promise; diff --git a/packages/gateway/test/bedrock-routing.test.ts b/packages/gateway/test/bedrock-routing.test.ts index 01f4847b..5422db54 100644 --- a/packages/gateway/test/bedrock-routing.test.ts +++ b/packages/gateway/test/bedrock-routing.test.ts @@ -16,6 +16,7 @@ */ import { describe, test, expect, afterEach } from "vitest"; import { existsSync, unlinkSync } from "node:fs"; +import { loopbackRequest } from "./helpers/loopback-request"; /** A non-streaming Anthropic message response (mantle returns native shape). */ function mantleJSONResponse(): Response { @@ -88,7 +89,7 @@ describe("X-Lore-Provider: bedrock routing (bedrock-mantle)", () => { } }; - const resp = await fetch(`${baseURL}/v1/messages`, { + const resp = await loopbackRequest(`${baseURL}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", diff --git a/packages/gateway/test/bedrock-runtime.test.ts b/packages/gateway/test/bedrock-runtime.test.ts index e4aa3dd9..aab58695 100644 --- a/packages/gateway/test/bedrock-runtime.test.ts +++ b/packages/gateway/test/bedrock-runtime.test.ts @@ -22,6 +22,19 @@ import { resetPipelineState } from "../src/pipeline"; import { startServer } from "../src/server"; import { loadConfig } from "../src/config"; import { close as closeDB } from "@loreai/core"; +import { loopbackRequest } from "./helpers/loopback-request"; + +function localRequest( + baseURL: string, + path: string, + options: { + method: string; + headers?: Record; + body?: string; + }, +): Promise { + return loopbackRequest(`${baseURL}${path}`, options); +} describe("bedrockRuntimeUrl", () => { test("builds the regional bedrock-runtime origin (no trailing slash)", () => { @@ -556,7 +569,7 @@ describe("POST /v1/model/{modelId}/{verb} — Bedrock Runtime API passthrough", inferenceConfig: { maxTokens: 64 }, }); - const resp = await fetch(`${baseURL}/v1/model/${modelId}/${verb}`, { + const resp = await localRequest(baseURL, `/v1/model/${modelId}/${verb}`, { method: "POST", headers: { "content-type": "application/json", @@ -673,7 +686,7 @@ describe("POST /v1/model/{modelId}/{verb} — Bedrock Runtime API passthrough", } }; - const resp = await fetch(`${baseURL}/v1/model/${modelId}/${verb}`, { + const resp = await localRequest(baseURL, `/v1/model/${modelId}/${verb}`, { method: "POST", headers: { "content-type": "application/json", @@ -747,7 +760,7 @@ describe("POST /v1/model/{modelId}/{verb} — Bedrock Runtime API passthrough", } }; - const resp = await fetch(`${baseURL}/v1/model/${modelId}/${verb}`, { + const resp = await localRequest(baseURL, `/v1/model/${modelId}/${verb}`, { method: "POST", headers: { "content-type": "application/json", @@ -803,7 +816,7 @@ describe("POST /v1/model/{modelId}/{verb} — Bedrock Runtime API passthrough", // /v1/models is the Anthropic-protocol models list passthrough — must // NOT be misclassified as a Bedrock Runtime call. - const resp = await fetch(`${baseURL}/v1/models`, { method: "GET" }); + const resp = await localRequest(baseURL, "/v1/models", { method: "GET" }); // 404 because no real upstream is configured; the load-bearing assertion // is that we reached the Anthropic passthrough route, not the Bedrock one. expect(resp.status).not.toBe(200); diff --git a/packages/gateway/test/cache-warmer.test.ts b/packages/gateway/test/cache-warmer.test.ts index 1a37b6a6..8cd30cfe 100644 --- a/packages/gateway/test/cache-warmer.test.ts +++ b/packages/gateway/test/cache-warmer.test.ts @@ -1,4 +1,6 @@ import { describe, test, expect, beforeEach, afterEach } from "vitest"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; import { createHistogram, recordGap, @@ -55,7 +57,14 @@ import { compressBody, normalizeBodyForComparison, } from "../src/cache-analytics"; -import { getKV, setKV } from "@loreai/core"; +import { + decodeWarmupHistogram, + encodeWarmupHistogram, + getKV, + MAX_WARMUP_HISTOGRAM_TOTAL, + normalizeWarmupHistogram, + setKV, +} from "@loreai/core"; import { setCacheSizeSnapshot, setCachePricing, @@ -2331,6 +2340,7 @@ import { getGlobalHistogram, loadGlobalHistograms, flushGlobalHistograms, + getGlobalHistogramsSnapshot, blendedHistogramForSession, } from "../src/cache-warmer"; @@ -2512,12 +2522,50 @@ describe("undefined pid fallback", () => { // Global histogram persistence: backward-compat migration // --------------------------------------------------------------------------- -import { db, projectId } from "@loreai/core"; +import { + db, + ensureProject, + mergeProjectInternal, + projectId, + temporal, + withSavepoint, +} from "@loreai/core"; describe("global histogram persistence", () => { const TEST_PROJECT_PATH = "/tmp/test-histogram-project"; + const projectMutationChild = fileURLToPath( + new URL("./helpers/project-mutation-child.ts", import.meta.url), + ); let pid: string; + function runProjectMutation( + operation: "merge" | "delete", + sourceId: string, + targetId?: string, + ): void { + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + projectMutationChild, + operation, + sourceId, + ...(targetId ? [targetId] : []), + ], + { + cwd: process.cwd(), + env: { + ...process.env, + LORE_DB_PATH: process.env.LORE_DB_PATH, + LORE_NO_DB_TRACING: "1", + }, + encoding: "utf8", + }, + ); + expect(result.status, result.stderr || result.stdout).toBe(0); + } + beforeEach(() => { _resetForTest(); @@ -2562,6 +2610,190 @@ describe("global histogram persistence", () => { expect(hist.counts[3]).toBe(5); }); + test("keeps a safe persisted histogram when valid slot totals overflow in aggregate", () => { + const d = db(); + const binCount = HISTOGRAM_BINS.length + 1; + const perSlotTotal = 5_000_000_000_000_000; + const workCounts = Array.from({ length: binCount }, () => 0); + const eveningCounts = [...workCounts]; + workCounts[0] = perSlotTotal; + eveningCounts[0] = perSlotTotal; + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'work', ?, ?, 100), (?, 'evening', ?, ?, 100)`, + ).run( + pid, + JSON.stringify(workCounts), + perSlotTotal, + pid, + JSON.stringify(eveningCounts), + perSlotTotal, + ); + + loadGlobalHistograms(TEST_PROJECT_PATH); + const loaded = getGlobalHistogram(pid); + expect(Number.isSafeInteger(loaded.total)).toBe(true); + expect(loaded.total).toBeGreaterThan(0); + expect(loaded.counts.reduce((sum, count) => sum + count, 0)).toBe( + loaded.total, + ); + + recordGlobalGap(TEST_PROJECT_PATH, 120_000); + flushGlobalHistograms(); + const row = d + .query( + "SELECT counts, total FROM warmup_histograms WHERE project_id = ? AND time_slot = 'all'", + ) + .get(pid) as { counts: string; total: number }; + expect(Number.isSafeInteger(row.total)).toBe(true); + const exact = decodeWarmupHistogram(row.counts, row.total); + expect(exact).toBeDefined(); + expect(normalizeWarmupHistogram(exact ?? []).total).toBe(row.total); + + _resetForTest(); + loadGlobalHistograms(TEST_PROJECT_PATH); + expect(getGlobalHistogram(pid).total).toBe(row.total); + }); + + test("preserves rare buckets independent of persisted slot order", () => { + const d = db(); + const firstPath = "/tmp/test-histogram-rare-first"; + const secondPath = "/tmp/test-histogram-rare-second"; + const firstId = ensureProject(firstPath); + const secondId = ensureProject(secondPath); + const largeTotal = Number.MAX_SAFE_INTEGER; + const large = Array.from({ length: HISTOGRAM_BINS.length + 1 }, () => 0); + const small = [...large]; + large[0] = largeTotal - 1; + large[1] = 1; + small[2] = 1; + const insert = (projectId: string, largeFirst: boolean) => { + const rows = largeFirst + ? ([ + ["large", large, largeTotal], + ["small", small, 1], + ] as const) + : ([ + ["small", small, 1], + ["large", large, largeTotal], + ] as const); + for (const [slot, counts, total] of rows) { + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, ?, ?, ?, 100)`, + ).run(projectId, slot, JSON.stringify(counts), total); + } + }; + insert(firstId, true); + insert(secondId, false); + + loadGlobalHistograms(firstPath); + loadGlobalHistograms(secondPath); + const first = getGlobalHistogram(firstId); + const second = getGlobalHistogram(secondId); + expect(first).toEqual(second); + expect(first.counts.slice(0, 3).every((count) => count > 0)).toBe(true); + + recordGlobalGap(firstPath, 120_000); + recordGlobalGap(secondPath, 120_000); + flushGlobalHistograms(); + for (const projectId of [firstId, secondId]) { + const row = d + .query( + "SELECT counts, total FROM warmup_histograms WHERE project_id = ? AND time_slot = 'all'", + ) + .get(projectId) as { counts: string; total: number }; + const exact = decodeWarmupHistogram(row.counts, row.total); + expect(exact?.slice(0, 3)).toEqual([BigInt(largeTotal - 1), 1n, 1n]); + } + + _resetForTest(); + loadGlobalHistograms(firstPath); + loadGlobalHistograms(secondPath); + expect(getGlobalHistogram(firstId)).toEqual(getGlobalHistogram(secondId)); + }); + + test("publishes the retained weights after storage-budget compaction", () => { + const d = db(); + const huge = 10n ** 3199n; + const firstWeights = Array.from({ length: 21 }, (_, index) => + index < 10 ? huge * BigInt(index + 1) : 0n, + ); + const secondWeights = Array.from({ length: 21 }, (_, index) => + index >= 10 ? huge * BigInt(index + 1) : 0n, + ); + const first = encodeWarmupHistogram(firstWeights); + const second = encodeWarmupHistogram(secondWeights); + expect(first.counts.length).toBeLessThan(64 * 1024); + expect(second.counts.length).toBeLessThan(64 * 1024); + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'first', ?, ?, 100), (?, 'second', ?, ?, 100)`, + ).run(pid, first.counts, first.total, pid, second.counts, second.total); + + loadGlobalHistograms(TEST_PROJECT_PATH); + recordGlobalGap(TEST_PROJECT_PATH, 120_000); + flushGlobalHistograms(); + + const row = d + .query( + "SELECT counts, total FROM warmup_histograms WHERE project_id = ? AND time_slot = 'all'", + ) + .get(pid) as { counts: string; total: number }; + const retained = decodeWarmupHistogram(row.counts, row.total); + expect(retained).toBeDefined(); + const expected = normalizeWarmupHistogram(retained ?? []); + expect(expected.total).toBe(MAX_WARMUP_HISTOGRAM_TOTAL); + expect(getGlobalHistogram(pid)).toEqual(expected); + + _resetForTest(); + loadGlobalHistograms(TEST_PROJECT_PATH); + expect(getGlobalHistogram(pid)).toEqual(expected); + }); + + test.each([ + [ + "negative compensated counts", + JSON.stringify([-1, 2, ...Array.from({ length: 19 }, () => 0)]), + "1", + ], + [ + "unsafe numeric count", + JSON.stringify([ + Number.MAX_SAFE_INTEGER + 1, + ...Array.from({ length: 20 }, () => 0), + ]), + String(Number.MAX_SAFE_INTEGER + 1), + ], + [ + "forged exact representation", + JSON.stringify({ + v: 1, + counts: [1, ...Array.from({ length: 20 }, () => 0)], + exact: Array.from({ length: 21 }, () => "0"), + }), + "1", + ], + ] as const)("ignores persisted %s", (_name, malformedCounts, total) => { + const d = db(); + const validCounts = Array.from( + { length: HISTOGRAM_BINS.length + 1 }, + () => 0, + ); + validCounts[0] = 3; + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'valid', ?, 3, 100), (?, 'malformed', ?, ?, 100)`, + ).run(pid, JSON.stringify(validCounts), pid, malformedCounts, total); + + loadGlobalHistograms(TEST_PROJECT_PATH); + expect(getGlobalHistogram(pid)).toEqual({ counts: validCounts, total: 3 }); + }); + test("flush writes 'all' row and deletes old slot rows", () => { const d = db(); const now = Date.now(); @@ -2619,6 +2851,509 @@ describe("global histogram persistence", () => { expect(hist.counts[0]).toBe(7); expect(hist.counts[5]).toBe(3); }); + + test("project merge preserves persisted history and unflushed target gaps", () => { + const d = db(); + const sourcePath = "/tmp/test-histogram-merge-source"; + const targetPath = "/tmp/test-histogram-merge-target"; + const sourceId = ensureProject(sourcePath); + const targetId = ensureProject(targetPath); + const sourceCounts = Array.from( + { length: HISTOGRAM_BINS.length + 1 }, + () => 0, + ); + const targetCounts = [...sourceCounts]; + sourceCounts[0] = 7; + targetCounts[0] = 3; + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, 7, 100), (?, 'all', ?, 3, 100)`, + ).run( + sourceId, + JSON.stringify(sourceCounts), + targetId, + JSON.stringify(targetCounts), + ); + + loadGlobalHistograms(targetPath); + recordGlobalGap(targetPath, 5_000); + mergeProjectInternal(sourceId, targetId); + flushGlobalHistograms(); + + const row = d + .query( + "SELECT counts, total FROM warmup_histograms WHERE project_id = ? AND time_slot = 'all'", + ) + .get(targetId) as { counts: string; total: number }; + const counts = JSON.parse(row.counts) as number[]; + expect(row.total).toBe(11); + expect(counts[0]).toBe(11); + expect(getGlobalHistogram(targetId).total).toBe(11); + expect( + d + .query( + "SELECT COUNT(*) AS count FROM warmup_histograms WHERE project_id = ?", + ) + .get(sourceId), + ).toEqual({ count: 0 }); + }); + + test("project merge carries an unflushed source gap into the target", () => { + const d = db(); + const sourcePath = "/tmp/test-histogram-source-dirty"; + const targetPath = "/tmp/test-histogram-source-dirty-target"; + const sourceId = ensureProject(sourcePath); + const targetId = ensureProject(targetPath); + const sourceCounts = Array.from( + { length: HISTOGRAM_BINS.length + 1 }, + () => 0, + ); + const targetCounts = [...sourceCounts]; + sourceCounts[0] = 7; + targetCounts[0] = 3; + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, 7, 100), (?, 'all', ?, 3, 100)`, + ).run( + sourceId, + JSON.stringify(sourceCounts), + targetId, + JSON.stringify(targetCounts), + ); + + loadGlobalHistograms(sourcePath); + recordGlobalGap(sourcePath, 5_000); + mergeProjectInternal(sourceId, targetId); + flushGlobalHistograms(); + + const row = d + .query( + "SELECT counts, total FROM warmup_histograms WHERE project_id = ? AND time_slot = 'all'", + ) + .get(targetId) as { counts: string; total: number }; + expect(row.total).toBe(11); + expect((JSON.parse(row.counts) as number[])[0]).toBe(11); + }); + + test("cross-process merge rehomes an unflushed source gap by retired UUID", () => { + const d = db(); + const sourcePath = "/tmp/test-histogram-cross-process-source"; + const aliasPath = "/tmp/test-histogram-cross-process-alias"; + const targetPath = "/tmp/test-histogram-cross-process-target"; + const sourceId = ensureProject(sourcePath); + const targetId = ensureProject(targetPath); + d.query( + "INSERT INTO project_path_aliases (path, project_id) VALUES (?, ?)", + ).run(aliasPath, sourceId); + expect(projectId(aliasPath)).toBe(sourceId); // prime the process-local memo + const counts = Array.from({ length: HISTOGRAM_BINS.length + 1 }, () => 0); + const sourceCounts = [...counts]; + const targetCounts = [...counts]; + sourceCounts[0] = 7; + targetCounts[0] = 3; + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, 7, 100), (?, 'all', ?, 3, 100)`, + ).run( + sourceId, + JSON.stringify(sourceCounts), + targetId, + JSON.stringify(targetCounts), + ); + + loadGlobalHistograms(aliasPath); + recordGlobalGap(aliasPath, 5_000); + runProjectMutation("merge", sourceId, targetId); + expect(projectId(aliasPath)).toBe(targetId); + expect(ensureProject(aliasPath)).toBe(targetId); + flushGlobalHistograms(); + flushGlobalHistograms(); + + const row = d + .query( + "SELECT counts, total FROM warmup_histograms WHERE project_id = ? AND time_slot = 'all'", + ) + .get(targetId) as { counts: string; total: number }; + expect(row.total).toBe(11); + expect((JSON.parse(row.counts) as number[])[0]).toBe(11); + expect(getGlobalHistogramsSnapshot().has(sourceId)).toBe(false); + expect(getGlobalHistogram(sourceId).total).toBe(11); + expect(getGlobalHistogram(targetId).total).toBe(11); + + const messageId = crypto.randomUUID(); + const storedMessageId = temporal.store({ + projectPath: aliasPath, + info: { + id: messageId, + sessionID: "cross-process-memo-session", + role: "user", + time: { created: Date.now() }, + agent: "build", + model: { providerID: "test", modelID: "test" }, + }, + parts: [ + { + id: crypto.randomUUID(), + sessionID: "cross-process-memo-session", + messageID: messageId, + type: "text", + text: "stored after external project merge", + }, + ], + }); + expect(storedMessageId).toBeDefined(); + expect( + d + .query( + "SELECT source_id, project_id FROM temporal_messages WHERE id = ?", + ) + .get(storedMessageId), + ).toEqual({ source_id: messageId, project_id: targetId }); + expect( + blendedHistogramForSession( + makeSessionState({ + projectPath: aliasPath, + survivalModel: createHistogram(), + }), + ).total, + ).toBe(11); + }); + + test("path reuse cannot redirect an old UUID's dirty gap", () => { + const d = db(); + const sourcePath = "/tmp/test-histogram-reused-source-path"; + const targetPath = "/tmp/test-histogram-reused-target"; + const otherTargetPath = "/tmp/test-histogram-reused-other-target"; + const sourceId = ensureProject(sourcePath); + const targetId = ensureProject(targetPath); + const otherTargetId = ensureProject(otherTargetPath); + const counts = Array.from({ length: HISTOGRAM_BINS.length + 1 }, () => 0); + const sourceCounts = [...counts]; + const targetCounts = [...counts]; + const otherCounts = [...counts]; + sourceCounts[0] = 7; + targetCounts[0] = 3; + otherCounts[0] = 5; + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, 7, 100), + (?, 'all', ?, 3, 100), + (?, 'all', ?, 5, 100)`, + ).run( + sourceId, + JSON.stringify(sourceCounts), + targetId, + JSON.stringify(targetCounts), + otherTargetId, + JSON.stringify(otherCounts), + ); + + loadGlobalHistograms(sourcePath); + recordGlobalGap(sourcePath, 5_000); + runProjectMutation("merge", sourceId, targetId); + const recreatedId = crypto.randomUUID(); + d.query( + "INSERT INTO projects (id, path, name, created_at) VALUES (?, ?, ?, ?)", + ).run(recreatedId, sourcePath, "recreated source", Date.now()); + runProjectMutation("merge", recreatedId, otherTargetId); + flushGlobalHistograms(); + + const rows = d + .query( + "SELECT project_id, total FROM warmup_histograms WHERE project_id IN (?, ?) ORDER BY project_id", + ) + .all(targetId, otherTargetId) as Array<{ + project_id: string; + total: number; + }>; + expect(new Map(rows.map((row) => [row.project_id, row.total]))).toEqual( + new Map([ + [targetId, 11], + [otherTargetId, 5], + ]), + ); + }); + + test("cross-process deletion discards an orphan delta without path misattribution", () => { + const d = db(); + const sourcePath = "/tmp/test-histogram-cross-process-delete"; + const aliasPath = "/tmp/test-histogram-cross-process-delete-alias"; + const sourceId = ensureProject(sourcePath); + d.query( + "INSERT INTO project_path_aliases (path, project_id) VALUES (?, ?)", + ).run(aliasPath, sourceId); + expect(projectId(aliasPath)).toBe(sourceId); // prime the stale-cache precondition + loadGlobalHistograms(aliasPath); + recordGlobalGap(aliasPath, 5_000); + + runProjectMutation("delete", sourceId); + flushGlobalHistograms(); + flushGlobalHistograms(); + + expect(getGlobalHistogramsSnapshot().has(sourceId)).toBe(false); + expect( + d + .query( + "SELECT COUNT(*) AS count FROM warmup_histograms WHERE project_id = ?", + ) + .get(sourceId), + ).toEqual({ count: 0 }); + + const recreatedId = crypto.randomUUID(); + d.query( + "INSERT INTO projects (id, path, name, created_at) VALUES (?, ?, ?, ?)", + ).run(recreatedId, aliasPath, "recreated after delete", Date.now()); + expect(projectId(aliasPath)).toBe(recreatedId); + expect(ensureProject(aliasPath)).toBe(recreatedId); + recordGlobalGap(aliasPath, 5_000); + flushGlobalHistograms(); + expect( + d + .query( + "SELECT total FROM warmup_histograms WHERE project_id = ? AND time_slot = 'all'", + ) + .get(recreatedId), + ).toEqual({ total: 1 }); + }); + + test("outer rollback keeps unflushed histogram state on the source project", () => { + const d = db(); + const sourcePath = "/tmp/test-histogram-merge-rollback-source"; + const targetPath = "/tmp/test-histogram-merge-rollback-target"; + const sourceId = ensureProject(sourcePath); + const targetId = ensureProject(targetPath); + const sourceCounts = Array.from( + { length: HISTOGRAM_BINS.length + 1 }, + () => 0, + ); + const targetCounts = [...sourceCounts]; + sourceCounts[0] = 7; + targetCounts[0] = 3; + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, 7, 100), (?, 'all', ?, 3, 100)`, + ).run( + sourceId, + JSON.stringify(sourceCounts), + targetId, + JSON.stringify(targetCounts), + ); + + loadGlobalHistograms(sourcePath); + recordGlobalGap(sourcePath, 5_000); + expect(() => + withSavepoint("rollback_histogram_merge", () => { + mergeProjectInternal(sourceId, targetId); + throw new Error("rollback merge"); + }), + ).toThrow("rollback merge"); + flushGlobalHistograms(); + + const rows = d + .query( + "SELECT project_id, total FROM warmup_histograms WHERE project_id IN (?, ?) ORDER BY project_id", + ) + .all(sourceId, targetId) as Array<{ project_id: string; total: number }>; + expect(new Map(rows.map((row) => [row.project_id, row.total]))).toEqual( + new Map([ + [sourceId, 8], + [targetId, 3], + ]), + ); + }); + + test("retired histogram IDs ignore a speculative merge in an outer savepoint", () => { + const d = db(); + const sourcePath = "/tmp/test-histogram-retired-in-transaction-source"; + const targetPath = "/tmp/test-histogram-retired-in-transaction-target"; + const speculativePath = + "/tmp/test-histogram-retired-in-transaction-speculative"; + const sourceId = ensureProject(sourcePath); + const targetId = ensureProject(targetPath); + const speculativeId = ensureProject(speculativePath); + const counts = Array.from({ length: HISTOGRAM_BINS.length + 1 }, () => 0); + const sourceCounts = [...counts]; + const targetCounts = [...counts]; + sourceCounts[0] = 7; + targetCounts[0] = 3; + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, 7, 100), (?, 'all', ?, 3, 100)`, + ).run( + sourceId, + JSON.stringify(sourceCounts), + targetId, + JSON.stringify(targetCounts), + ); + mergeProjectInternal(sourceId, targetId); + loadGlobalHistograms(targetPath); + + expect(() => + withSavepoint("speculative_histogram_read", () => { + mergeProjectInternal(targetId, speculativeId); + expect(getGlobalHistogram(sourceId).total).toBe(10); + expect(getGlobalHistogramsSnapshot().has(sourceId)).toBe(false); + expect(getGlobalHistogramsSnapshot().has(speculativeId)).toBe(false); + throw new Error("rollback speculative histogram merge"); + }), + ).toThrow("rollback speculative histogram merge"); + expect(getGlobalHistogram(sourceId).total).toBe(10); + }); + + test("rolled-back merge cannot publish a speculative histogram load", () => { + const d = db(); + const sourcePath = "/tmp/test-histogram-speculative-source"; + const targetPath = "/tmp/test-histogram-speculative-target"; + const sourceId = ensureProject(sourcePath); + const targetId = ensureProject(targetPath); + const counts = Array.from({ length: HISTOGRAM_BINS.length + 1 }, () => 0); + const sourceCounts = [...counts]; + const targetCounts = [...counts]; + sourceCounts[0] = 7; + targetCounts[0] = 3; + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, 7, 100), (?, 'all', ?, 3, 100)`, + ).run( + sourceId, + JSON.stringify(sourceCounts), + targetId, + JSON.stringify(targetCounts), + ); + + expect(() => + withSavepoint("rollback_speculative_histogram_load", () => { + mergeProjectInternal(sourceId, targetId); + expect(loadGlobalHistograms(sourcePath)).toBeUndefined(); + throw new Error("rollback speculative histogram load"); + }), + ).toThrow("rollback speculative histogram load"); + + expect(loadGlobalHistograms(targetPath)).toBe(targetId); + expect(getGlobalHistogram(targetId).total).toBe(3); + expect(getGlobalHistogramsSnapshot().has(sourceId)).toBe(false); + }); + + test("rolled-back merge notification cannot steal a later merge's dirty gap", () => { + const d = db(); + const sourcePath = "/tmp/test-histogram-remerge-source"; + const firstTargetPath = "/tmp/test-histogram-remerge-first-target"; + const actualTargetPath = "/tmp/test-histogram-remerge-actual-target"; + const sourceId = ensureProject(sourcePath); + const firstTargetId = ensureProject(firstTargetPath); + const actualTargetId = ensureProject(actualTargetPath); + const counts = Array.from({ length: HISTOGRAM_BINS.length + 1 }, () => 0); + const sourceCounts = [...counts]; + const firstTargetCounts = [...counts]; + const actualTargetCounts = [...counts]; + sourceCounts[0] = 7; + firstTargetCounts[0] = 3; + actualTargetCounts[0] = 5; + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, 7, 100), + (?, 'all', ?, 3, 100), + (?, 'all', ?, 5, 100)`, + ).run( + sourceId, + JSON.stringify(sourceCounts), + firstTargetId, + JSON.stringify(firstTargetCounts), + actualTargetId, + JSON.stringify(actualTargetCounts), + ); + + loadGlobalHistograms(sourcePath); + recordGlobalGap(sourcePath, 5_000); + expect(() => + withSavepoint("rollback_first_histogram_merge", () => { + mergeProjectInternal(sourceId, firstTargetId); + throw new Error("rollback first merge"); + }), + ).toThrow("rollback first merge"); + mergeProjectInternal(sourceId, actualTargetId); + flushGlobalHistograms(); + + const rows = d + .query( + "SELECT project_id, total FROM warmup_histograms WHERE project_id IN (?, ?) ORDER BY project_id", + ) + .all(firstTargetId, actualTargetId) as Array<{ + project_id: string; + total: number; + }>; + expect(new Map(rows.map((row) => [row.project_id, row.total]))).toEqual( + new Map([ + [firstTargetId, 3], + [actualTargetId, 13], + ]), + ); + }); + + test("rolled-back outgoing edge cannot orphan an earlier committed merge", () => { + const d = db(); + const sourcePath = "/tmp/test-histogram-chain-source"; + const committedTargetPath = "/tmp/test-histogram-chain-committed"; + const rolledBackTargetPath = "/tmp/test-histogram-chain-rolled-back"; + const sourceId = ensureProject(sourcePath); + const committedTargetId = ensureProject(committedTargetPath); + const rolledBackTargetId = ensureProject(rolledBackTargetPath); + const counts = Array.from({ length: HISTOGRAM_BINS.length + 1 }, () => 0); + const sourceCounts = [...counts]; + const committedCounts = [...counts]; + const rolledBackCounts = [...counts]; + sourceCounts[0] = 7; + committedCounts[0] = 3; + rolledBackCounts[0] = 5; + d.query( + `INSERT INTO warmup_histograms + (project_id, time_slot, counts, total, updated_at) + VALUES (?, 'all', ?, 7, 100), + (?, 'all', ?, 3, 100), + (?, 'all', ?, 5, 100)`, + ).run( + sourceId, + JSON.stringify(sourceCounts), + committedTargetId, + JSON.stringify(committedCounts), + rolledBackTargetId, + JSON.stringify(rolledBackCounts), + ); + + loadGlobalHistograms(sourcePath); + recordGlobalGap(sourcePath, 5_000); + mergeProjectInternal(sourceId, committedTargetId); + expect(() => + withSavepoint("rollback_outgoing_histogram_merge", () => { + mergeProjectInternal(committedTargetId, rolledBackTargetId); + throw new Error("rollback outgoing merge"); + }), + ).toThrow("rollback outgoing merge"); + flushGlobalHistograms(); + + const rows = d + .query( + "SELECT project_id, total FROM warmup_histograms WHERE project_id IN (?, ?) ORDER BY project_id", + ) + .all(committedTargetId, rolledBackTargetId) as Array<{ + project_id: string; + total: number; + }>; + expect(new Map(rows.map((row) => [row.project_id, row.total]))).toEqual( + new Map([ + [committedTargetId, 11], + [rolledBackTargetId, 5], + ]), + ); + }); }); // --------------------------------------------------------------------------- diff --git a/packages/gateway/test/codex-compression.test.ts b/packages/gateway/test/codex-compression.test.ts index db08e091..2947753d 100644 --- a/packages/gateway/test/codex-compression.test.ts +++ b/packages/gateway/test/codex-compression.test.ts @@ -48,7 +48,7 @@ describe("zstd-compressed request bodies (issue #1032)", () => { const compressed = zstdCompressSync( Buffer.from(JSON.stringify(anthropicBody(marker))), ); - const resp = await fetch(`${harness.baseURL}/v1/messages`, { + const resp = await harness.request("/v1/messages", { method: "POST", headers: { "content-type": "application/json", @@ -102,7 +102,7 @@ describe("zstd-compressed request bodies (issue #1032)", () => { const compressed = zstdCompressSync( Buffer.from(JSON.stringify(responsesBody)), ); - const resp = await fetch(`${harness.baseURL}/v1/responses`, { + const resp = await harness.request("/v1/responses", { method: "POST", headers: { "content-type": "application/json", @@ -173,7 +173,7 @@ describe("zstd-compressed request bodies (issue #1032)", () => { Buffer.from(JSON.stringify(anthropicBody(marker))), ); try { - await fetch(`${harness.baseURL}/v1/messages`, { + await harness.request("/v1/messages", { method: "POST", headers: { "content-type": "application/json", diff --git a/packages/gateway/test/compact-endpoint-integration.test.ts b/packages/gateway/test/compact-endpoint-integration.test.ts index 1a679af9..a3f77298 100644 --- a/packages/gateway/test/compact-endpoint-integration.test.ts +++ b/packages/gateway/test/compact-endpoint-integration.test.ts @@ -20,23 +20,54 @@ import { createHarness, type Harness } from "./helpers/harness"; import { makeFixtureEntry } from "./helpers/fixtures"; async function postCompact( - baseURL: string, + harness: Harness, body: string, sessionID: string, apiKey: string | null = "test-key", + extraHeaders: Record = {}, ): Promise { const headers: Record = { "content-type": "application/json", "x-lore-session-id": sessionID, + ...extraHeaders, }; if (apiKey) headers["x-api-key"] = apiKey; - return fetch(`${baseURL}/v1/compact`, { + return harness.request("/v1/compact", { method: "POST", headers, body, }); } +async function postResponsesCompact( + harness: Harness, + sessionID: string, + apiKey: string, + extraHeaders: Record = {}, +): Promise { + return harness.request("/v1/responses/compact", { + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": apiKey, + "x-lore-session-id": sessionID, + "x-lore-project": process.cwd(), + ...extraHeaders, + }, + body: JSON.stringify({ + model: "gpt-5.6-sol", + instructions: "You are a coding agent.", + input: [ + { + role: "user", + content: [{ type: "input_text", text: "compact" }], + }, + ], + tools: [], + }), + }); +} + async function chatWithSession( harness: Harness, body: Record, @@ -131,7 +162,7 @@ describe("POST /v1/compact — integration (real session populated via chat turn // 50_000 tokens fits in claude-sonnet-4-6's 872K budget (1M context // minus 128K output) — the gateway should cancel and skip the summary. const compactResp = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: PROJECT_PATH, tokens_before: 50_000, @@ -175,7 +206,7 @@ describe("POST /v1/compact — integration (real session populated via chat turn const mismatchedProject = `${PROJECT_PATH}-other-tenant`; const compactResp = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: mismatchedProject, tokens_before: tokensBefore, @@ -197,6 +228,92 @@ describe("POST /v1/compact — integration (real session populated via chat turn }, ); + it("does not bootstrap a session mapping from unauthenticated compaction", async () => { + harness = await createHarness({ + fixtures: [ + makeFixtureEntry({ seq: 0, requestMessages: [], responseText: "hi" }), + ], + projectPath: PROJECT_PATH, + }); + expect( + ( + await chatWithSession( + harness, + realConversationBody("claude-sonnet-4-6"), + "victim-session", + ) + ).status, + ).toBe(200); + + const attackerSession = "attacker-bootstrap-session"; + const bootstrap = await postCompact( + harness, + JSON.stringify({ project_path: PROJECT_PATH }), + attackerSession, + "garbage-credential", + ); + expect(bootstrap.status).toBe(404); + + const exfiltration = await postResponsesCompact( + harness, + attackerSession, + "garbage-credential", + ); + expect(exfiltration.status).toBe(404); + expect(await exfiltration.text()).not.toContain("output_text"); + expect( + harness.queryDB<{ session_id: string }>( + `SELECT session_id FROM session_state + WHERE header_name = 'x-lore-session-id' AND header_session_id = ?`, + [attackerSession], + ), + ).toEqual([]); + }); + + it("does not migrate an unknown canonical compaction header through an alias", async () => { + harness = await createHarness({ + fixtures: [ + makeFixtureEntry({ seq: 0, requestMessages: [], responseText: "hi" }), + ], + projectPath: PROJECT_PATH, + }); + const alias = "established-compaction-alias"; + const established = await harness.chat( + realConversationBody("claude-sonnet-4-6"), + "test-key", + { + "x-session-affinity": alias, + "x-lore-provider": "anthropic", + }, + ); + expect(established.status).toBe(200); + + const unknownCanonical = "unknown-canonical-compaction"; + const compact = await postCompact( + harness, + JSON.stringify({ project_path: PROJECT_PATH }), + unknownCanonical, + "test-key", + { "x-session-affinity": alias }, + ); + expect(compact.status).toBe(404); + + const responsesCompact = await postResponsesCompact( + harness, + unknownCanonical, + "test-key", + { "x-session-affinity": alias }, + ); + expect(responsesCompact.status).toBe(404); + expect( + harness.queryDB<{ session_id: string }>( + `SELECT session_id FROM session_state + WHERE header_name = 'x-lore-session-id' AND header_session_id = ?`, + [unknownCanonical], + ), + ).toEqual([]); + }); + it("rejects a compact request without a provider credential", async () => { harness = await createHarness({ fixtures: [ @@ -216,7 +333,7 @@ describe("POST /v1/compact — integration (real session populated via chat turn ).toBe(200); const response = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: PROJECT_PATH, tokens_before: 50_000 }), sessionID, null, @@ -252,7 +369,7 @@ describe("POST /v1/compact — integration (real session populated via chat turn ).toBe(200); const response = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: PROJECT_PATH, tokens_before: tokensBefore, @@ -293,7 +410,7 @@ describe("POST /v1/compact — integration (real session populated via chat turn await harness.restartPipeline(); const response = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: PROJECT_PATH, tokens_before: tokensBefore, @@ -339,7 +456,7 @@ describe("POST /v1/compact — integration (real session populated via chat turn // Exactly at the budget: budget = 1_000_000 - 128_000 = 872_000. The // boundary is INCLUSIVE — see shouldCancelCompactionFromBudget docstring. const compactResp = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: PROJECT_PATH, tokens_before: 872_000, @@ -381,7 +498,7 @@ describe("POST /v1/compact — integration (real session populated via chat turn // signal is absent and the gateway attempted (or attempted to attempt) // a summary rather than silently canceling. const compactResp = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: PROJECT_PATH, tokens_before: 873_000, @@ -409,12 +526,30 @@ describe("POST /v1/compact — integration (real session populated via chat turn // NOT hardcoded to 200K (the bug the prior client-side design had). harness = await createHarness({ fixtures: [ - makeFixtureEntry({ - seq: 0, - requestMessages: [{ role: "user", content: "hello" }], - responseText: "hi", - model: "gpt-4o-mini", - }), + { + ...makeFixtureEntry({ + seq: 0, + requestMessages: [{ role: "user", content: "hello" }], + responseText: "hi", + model: "gpt-4o-mini", + }), + response: { + id: "resp_gpt_mini", + object: "response", + model: "gpt-4o-mini", + status: "completed", + output: [ + { + type: "message", + id: "msg_gpt_mini", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "hi", annotations: [] }], + }, + ], + usage: { input_tokens: 100, output_tokens: 10 }, + }, + }, ], projectPath: PROJECT_PATH, }); @@ -430,7 +565,7 @@ describe("POST /v1/compact — integration (real session populated via chat turn // 100K fits in 111_616 budget → cancel. const cancelResp = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: PROJECT_PATH, tokens_before: 100_000, @@ -446,7 +581,7 @@ describe("POST /v1/compact — integration (real session populated via chat turn // 130K exceeds 111_616 budget → must compact (cancel:false). const compactResp = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: PROJECT_PATH, tokens_before: 130_000, diff --git a/packages/gateway/test/compact-endpoint.test.ts b/packages/gateway/test/compact-endpoint.test.ts index e2d26373..087fea6a 100644 --- a/packages/gateway/test/compact-endpoint.test.ts +++ b/packages/gateway/test/compact-endpoint.test.ts @@ -1,12 +1,17 @@ /** * Coverage for the explicit compaction endpoints (`POST /v1/compact`, used by - * the Pi plugin). Focuses on the request-validation + no-session branches of - * handleCompactEndpoint, which return deterministic responses without any - * upstream call. + * the Pi plugin). Focuses on strict session preflight and request validation + * after a session has been authenticated. */ import { describe, it, expect, afterEach } from "vitest"; import type { Harness } from "./helpers/harness"; import { createHarness } from "./helpers/harness"; +import { + DEFAULT_MODEL, + DEFAULT_SYSTEM, + makeFixtureEntry, + STANDARD_TOOLS, +} from "./helpers/fixtures"; import { generateCompactionSummary, handleCompactEndpoint, @@ -14,29 +19,76 @@ import { } from "../src/pipeline"; import { loadConfig } from "../src/config"; -async function postCompact(baseURL: string, body: string): Promise { - return fetch(`${baseURL}/v1/compact`, { +async function postCompact( + harness: Harness, + body: string, + sessionID?: string, +): Promise { + const headers: Record = { + "content-type": "application/json", + "x-api-key": "test-key", + }; + if (sessionID) headers["x-lore-session-id"] = sessionID; + return harness.request("/v1/compact", { method: "POST", - headers: { - "content-type": "application/json", - "x-api-key": "test-key", - }, + headers, body, }); } +async function establishSession( + harness: Harness, + sessionID: string, +): Promise { + return harness.chat( + { + model: DEFAULT_MODEL, + max_tokens: 1024, + stream: false, + system: DEFAULT_SYSTEM, + messages: [{ role: "user", content: "Establish this session." }], + tools: STANDARD_TOOLS, + }, + "test-key", + { "x-lore-session-id": sessionID }, + ); +} + describe("POST /v1/compact", () => { let harness: Harness; afterEach(() => harness?.teardown()); - it("returns 400 on invalid JSON", async () => { + it("rejects an unknown session before parsing invalid JSON", async () => { harness = await createHarness({ fixtures: [] }); - const resp = await postCompact(harness.baseURL, "{ not json"); - expect(resp.status).toBe(400); + const resp = await postCompact(harness, "{ not json"); + expect(resp.status).toBe(404); const body = (await resp.json()) as { error: string; message: string }; - expect(body.error).toBe("invalid_request"); - expect(body.message).toBe("Invalid JSON body"); + expect(body.error).toBe("session_not_found"); + }); + + it("rejects invalid JSON after authenticating a valid session", async () => { + harness = await createHarness({ + fixtures: [ + makeFixtureEntry({ + seq: 0, + requestMessages: [ + { role: "user", content: "Establish this session." }, + ], + responseText: "Session established.", + model: DEFAULT_MODEL, + }), + ], + }); + const sessionID = "compact-invalid-json-session"; + expect((await establishSession(harness, sessionID)).status).toBe(200); + + const resp = await postCompact(harness, "{ not json", sessionID); + expect(resp.status).toBe(400); + expect(await resp.json()).toEqual({ + error: "invalid_request", + message: "Invalid JSON body", + }); }); it("rejects missing authentication without reading an indefinite body", async () => { @@ -57,25 +109,70 @@ describe("POST /v1/compact", () => { await response.body?.cancel(); }); - it("returns 400 when project_path is missing", async () => { + it("rejects an unknown session without reading an indefinite body", async () => { + const source = new ReadableStream({ + type: "bytes", + pull() { + return new Promise(() => {}); + }, + }); + const req = new Request("http://gateway.test/v1/compact", { + method: "POST", + headers: { + authorization: "Bearer test-key", + "x-lore-session-id": "unknown-stalled-session", + }, + body: source, + duplex: "half", + } as RequestInit & { duplex: "half" }); + const response = await handleCompactEndpoint(req, loadConfig()); + expect(response.status).toBe(404); + expect(req.bodyUsed).toBe(false); + await response.body?.cancel(); + }); + + it("rejects an unknown session before validating project_path", async () => { harness = await createHarness({ fixtures: [] }); - const resp = await postCompact(harness.baseURL, JSON.stringify({})); - expect(resp.status).toBe(400); + const resp = await postCompact(harness, JSON.stringify({})); + expect(resp.status).toBe(404); const body = (await resp.json()) as { error: string; message: string }; - expect(body.error).toBe("invalid_request"); - expect(body.message).toContain("project_path is required"); + expect(body.error).toBe("session_not_found"); + }); + + it("rejects missing project_path after authenticating a valid session", async () => { + harness = await createHarness({ + fixtures: [ + makeFixtureEntry({ + seq: 0, + requestMessages: [ + { role: "user", content: "Establish this session." }, + ], + responseText: "Session established.", + model: DEFAULT_MODEL, + }), + ], + }); + const sessionID = "compact-missing-project-session"; + expect((await establishSession(harness, sessionID)).status).toBe(200); + + const resp = await postCompact(harness, JSON.stringify({}), sessionID); + expect(resp.status).toBe(400); + expect(await resp.json()).toEqual({ + error: "invalid_request", + message: "project_path is required", + }); }); it("returns 404 when no active session exists for the project", async () => { harness = await createHarness({ fixtures: [] }); const resp = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: process.cwd() }), ); expect(resp.status).toBe(404); const body = (await resp.json()) as { error: string; message: string }; expect(body.error).toBe("session_not_found"); - expect(body.message).toContain("No active session found"); + expect(body.message).toContain("No authenticated session found"); }); }); @@ -114,7 +211,7 @@ describe("POST /v1/compact — tokens_before field", () => { it("ignores tokens_before when the project has no active session (404 wins)", async () => { harness = await createHarness({ fixtures: [] }); const resp = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: process.cwd(), tokens_before: 50_000, @@ -130,7 +227,7 @@ describe("POST /v1/compact — tokens_before field", () => { // schema is exercised. harness = await createHarness({ fixtures: [] }); const resp = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: process.cwd(), tokens_before: 0, @@ -145,7 +242,7 @@ describe("POST /v1/compact — tokens_before field", () => { // The `typeof === "number"` guard drops all of these safely. for (const bad of [null, "100", '"NaN"', true, false, {}]) { const resp = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: process.cwd(), tokens_before: bad, @@ -159,7 +256,7 @@ describe("POST /v1/compact — tokens_before field", () => { it("rejects negative tokens_before — falls through to summary path", async () => { harness = await createHarness({ fixtures: [] }); const resp = await postCompact( - harness.baseURL, + harness, JSON.stringify({ project_path: process.cwd(), tokens_before: -100, diff --git a/packages/gateway/test/copilot-routing.e2e.test.ts b/packages/gateway/test/copilot-routing.e2e.test.ts index f0e73e02..4e9cf5ed 100644 --- a/packages/gateway/test/copilot-routing.e2e.test.ts +++ b/packages/gateway/test/copilot-routing.e2e.test.ts @@ -63,7 +63,7 @@ async function captureUpstreamUrl( mockFetch.mockReset(); mockFetch.mockResolvedValue(openAIResponse()); - const res = await fetch(`${harness.baseURL}${ingressPath}`, { + const res = await harness.request(ingressPath, { method: "POST", headers: { "content-type": "application/json", diff --git a/packages/gateway/test/early-flush-stream.test.ts b/packages/gateway/test/early-flush-stream.test.ts index 6a57740e..5d158b81 100644 --- a/packages/gateway/test/early-flush-stream.test.ts +++ b/packages/gateway/test/early-flush-stream.test.ts @@ -109,7 +109,8 @@ describe("earlyFlushStreamingResponse", () => { // the `response:{}` envelope or splits the data field will fail. const match = out.match(FAILED_EVENT_RE); expect(match).not.toBeNull(); - expect(match?.[1]).toBe("boom"); + expect(match?.[1]).toBe("Gateway request failed"); + expect(out).not.toContain("boom"); }); test("emits a canonical response.failed envelope when the inner response is a non-SSE error body", async () => { @@ -120,9 +121,8 @@ describe("earlyFlushStreamingResponse", () => { const out = await drain(resp); const match = out.match(FAILED_EVENT_RE); expect(match).not.toBeNull(); - // The HTTP status and the truncated upstream body surface in the message. - expect(match?.[1]).toContain("429"); - expect(match?.[1]).toContain("upstream 429"); + expect(match?.[1]).toBe("Gateway request failed"); + expect(out).not.toContain("upstream 429"); }); test("the pipeline runs once when downstream starts reading", async () => { diff --git a/packages/gateway/test/empty-completion-telemetry.e2e.test.ts b/packages/gateway/test/empty-completion-telemetry.e2e.test.ts index 8fb18f8d..7a6d356c 100644 --- a/packages/gateway/test/empty-completion-telemetry.e2e.test.ts +++ b/packages/gateway/test/empty-completion-telemetry.e2e.test.ts @@ -30,7 +30,7 @@ async function runWithUpstream( headers: { "content-type": contentType }, }), ); - await fetch(`${harness.baseURL}/v1/chat/completions`, { + await harness.request("/v1/chat/completions", { method: "POST", headers: { "content-type": "application/json", diff --git a/packages/gateway/test/eviction.test.ts b/packages/gateway/test/eviction.test.ts index 884a201f..15b434eb 100644 --- a/packages/gateway/test/eviction.test.ts +++ b/packages/gateway/test/eviction.test.ts @@ -390,6 +390,30 @@ describe("evictIdleSessions", () => { expect(sessions.has("warming-sess")).toBe(true); }); + test("does not evict sessions with an externally-owned response finalizer", () => { + const sessions = new Map(); + sessions.set( + "response-active", + makeSessionState({ + sessionID: "response-active", + lastRequestTime: Date.now() - 2_000_000, + }), + ); + + const evicted = evictIdleSessions( + makeConfig({ sessionEvictionTimeoutSeconds: 1800 }), + sessions, + EMPTY_SET, + EMPTY_SET, + Date.now(), + undefined, + (sessionID) => sessionID === "response-active", + ); + + expect(evicted).toBe(0); + expect(sessions.has("response-active")).toBe(true); + }); + test("does not evict sessions still executing tools", () => { const sessions = new Map(); sessions.set( @@ -566,6 +590,32 @@ describe("startIdleScheduler", () => { } }); + test("amnesia sessions never enter idle background work", async () => { + vi.useFakeTimers(); + try { + const sessions = new Map(); + sessions.set( + "amnesia-idle-session", + makeSessionState({ + sessionID: "amnesia-idle-session", + amnesia: true, + lastRequestTime: Date.now() - 10 * 60 * 1000, + lastStopReason: "end_turn", + }), + ); + let idleRuns = 0; + const stop = startIdleScheduler(makeConfig(), sessions, async () => { + idleRuns++; + }); + + await vi.advanceTimersByTimeAsync(31_000); + expect(idleRuns).toBe(0); + stop(); + } finally { + vi.useRealTimers(); + } + }); + test("skips background work when the worker model's provider has no credential (#894)", async () => { vi.useFakeTimers(); try { diff --git a/packages/gateway/test/foreground-body-limit.test.ts b/packages/gateway/test/foreground-body-limit.test.ts index aa851b87..cb8be57f 100644 --- a/packages/gateway/test/foreground-body-limit.test.ts +++ b/packages/gateway/test/foreground-body-limit.test.ts @@ -187,4 +187,173 @@ describe("foreground response body limits", () => { const result = await accumulateNonStreamResponse(response, "openai"); expect(result.usage).toMatchObject({ inputTokens: 7, outputTokens: 2 }); }); + + test("rejects malformed validated non-stream Responses incomplete details", async () => { + const response = new Response( + JSON.stringify({ + id: "resp_incomplete", + model: "gpt-test", + status: "incomplete", + incomplete_details: { reason: 7 }, + output: [], + usage: { input_tokens: 7, output_tokens: 2 }, + }), + { headers: { "content-type": "application/json" } }, + ); + + await expect( + accumulateNonStreamResponse( + response, + "openai-responses", + false, + undefined, + true, + ), + ).rejects.toThrow("upstream Responses request did not complete"); + }); + + test("rejects provider-specific non-stream incomplete reasons for Codex", async () => { + const response = new Response( + JSON.stringify({ + id: "resp_codex_incomplete", + model: "gpt-test", + status: "incomplete", + incomplete_details: { reason: "provider_specific" }, + output: [], + usage: { input_tokens: 7, output_tokens: 2 }, + }), + { headers: { "content-type": "application/json" } }, + ); + + await expect( + accumulateNonStreamResponse( + response, + "openai-responses", + true, + undefined, + true, + ), + ).rejects.toThrow("upstream Responses request did not complete"); + }); + + test("rejects in-progress items in a completed non-stream response", async () => { + const response = new Response( + JSON.stringify({ + id: "resp_in_progress_item", + model: "gpt-test", + status: "completed", + output: [ + { + type: "message", + id: "msg_in_progress_item", + role: "assistant", + status: "in_progress", + content: [{ type: "output_text", text: "partial" }], + }, + ], + usage: { input_tokens: 7, output_tokens: 2 }, + }), + { headers: { "content-type": "application/json" } }, + ); + + await expect( + accumulateNonStreamResponse(response, "openai-responses"), + ).rejects.toThrow("upstream Responses request did not complete"); + }); + + test.each([ + { + type: "reasoning", + id: "rs_without_status", + summary: [], + }, + { + type: "web_search_call", + id: "ws_completed", + status: "completed", + action: { type: "search", query: "lore" }, + }, + { + type: "function_call_output", + id: "fc_output", + call_id: "call_output", + output: "result", + status: "completed", + }, + { + type: "function_call", + id: "fc_failed", + call_id: "call_failed", + name: "lookup", + arguments: "{}", + status: "failed", + }, + { + type: "image_generation_call", + id: "image_failed", + status: "failed", + result: null, + }, + ])("accepts standard non-stream output item $type", async (item) => { + const result = await accumulateNonStreamResponse( + new Response( + JSON.stringify({ + id: "resp_standard_item", + model: "gpt-test", + status: "completed", + output: [item], + usage: { input_tokens: 7, output_tokens: 2 }, + }), + { headers: { "content-type": "application/json" } }, + ), + "openai-responses", + ); + + expect(result.rawOutputItems).toEqual([item]); + }); + + test.each([ + { + type: "reasoning", + id: "rs_in_progress", + status: "in_progress", + summary: [], + }, + { + type: "item_reference", + id: "ref_extra", + content: [], + }, + { + type: "provider_specific_output", + id: "unknown_output", + }, + ])("rejects malformed non-stream output item $type", async (item) => { + const response = new Response( + JSON.stringify({ + id: "resp_malformed_item", + model: "gpt-test", + status: "completed", + output: [item], + usage: { input_tokens: 7, output_tokens: 2 }, + }), + { headers: { "content-type": "application/json" } }, + ); + + await expect( + accumulateNonStreamResponse(response, "openai-responses"), + ).rejects.toThrow("upstream Responses request did not complete"); + }); + + test("rejects provider-specific incomplete reasons in sniffed Codex SSE", async () => { + const wire = + 'event: response.incomplete\ndata: {"type":"response.incomplete","response":{"id":"resp_codex_sse_incomplete","status":"incomplete","incomplete_details":{"reason":"provider_specific"}}}\n\n'; + const response = new Response(wire, { + headers: { "content-type": "text/event-stream" }, + }); + + await expect( + accumulateNonStreamResponse(response, "openai-responses", true), + ).rejects.toThrow("malformed Responses terminal event"); + }); }); diff --git a/packages/gateway/test/foreground-routes-abort.test.ts b/packages/gateway/test/foreground-routes-abort.test.ts index 23ae2922..efb8f546 100644 --- a/packages/gateway/test/foreground-routes-abort.test.ts +++ b/packages/gateway/test/foreground-routes-abort.test.ts @@ -1,9 +1,15 @@ import { afterEach, describe, expect, test, vi } from "vitest"; import { loadConfig } from "../src/config"; import { + getActiveSessions, + handleCompactEndpoint, + handleRequest, + handleResponsesCompactEndpoint, passthroughResponsesCompact, resetPipelineState, + setUpstreamInterceptor, } from "../src/pipeline"; +import type { GatewayRequest } from "../src/translate/types"; import { handleModelsPassthrough, startServer } from "../src/server"; import { upstreamFetch } from "../src/fetch"; @@ -24,6 +30,63 @@ function modelsRequest(signal?: AbortSignal): Request { return new Request("http://gateway.test/v1/models", { signal }); } +function successfulResponsesResponse(): Response { + return new Response( + JSON.stringify({ + id: "resp_session_setup", + object: "response", + created_at: 0, + model: "unrouted-model", + status: "completed", + output: [ + { + type: "message", + id: "msg_session_setup", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "ok", annotations: [] }], + }, + ], + usage: { input_tokens: 1, output_tokens: 1 }, + }), + { headers: { "content-type": "application/json" } }, + ); +} + +async function establishSession( + sessionID: string, + upstreamUrl?: string, +): Promise { + setUpstreamInterceptor(async () => successfulResponsesResponse()); + const request: GatewayRequest = { + protocol: "openai-responses", + model: "unrouted-model", + system: "You are a coding agent. ".repeat(30), + messages: [ + { role: "user", content: [{ type: "text", text: "one" }] }, + { role: "assistant", content: [{ type: "text", text: "two" }] }, + { role: "user", content: [{ type: "text", text: "three" }] }, + ], + tools: [ + { name: "read", description: "read", inputSchema: {} }, + { name: "write", description: "write", inputSchema: {} }, + { name: "edit", description: "edit", inputSchema: {} }, + ], + stream: false, + maxTokens: 1024, + metadata: {}, + rawHeaders: { + authorization: "Bearer test-key", + "x-lore-session-id": sessionID, + "x-lore-project": "/tmp", + "x-lore-no-store": "true", + ...(upstreamUrl ? { "x-lore-upstream-url": upstreamUrl } : {}), + }, + }; + await (await handleRequest(request, config)).text(); + setUpstreamInterceptor(undefined); +} + const ROUTES = [ { name: "POST /v1/responses/compact fallback", @@ -47,10 +110,180 @@ const ROUTES = [ afterEach(async () => { vi.useRealTimers(); mockedFetch.mockReset(); + setUpstreamInterceptor(undefined); await resetPipelineState({ fast: true }); }); describe("foreground passthrough route aborts", () => { + test.each(["compact", "responses-compact"] as const)( + "applies the foreground deadline to a stalled %s upload", + async (route) => { + const sessionID = `stalled-${route}`; + await establishSession(sessionID); + vi.useFakeTimers(); + const source = new ReadableStream({ + type: "bytes", + pull() { + return new Promise(() => {}); + }, + }); + const request = new Request(`http://gateway.test/v1/${route}`, { + method: "POST", + headers: { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-session-id": sessionID, + "x-lore-project": "/tmp", + }, + body: source, + duplex: "half", + } as RequestInit & { duplex: "half" }); + const pending = + route === "compact" + ? handleCompactEndpoint(request, config) + : handleResponsesCompactEndpoint(request, config); + + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(300_000); + const response = await pending; + expect(response.status).toBe(502); + expect(await response.text()).not.toContain("timed out"); + }, + ); + + test("routes Responses compaction credentials only to the explicit upstream", async () => { + mockedFetch.mockResolvedValue(new Response("{}")); + const response = await passthroughResponsesCompact( + JSON.stringify({ model: "custom-model", input: [] }), + { + authorization: "Bearer custom-provider-key", + "x-lore-provider": "custom-provider", + "x-lore-upstream-url": "https://custom.example.test/v1", + }, + config, + ); + await response.text(); + + expect(fetchUrl(mockedFetch.mock.calls[0]?.[0])).toBe( + "https://custom.example.test/v1/responses/compact", + ); + expect(mockedFetch.mock.calls[0]?.[1]?.headers).toMatchObject({ + authorization: "Bearer custom-provider-key", + }); + }); + + test("does not forward credentials for an unresolved explicit provider", async () => { + const response = await passthroughResponsesCompact( + JSON.stringify({ model: "custom-model", input: [] }), + { + authorization: "Bearer custom-provider-key", + "x-lore-provider": "custom-provider", + }, + config, + ); + + expect(response.status).toBe(502); + expect(mockedFetch).not.toHaveBeenCalled(); + }); + + test("pins Responses compaction to the captured session route", async () => { + const sessionID = "route-less-compact-session"; + await establishSession(sessionID); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === sessionID, + ); + const trustedRoute = state?.lastUpstream; + expect(trustedRoute?.url).toBe("https://api.openai.com"); + let attackerCalled = false; + let trustedCalled = false; + mockedFetch.mockImplementation((url) => { + const target = fetchUrl(url); + if (target.startsWith("https://attacker.example.test")) { + attackerCalled = true; + } + if (target === "https://api.openai.com/v1/responses/compact") { + trustedCalled = true; + } + return Promise.resolve(new Response("{}")); + }); + + const response = await handleResponsesCompactEndpoint( + new Request("http://gateway.test/v1/responses/compact", { + method: "POST", + headers: { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-session-id": sessionID, + "x-lore-project": "/tmp", + "x-lore-upstream-url": "https://attacker.example.test/v1", + }, + body: JSON.stringify({ + model: "unrouted-model", + instructions: "You are a coding agent.", + input: [ + { + role: "user", + content: [{ type: "input_text", text: "compact" }], + }, + ], + tools: [], + }), + }), + config, + ); + + expect(response.status).toBe(200); + await response.text(); + expect(attackerCalled).toBe(false); + expect(trustedCalled).toBe(true); + expect(state?.lastUpstream).toBe(trustedRoute); + }); + + test("pins structural compaction fallback to the session route", async () => { + const sessionID = "trusted-structural-route-session"; + await establishSession(sessionID, "https://trusted.example.test/v1"); + let attackerCalled = false; + let trustedCalled = false; + mockedFetch.mockImplementation((url) => { + const target = fetchUrl(url); + attackerCalled ||= target.startsWith("https://attacker.example.test"); + trustedCalled ||= target.startsWith("https://trusted.example.test"); + return Promise.resolve(successfulResponsesResponse()); + }); + const request: GatewayRequest = { + protocol: "openai-responses", + model: "unrouted-model", + system: "You are an anchored context summarization assistant.", + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "Create an anchored summary from the conversation history above.", + }, + ], + }, + ], + tools: [], + stream: false, + maxTokens: 1024, + metadata: {}, + rawHeaders: { + authorization: "Bearer test-key", + "x-lore-session-id": sessionID, + "x-lore-project": "/tmp", + "x-lore-upstream-url": "https://attacker.example.test/v1", + }, + }; + + const response = await handleRequest(request, config); + expect(response.status).toBe(200); + await response.text(); + expect(trustedCalled).toBe(true); + expect(attackerCalled).toBe(false); + }); + test("pipeline reset aborts an actual Bedrock streaming route and unblocks listener close", async () => { let upstreamSignal: AbortSignal | undefined; let markUpstreamStarted!: () => void; @@ -108,13 +341,9 @@ describe("foreground passthrough route aborts", () => { ); const stalledRead = reader.read(); - // Match production shutdown ordering: stop accepting requests first, - // then abort registered foreground work so server.close() can settle. const listenerClose = server.stop(); await resetPipelineState({ fast: true }); - // The Web-stream AbortError crosses node:http as a terminated socket; - // either transport surface is acceptable, but the read must settle. await expect(stalledRead).rejects.toBeDefined(); await expect(listenerClose).resolves.toBeUndefined(); expect(upstreamSignal?.aborted).toBe(true); diff --git a/packages/gateway/test/gateway-access-auth.e2e.test.ts b/packages/gateway/test/gateway-access-auth.e2e.test.ts index bbd094d3..b3bf05e6 100644 --- a/packages/gateway/test/gateway-access-auth.e2e.test.ts +++ b/packages/gateway/test/gateway-access-auth.e2e.test.ts @@ -124,7 +124,7 @@ const DATA_PLANE_ROUTES: DataPlaneRoute[] = [ path: "/v1/responses/compact", method: "POST", headers: { - ...projectHeaders, + "x-lore-project": PROJECT, authorization: "Bearer client-openai-token", "content-type": "application/json", "x-lore-provider": "openai", diff --git a/packages/gateway/test/gemini-ingress.e2e.test.ts b/packages/gateway/test/gemini-ingress.e2e.test.ts index 1c941de2..35b4facb 100644 --- a/packages/gateway/test/gemini-ingress.e2e.test.ts +++ b/packages/gateway/test/gemini-ingress.e2e.test.ts @@ -12,6 +12,7 @@ */ import { describe, test, expect, afterEach, vi } from "vitest"; import { fetchArgUrl } from "./helpers/fetch-url"; +import { loopbackRequest } from "./helpers/loopback-request"; vi.mock("../src/fetch", () => ({ upstreamFetch: vi.fn() })); @@ -87,7 +88,7 @@ async function sendGemini( : geminiUpstreamResponse(), ); - const res = await fetch(`${harness.baseURL}${path}`, { + const res = await harness.request(path, { method: "POST", headers: { "content-type": "application/json", @@ -184,7 +185,7 @@ describe("native Gemini ingress → generativelanguage upstream (full pipeline)" mockFetch.mockResolvedValue(geminiUpstreamResponse()); // REST/google-generativeai style: key in the query, NO x-goog-api-key header. - await fetch( + await loopbackRequest( `${harness.baseURL}/v1beta/models/gemini-2.5-pro:generateContent?key=qkey123`, { method: "POST", diff --git a/packages/gateway/test/github-copilot-url.e2e.test.ts b/packages/gateway/test/github-copilot-url.e2e.test.ts index 02f78e1d..597dce05 100644 --- a/packages/gateway/test/github-copilot-url.e2e.test.ts +++ b/packages/gateway/test/github-copilot-url.e2e.test.ts @@ -55,7 +55,7 @@ async function captureUpstreamUrl( mockFetch.mockReset(); mockFetch.mockResolvedValue(openAIResponse()); - const res = await fetch(`${harness.baseURL}/v1/chat/completions`, { + const res = await harness.request("/v1/chat/completions", { method: "POST", headers: { "content-type": "application/json", diff --git a/packages/gateway/test/harness.test.ts b/packages/gateway/test/harness.test.ts index b952639c..8e594a97 100644 --- a/packages/gateway/test/harness.test.ts +++ b/packages/gateway/test/harness.test.ts @@ -60,7 +60,7 @@ describe("gateway test harness", () => { expect(port).not.toBe(1); }); - test("chat works when the OS selects a WHATWG fetch-blocked port", async () => { + test("loopback requests work on a WHATWG fetch-blocked port", async () => { const forbiddenPorts = [6667, 6666, 6668, 6669, 6697, 6566, 6000, 5060]; const userMessage = "Test the loopback harness transport."; const fixtures = makeConversationFixtures([ @@ -87,6 +87,9 @@ describe("gateway test harness", () => { ); } + const health = await harness.request("/health"); + expect(health.status).toBe(200); + const response = await harness.chat({ model: DEFAULT_MODEL, max_tokens: 128, diff --git a/packages/gateway/test/helpers/harness.ts b/packages/gateway/test/helpers/harness.ts index 24e60084..de5d14d9 100644 --- a/packages/gateway/test/helpers/harness.ts +++ b/packages/gateway/test/helpers/harness.ts @@ -16,11 +16,10 @@ */ import { unlinkSync, existsSync, mkdirSync, writeFileSync } from "node:fs"; import { DatabaseSync, type SQLInputValue } from "node:sqlite"; -import { request as httpRequest } from "node:http"; -import { Readable } from "node:stream"; import type { FixtureEntry } from "../../src/recorder"; import type { GatewayConfig } from "../../src/config"; import type { SimulatedCacheTurn } from "./simulated-cache"; +import { loopbackRequest } from "./loopback-request"; export const TEST_GATEWAY_AUTH_TOKEN = "test-gateway-access-token-32-bytes-minimum"; @@ -63,6 +62,15 @@ export interface Harness { baseURL: string; /** Path to the isolated temp DB */ dbPath: string; + /** Send a loopback request without WHATWG fetch's browser-only port blocklist. */ + request( + path: string, + init?: { + method?: string; + headers?: HeadersInit; + body?: string | Uint8Array; + }, + ): Promise; /** Send a POST /v1/messages request, return the raw Response */ chat( requestBody: unknown, @@ -219,16 +227,30 @@ export async function createHarness(opts: HarnessOptions): Promise { } } - // --- 7. chat() helper --- + // --- 7. Loopback request helpers --- + async function request( + path: string, + init: { + method?: string; + headers?: HeadersInit; + body?: string | Uint8Array; + } = {}, + ): Promise { + // WHATWG fetch rejects a browser-defined list of "bad ports". Port 0 can + // legitimately assign one of those ports to the local test server, making + // an otherwise valid harness request fail nondeterministically. Use Node's + // HTTP transport for all loopback endpoints while retaining a web + // Response and streaming body for callers. + return loopbackRequest(`${baseURL}${path}`, init); + } + async function chat( requestBody: unknown, apiKey: string | null = "test-key", extraHeaders?: Record, ): Promise { - const body = JSON.stringify(requestBody); const headers: Record = { "content-type": "application/json", - "content-length": String(Buffer.byteLength(body)), "anthropic-version": "2023-06-01", // Provide a confident project binding by default so the synthetic // project-resolution probe is never triggered in harness-based tests. @@ -238,42 +260,10 @@ export async function createHarness(opts: HarnessOptions): Promise { }; if (apiKey !== null) headers["x-api-key"] = apiKey; Object.assign(headers, extraHeaders); - // WHATWG fetch rejects a browser-defined list of "bad ports". Port 0 can - // legitimately assign one of those ports to the local test server, making - // an otherwise valid harness request fail nondeterministically. Use Node's - // HTTP transport for this loopback-only helper while retaining a web - // Response and streaming body for callers. - return new Promise((resolve, reject) => { - const request = httpRequest( - { - hostname: "127.0.0.1", - port: server.port, - path: "/v1/messages", - method: "POST", - headers, - }, - (incoming) => { - const responseHeaders = new Headers(); - for (let i = 0; i < incoming.rawHeaders.length; i += 2) { - responseHeaders.append( - incoming.rawHeaders[i], - incoming.rawHeaders[i + 1], - ); - } - resolve( - new Response( - Readable.toWeb(incoming) as unknown as ReadableStream, - { - status: incoming.statusCode ?? 500, - statusText: incoming.statusMessage, - headers: responseHeaders, - }, - ), - ); - }, - ); - request.once("error", reject); - request.end(body); + return request("/v1/messages", { + method: "POST", + headers, + body: JSON.stringify(requestBody), }); } @@ -318,6 +308,7 @@ export async function createHarness(opts: HarnessOptions): Promise { return { baseURL, dbPath, + request, chat, queryDB, upstreamBodies, diff --git a/packages/gateway/test/helpers/loopback-request.ts b/packages/gateway/test/helpers/loopback-request.ts new file mode 100644 index 00000000..070e844b --- /dev/null +++ b/packages/gateway/test/helpers/loopback-request.ts @@ -0,0 +1,99 @@ +import { request as httpRequest } from "node:http"; +import { isIP } from "node:net"; +import { Readable } from "node:stream"; + +export interface LoopbackRequestInit { + method?: string; + headers?: HeadersInit; + body?: BodyInit | Uint8Array | null; +} + +export async function loopbackRequest( + input: string | URL, + init: LoopbackRequestInit = {}, +): Promise { + const url = input instanceof URL ? input : new URL(input); + if (url.protocol !== "http:") { + throw new Error(`loopbackRequest requires http, received ${url.protocol}`); + } + const hostname = + url.hostname.startsWith("[") && url.hostname.endsWith("]") + ? url.hostname.slice(1, -1) + : url.hostname; + if ( + hostname !== "localhost" && + hostname !== "::1" && + !(isIP(hostname) === 4 && hostname.startsWith("127.")) + ) { + throw new Error( + `loopbackRequest requires a loopback host, received ${hostname}`, + ); + } + let body: string | Uint8Array | undefined; + if (typeof init.body === "string") body = init.body; + else if (init.body instanceof URLSearchParams) body = init.body.toString(); + else if (init.body instanceof Blob) { + body = new Uint8Array(await init.body.arrayBuffer()); + } else if (init.body instanceof ArrayBuffer) body = new Uint8Array(init.body); + else if (ArrayBuffer.isView(init.body)) { + body = new Uint8Array( + init.body.buffer, + init.body.byteOffset, + init.body.byteLength, + ); + } else if (init.body != null) { + throw new Error( + "loopbackRequest does not support streaming or FormData bodies", + ); + } + const headers = Object.fromEntries(new Headers(init.headers).entries()); + if (body !== undefined && headers["content-length"] === undefined) { + headers["content-length"] = String( + typeof body === "string" ? Buffer.byteLength(body) : body.byteLength, + ); + } + const method = init.method ?? "GET"; + return new Promise((resolve, reject) => { + const outgoing = httpRequest( + { + hostname, + port: url.port ? Number(url.port) : 80, + path: `${url.pathname}${url.search}`, + method, + headers, + }, + (incoming) => { + const status = incoming.statusCode ?? 500; + const responseHeaders = new Headers(); + for (let i = 0; i < incoming.rawHeaders.length; i += 2) { + responseHeaders.append( + incoming.rawHeaders[i], + incoming.rawHeaders[i + 1], + ); + } + const hasBody = + method.toUpperCase() !== "HEAD" && + status !== 204 && + status !== 205 && + status !== 304; + if (!hasBody) incoming.resume(); + resolve( + new Response( + hasBody + ? (Readable.toWeb( + incoming, + ) as unknown as ReadableStream) + : null, + { + status, + statusText: incoming.statusMessage, + headers: responseHeaders, + }, + ), + ); + }, + ); + outgoing.once("error", reject); + outgoing.end(body); + }); +} diff --git a/packages/gateway/test/helpers/pg-harness.ts b/packages/gateway/test/helpers/pg-harness.ts index 8c1b572c..e43fc7a6 100644 --- a/packages/gateway/test/helpers/pg-harness.ts +++ b/packages/gateway/test/helpers/pg-harness.ts @@ -17,6 +17,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { Client, type ClientConfig } from "pg"; +import { loopbackRequest } from "./loopback-request"; const exec = promisify(execFile); // Test-only HS256 secret: signs tokens for the throwaway local Postgres only. @@ -109,7 +110,7 @@ async function waitForRest(url: string, timeoutMs = 30_000): Promise { const deadline = Date.now() + timeoutMs; for (;;) { try { - const r = await fetch(`${url}/`, { method: "GET" }); + const r = await loopbackRequest(`${url}/`, { method: "GET" }); // PostgREST answers the root with the OpenAPI spec once schema is loaded. if (r.status < 500) return; } catch { diff --git a/packages/gateway/test/helpers/project-mutation-child.ts b/packages/gateway/test/helpers/project-mutation-child.ts new file mode 100644 index 00000000..75e8d440 --- /dev/null +++ b/packages/gateway/test/helpers/project-mutation-child.ts @@ -0,0 +1,19 @@ +import { close, mergeProjectInternal } from "../../../core/src/db"; +import { deleteProject } from "../../../core/src/data"; + +const [operation, sourceId, targetId] = process.argv.slice(2); +if (!operation || !sourceId) + throw new Error("missing project mutation arguments"); + +try { + if (operation === "merge") { + if (!targetId) throw new Error("missing merge target"); + mergeProjectInternal(sourceId, targetId); + } else if (operation === "delete") { + deleteProject(sourceId); + } else { + throw new Error(`unknown project mutation: ${operation}`); + } +} finally { + close(); +} diff --git a/packages/gateway/test/loopback-request.test.ts b/packages/gateway/test/loopback-request.test.ts new file mode 100644 index 00000000..ca57a7e5 --- /dev/null +++ b/packages/gateway/test/loopback-request.test.ts @@ -0,0 +1,50 @@ +import { createServer } from "node:http"; +import { afterEach, describe, expect, test } from "vitest"; +import { loopbackRequest } from "./helpers/loopback-request"; + +describe("loopbackRequest", () => { + const servers: ReturnType[] = []; + + afterEach(async () => { + await Promise.all( + servers + .splice(0) + .map( + (server) => + new Promise((resolve) => server.close(() => resolve())), + ), + ); + }); + + test.each([204, 205, 304])( + "constructs a bodyless Response for status %s", + async (status) => { + const server = createServer((_request, response) => { + response.writeHead(status); + response.end(); + }); + servers.push(server); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve), + ); + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("no address"); + + const response = await loopbackRequest( + `http://127.0.0.1:${address.port}/${status}`, + ); + expect(response.status).toBe(status); + expect(response.body).toBeNull(); + }, + ); + + test.each(["http://example.com/", "http://192.0.2.1/"])( + "rejects non-loopback destination %s", + async (url) => { + await expect(loopbackRequest(url)).rejects.toThrow( + "loopbackRequest requires a loopback host", + ); + }, + ); +}); diff --git a/packages/gateway/test/openai-responses-recall-aware-stream.test.ts b/packages/gateway/test/openai-responses-recall-aware-stream.test.ts index 9257eb6d..711dcdf2 100644 --- a/packages/gateway/test/openai-responses-recall-aware-stream.test.ts +++ b/packages/gateway/test/openai-responses-recall-aware-stream.test.ts @@ -174,6 +174,53 @@ describe("streamResponsesRecallAware", () => { expect(out).not.toContain("response.failed"); }); + test("finalizes when the client cancels immediately after a no-recall terminal", async () => { + let upstreamCancelled = false; + const upstream = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode( + created("resp_cancel_terminal", "gpt-5.6-terra") + + completed("resp_cancel_terminal", { + input_tokens: 1, + output_tokens: 0, + }), + ), + ); + }, + pull() { + return new Promise(() => {}); + }, + cancel() { + upstreamCancelled = true; + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + let completeCalls = 0; + const client = streamResponsesRecallAware(upstream, { + onComplete: () => completeCalls++, + onRecall: async () => ({ anchorText: "", resultText: "" }), + runFollowUp: async () => { + throw new Error("should not be called"); + }, + }); + if (!client.body) throw new Error("test response has no body"); + const reader = client.body.getReader(); + const decoder = new TextDecoder(); + let seen = ""; + while (!seen.includes("event: response.completed")) { + const { done, value } = await reader.read(); + if (done) throw new Error("stream closed before terminal event"); + if (value) seen += decoder.decode(value, { stream: true }); + } + await reader.cancel(); + + expect(completeCalls).toBe(1); + expect(upstreamCancelled).toBe(true); + }); + test("forwards a principal response.failed exactly once when no recall occurs", async () => { const client = streamResponsesRecallAware( streamFrom([ @@ -712,6 +759,60 @@ describe("streamResponsesRecallAware", () => { expect(await drain(client)).toContain("response.failed"); }); + test("rejects output_item.done changing hosted-tool semantics", async () => { + const item = { + type: "web_search_call", + id: "ws_done_changed", + action: { type: "search", query: "good" }, + }; + const client = streamResponsesRecallAware( + streamFrom([ + created("resp_hosted_done_changed", "gpt-5.6-terra"), + sseEvent("response.output_item.added", { + output_index: 0, + item, + }), + sseEvent("response.output_item.done", { + output_index: 0, + item: { + ...item, + action: { type: "search", query: "EVIL" }, + }, + }), + ]), + { + onComplete: () => {}, + onRecall: async () => ({ anchorText: "", resultText: "" }), + runFollowUp: async () => { + throw new Error("should not run"); + }, + }, + ); + + expect(await drain(client)).toContain("response.failed"); + }); + + test("rejects unknown output item types", async () => { + const client = streamResponsesRecallAware( + streamFrom([ + created("resp_unknown_item", "gpt-5.6-terra"), + sseEvent("response.output_item.added", { + output_index: 0, + item: { type: "provider_specific_output", id: "unknown_item" }, + }), + ]), + { + onComplete: () => {}, + onRecall: async () => ({ anchorText: "", resultText: "" }), + runFollowUp: async () => { + throw new Error("should not run"); + }, + }, + ); + + expect(await drain(client)).toContain("response.failed"); + }); + test("rejects terminal output changing streamed identity", async () => { const client = streamResponsesRecallAware( streamFrom([ @@ -744,6 +845,95 @@ describe("streamResponsesRecallAware", () => { expect(await drain(client)).toContain("response.failed"); }); + test("rejects an unknown public incomplete reason", async () => { + const client = streamResponsesRecallAware( + streamFrom([ + created("resp_unknown_incomplete", "gpt-5.6-terra"), + sseEvent("response.incomplete", { + response: { + id: "resp_unknown_incomplete", + model: "gpt-5.6-terra", + status: "incomplete", + incomplete_details: { reason: "provider_specific" }, + output: [], + }, + }), + ]), + { + onComplete: () => {}, + onRecall: async () => ({ anchorText: "", resultText: "" }), + runFollowUp: async () => { + throw new Error("should not run"); + }, + }, + ); + + const output = await drain(client); + expect(output).toContain("response.failed"); + expect(output).not.toContain("provider_specific"); + }); + + test("public validation rejects a terminal without an output snapshot", async () => { + const client = streamResponsesRecallAware( + streamFrom([ + created("resp_missing_terminal_output", "gpt-5.6-terra"), + sseEvent("response.completed", { + response: { + id: "resp_missing_terminal_output", + model: "gpt-5.6-terra", + status: "completed", + }, + }), + ]), + { + validation: "public", + onComplete: () => {}, + onRecall: async () => ({ anchorText: "", resultText: "" }), + runFollowUp: async () => { + throw new Error("should not run"); + }, + }, + ); + + expect(await drain(client)).toContain("response.failed"); + }); + + test("Codex validation accepts queued creation and completed event with incomplete status", async () => { + const client = streamResponsesRecallAware( + streamFrom([ + sseEvent("response.created", { + response: { + id: "resp_codex_lifecycle", + model: "gpt-5.6-terra", + status: "queued", + output: [], + }, + }), + sseEvent("response.completed", { + response: { + id: "resp_codex_lifecycle", + model: "gpt-5.6-terra", + status: "incomplete", + incomplete_details: { reason: "max_output_tokens" }, + }, + }), + ]), + { + validation: "codex", + onComplete: () => {}, + onRecall: async () => ({ anchorText: "", resultText: "" }), + runFollowUp: async () => { + throw new Error("should not run"); + }, + }, + ); + + const output = await drain(client); + expect(output).toContain("event: response.completed"); + expect(output).toContain('"status":"incomplete"'); + expect(output).not.toContain("event: response.failed"); + }); + test("rejects terminal function calls changing the streamed tool name", async () => { const client = streamResponsesRecallAware( streamFrom([ @@ -979,6 +1169,55 @@ describe("streamResponsesRecallAware", () => { }, ); + test("rejects recursive explicit null replacement in the terminal snapshot", async () => { + let completedResponse: GatewayResponse | undefined; + const item = { + type: "image_generation_call", + id: "image_terminal_null", + status: "completed", + result: null, + details: { revised_prompt: null }, + }; + const terminalItem = { + ...item, + result: "base64-result", + details: { revised_prompt: "cat" }, + }; + const client = streamResponsesRecallAware( + streamFrom([ + created("resp_terminal_null", "gpt-5.6-terra"), + sseEvent("response.output_item.added", { + output_index: 0, + item: { ...item, status: "generating" }, + }), + sseEvent("response.output_item.done", { + output_index: 0, + item, + }), + sseEvent("response.completed", { + response: { + id: "resp_terminal_null", + model: "gpt-5.6-terra", + status: "completed", + output: [terminalItem], + }, + }), + ]), + { + onComplete: (response) => { + completedResponse = response; + }, + onRecall: async () => ({ anchorText: "", resultText: "" }), + runFollowUp: async () => { + throw new Error("should not run"); + }, + }, + ); + + expect(await drain(client)).toContain("response.failed"); + expect(completedResponse?.rawOutputItems).toEqual([item]); + }); + test("never forwards response-side item_reference lifecycle events", async () => { let completedResponse: GatewayResponse | undefined; const client = streamResponsesRecallAware( @@ -2604,7 +2843,7 @@ describe("streamResponsesRecallAware", () => { expect(recalled).toBe(1); }); - test("preserves continuation terminal and item metadata", async () => { + test("preserves content_filter continuation terminal and item metadata", async () => { const citation = { type: "url_citation", start_index: 0, @@ -2620,7 +2859,7 @@ describe("streamResponsesRecallAware", () => { id: "resp_terminal_metadata_followup", model: "gpt-5.6-terra", status: "incomplete", - incomplete_details: { reason: "max_output_tokens" }, + incomplete_details: { reason: "content_filter" }, output: [ { type: "message", @@ -2656,7 +2895,8 @@ describe("streamResponsesRecallAware", () => { ); const output = await drain(client); - expect(output).toContain('"reason":"max_output_tokens"'); + expect(output).toContain('"reason":"content_filter"'); + expect(output).toContain('"status":"incomplete"'); expect(output).toContain("https://example.com/lore"); expect(output).not.toContain(PUBLIC_RECALL_ERROR); }); @@ -3776,10 +4016,103 @@ describe("streamResponsesRecallAware", () => { const out = await drain(client); expect(out).toContain(PUBLIC_RECALL_ERROR); expect(out).not.toContain("intermediate"); + expect(out).not.toContain('"name":"recall"'); + expect(out).not.toContain("detail"); + expect(out).not.toContain("call_second"); expect(JSON.stringify(completedResponse)).not.toContain("intermediate"); expect(JSON.stringify(completedResponse)).not.toContain("lore-recall"); }); + test("commits deferred recall persistence after a successful continuation", async () => { + let committed = 0; + let rolledBack = 0; + let transaction: { commit: () => void; rollback: () => void } | undefined; + const client = streamResponsesRecallAware( + streamFrom([ + created("resp_commit_success", "gpt-5.6-terra"), + recallCall(0, { query: "architecture" }), + completed("resp_commit_success"), + ]), + { + onComplete: () => {}, + onTransactionReady: (ready) => { + transaction = ready; + }, + onRecall: async () => ({ + anchorText: buildAnchor("architecture"), + resultText: "results", + commit: () => committed++, + rollback: () => rolledBack++, + }), + runFollowUp: async () => { + const body = streamFrom([ + created("resp_commit_followup", "gpt-5.6-terra"), + textItem(0, "answer", "msg_commit_answer"), + completed("resp_commit_followup"), + ]).body; + if (!body) throw new Error("expected continuation body"); + return { reader: body.getReader() }; + }, + }, + ); + + const out = await drain(client); + expect(out).toContain("answer"); + expect(out.match(/^event: response\.completed$/gm)).toHaveLength(1); + expect(transaction).toBeDefined(); + expect(committed).toBe(0); + expect(rolledBack).toBe(0); + transaction?.commit(); + expect(committed).toBe(1); + expect(rolledBack).toBe(0); + }); + + test("rolls back every recall mutation when a commit callback fails", async () => { + let persisted = 0; + let rolledBack = 0; + let transaction: { commit: () => void; rollback: () => void } | undefined; + const client = streamResponsesRecallAware( + streamFrom([ + created("resp_commit_failure", "gpt-5.6-terra"), + recallCall(0, { query: "architecture" }), + completed("resp_commit_failure"), + ]), + { + onComplete: () => {}, + onTransactionReady: (ready) => { + transaction = ready; + }, + onRecall: async () => ({ + anchorText: buildAnchor("architecture"), + resultText: "results", + commit: () => { + persisted++; + throw new Error("recall persistence failed"); + }, + rollback: () => { + persisted--; + rolledBack++; + }, + }), + runFollowUp: async () => { + const body = streamFrom([ + created("resp_commit_failure_followup", "gpt-5.6-terra"), + textItem(0, "answer", "msg_commit_failure_answer"), + completed("resp_commit_failure_followup"), + ]).body; + if (!body) throw new Error("expected continuation body"); + return { reader: body.getReader() }; + }, + }, + ); + + expect(await drain(client)).toContain("answer"); + expect(transaction).toBeDefined(); + expect(() => transaction?.commit()).toThrow("recall persistence failed"); + expect(persisted).toBe(0); + expect(rolledBack).toBe(1); + }); + test("rolls back deferred persistence when a recall-only continuation fails", async () => { let committed = 0; let rolledBack = 0; @@ -4536,11 +4869,13 @@ describe("streamResponsesRecallAware", () => { response: { id: "resp_followup_failure", status: "failed", + usage: { input_tokens: 1_000, output_tokens: 100 }, error: { message: "provider failed" }, }, }), ]); - let completedResponse: unknown; + let completedResponse: GatewayResponse | undefined; + let successful: boolean | undefined; const client = streamResponsesRecallAware( streamFrom([ created("resp_failure", "gpt-5.6-terra"), @@ -4548,8 +4883,9 @@ describe("streamResponsesRecallAware", () => { completed("resp_failure"), ]), { - onComplete: (response) => { + onComplete: (response, didSucceed) => { completedResponse = response; + successful = didSucceed; }, onRecall: async ({ query }) => ({ anchorText: buildAnchor(query), @@ -4568,6 +4904,11 @@ describe("streamResponsesRecallAware", () => { expect(out).not.toContain("lore-recall"); expect(out).not.toContain("partial answer"); expect(JSON.stringify(completedResponse)).not.toContain("partial answer"); + expect(completedResponse?.usage).toMatchObject({ + inputTokens: 1_000, + outputTokens: 100, + }); + expect(successful).toBe(false); expect(out).not.toContain("response.completed"); }); @@ -4655,6 +4996,9 @@ describe("streamResponsesRecallAware", () => { }); test("preserves response.incomplete from the continuation", async () => { + const outcomes: boolean[] = []; + let commits = 0; + let rollbacks = 0; const followUp = streamFrom([ created("resp_followup", "gpt-5.6-terra"), textItem(0, "partial"), @@ -4667,10 +5011,12 @@ describe("streamResponsesRecallAware", () => { completed("resp_first"), ]), { - onComplete: () => {}, + onComplete: (_response, successful) => outcomes.push(successful), onRecall: async () => ({ anchorText: buildAnchor("architecture"), resultText: "results", + commit: () => commits++, + rollback: () => rollbacks++, }), runFollowUp: async () => ({ reader: followUp.body!.getReader() }), }, @@ -4679,6 +5025,9 @@ describe("streamResponsesRecallAware", () => { const out = await drain(client); expect(out.match(/^event: response\.incomplete$/gm)).toHaveLength(1); expect(out.match(/^event: response\.completed$/gm) ?? []).toHaveLength(0); + expect(outcomes).toEqual([false]); + expect(commits).toBe(0); + expect(rollbacks).toBe(1); }); test("never executes recall from an incomplete principal", async () => { @@ -4737,6 +5086,7 @@ describe("streamResponsesRecallAware", () => { }); test("maps response.done with incomplete status to response.incomplete", async () => { + const outcomes: boolean[] = []; const followUp = streamFrom([ created("resp_followup", "gpt-5.6-terra"), textItem(0, "partial"), @@ -4749,7 +5099,7 @@ describe("streamResponsesRecallAware", () => { completed("resp_first"), ]), { - onComplete: () => {}, + onComplete: (_response, successful) => outcomes.push(successful), onRecall: async () => ({ anchorText: buildAnchor("architecture"), resultText: "results", @@ -4761,6 +5111,7 @@ describe("streamResponsesRecallAware", () => { const out = await drain(client); expect(out.match(/^event: response\.incomplete$/gm)).toHaveLength(1); expect(out).not.toContain('"type":"response.completed"'); + expect(outcomes).toEqual([false]); }); test("retains reasoning items in the rebuilt terminal", async () => { diff --git a/packages/gateway/test/openai-responses-stream.test.ts b/packages/gateway/test/openai-responses-stream.test.ts index d13440f0..2f644265 100644 --- a/packages/gateway/test/openai-responses-stream.test.ts +++ b/packages/gateway/test/openai-responses-stream.test.ts @@ -11,6 +11,11 @@ import { describe, test, expect, vi } from "vitest"; import { accumulateResponsesSSEStream, + isSupportedResponsesOutputItemType, + isValidResponsesOutputItemStatus, + responsesDoneItemMatchesAdded, + responsesTerminalItemMatches, + SUPPORTED_RESPONSES_OUTPUT_ITEM_TYPES, streamResponsesPassthrough, translateAnthropicStreamToResponses, } from "../src/stream/openai-responses"; @@ -1232,6 +1237,642 @@ describe("accumulateResponsesSSEStream", () => { }, ); + test.each(["public", "codex"] as const)( + "%s rejects a terminal snapshot changing hosted-tool semantics", + async (validation) => { + const itemId = `ws-snapshot-${validation}`; + const reference = validation === "public" ? { output_index: 0 } : {}; + const item = { + type: "web_search_call", + id: itemId, + status: "completed", + action: { type: "search", query: "good" }, + }; + const addedItem = { ...item, status: "in_progress" }; + await expect( + accumulateResponsesSSEStream( + buildSSEResponse([ + { + event: "response.output_item.added", + data: { ...reference, item: addedItem }, + }, + { + event: "response.output_item.done", + data: { ...reference, item }, + }, + { + event: "response.completed", + data: { + response: { + status: "completed", + output: [ + { + ...item, + action: { type: "search", query: "EVIL" }, + }, + ], + }, + }, + }, + ]), + { validation, stopAtTerminal: true }, + ), + ).rejects.toThrow("malformed Responses terminal event"); + }, + ); + + test.each(["public", "codex"] as const)( + "%s rejects output_item.done changing hosted-tool semantics", + async (validation) => { + const item = { + type: "web_search_call", + id: `ws-done-${validation}`, + action: { type: "search", query: "good" }, + }; + await expect( + accumulateResponsesSSEStream( + buildSSEResponse([ + { + event: "response.output_item.added", + data: { + ...(validation === "public" ? { output_index: 0 } : {}), + item, + }, + }, + { + event: "response.output_item.done", + data: { + ...(validation === "public" ? { output_index: 0 } : {}), + item: { + ...item, + action: { type: "search", query: "EVIL" }, + }, + }, + }, + ]), + { validation, stopAtTerminal: true }, + ), + ).rejects.toThrow("malformed Responses stream event"); + }, + ); + + test.each(["public", "codex"] as const)( + "%s rejects terminal mutation of an established extension field", + async (validation) => { + const item = { + type: "function_call", + id: `fc-extension-${validation}`, + call_id: `call-extension-${validation}`, + name: "lookup", + arguments: "{}", + caller: "trusted", + }; + await expect( + accumulateResponsesSSEStream( + buildSSEResponse([ + { + event: "response.output_item.added", + data: { + ...(validation === "public" ? { output_index: 0 } : {}), + item, + }, + }, + { + event: "response.output_item.done", + data: { + ...(validation === "public" ? { output_index: 0 } : {}), + item, + }, + }, + { + event: "response.completed", + data: { + response: { + status: "completed", + output: [{ ...item, caller: "attacker" }], + }, + }, + }, + ]), + { validation, stopAtTerminal: true }, + ), + ).rejects.toThrow("malformed Responses terminal event"); + }, + ); + + test.each(["public", "codex"] as const)( + "%s accepts terminal status enrichment for a hosted-tool item", + async (validation) => { + const item = { + type: "web_search_call", + id: `ws-enriched-${validation}`, + action: { type: "search", query: "lore" }, + }; + await expect( + accumulateResponsesSSEStream( + buildSSEResponse([ + { + event: "response.output_item.added", + data: { + ...(validation === "public" ? { output_index: 0 } : {}), + item, + }, + }, + { + event: "response.output_item.done", + data: { + ...(validation === "public" ? { output_index: 0 } : {}), + item, + }, + }, + { + event: "response.completed", + data: { + response: { + status: "completed", + output: [{ ...item, status: "completed" }], + }, + }, + }, + ]), + { validation, stopAtTerminal: true }, + ), + ).resolves.toMatchObject({ + rawOutputItems: [{ ...item, status: "completed" }], + }); + }, + ); + + test.each(["public", "codex"] as const)( + "%s rejects recursive explicit null replacement in the terminal snapshot", + async (validation) => { + const item = { + type: "image_generation_call", + id: `image-null-${validation}`, + status: "completed", + result: null, + details: { revised_prompt: null }, + }; + const terminalItem = { + ...item, + result: "base64-result", + details: { revised_prompt: "cat" }, + }; + await expect( + accumulateResponsesSSEStream( + buildSSEResponse([ + { + event: "response.output_item.added", + data: { + ...(validation === "public" ? { output_index: 0 } : {}), + item: { ...item, status: "generating" }, + }, + }, + { + event: "response.output_item.done", + data: { + ...(validation === "public" ? { output_index: 0 } : {}), + item, + }, + }, + { + event: "response.completed", + data: { + response: { status: "completed", output: [terminalItem] }, + }, + }, + ]), + { validation, stopAtTerminal: true }, + ), + ).rejects.toThrow("malformed Responses terminal event"); + }, + ); + + test.each(["public", "codex"] as const)( + "%s accepts hosted-tool status transitions", + async (validation) => { + const item = { + type: "image_generation_call", + id: `image-status-${validation}`, + }; + await expect( + accumulateResponsesSSEStream( + buildSSEResponse([ + { + event: "response.output_item.added", + data: { + ...(validation === "public" ? { output_index: 0 } : {}), + item: { ...item, status: "generating" }, + }, + }, + { + event: "response.output_item.done", + data: { + ...(validation === "public" ? { output_index: 0 } : {}), + item: { ...item, status: "failed" }, + }, + }, + { + event: "response.completed", + data: { + response: { + status: "completed", + output: [{ ...item, status: "failed" }], + }, + }, + }, + ]), + { validation, stopAtTerminal: true }, + ), + ).resolves.toMatchObject({ + rawOutputItems: [{ ...item, status: "failed" }], + }); + }, + ); + + test.each(["public", "codex"] as const)( + "%s accepts a failed function-call companion", + async (validation) => { + const item = { + type: "function_call", + id: `fc-failed-${validation}`, + call_id: `call-failed-${validation}`, + name: "lookup", + arguments: "{}", + }; + const result = await accumulateResponsesSSEStream( + buildSSEResponse([ + { + event: "response.output_item.added", + data: { + ...(validation === "public" ? { output_index: 0 } : {}), + item: { ...item, status: "in_progress" }, + }, + }, + { + event: "response.output_item.done", + data: { + ...(validation === "public" ? { output_index: 0 } : {}), + item: { ...item, status: "failed" }, + }, + }, + { + event: "response.completed", + data: { + response: { + status: "completed", + output: [{ ...item, status: "failed" }], + }, + }, + }, + ]), + { validation, stopAtTerminal: true }, + ); + + expect(result.rawOutputItems).toEqual([{ ...item, status: "failed" }]); + }, + ); + + test.each(["public", "codex"] as const)( + "%s rejects unknown output item types", + async (validation) => { + await expect( + accumulateResponsesSSEStream( + buildSSEResponse([ + { + event: "response.output_item.added", + data: { + ...(validation === "public" ? { output_index: 0 } : {}), + item: { type: "provider_specific_output", id: "unknown" }, + }, + }, + ]), + { validation, stopAtTerminal: true }, + ), + ).rejects.toThrow("malformed Responses stream event"); + }, + ); + + test("keeps the standard output-item allowlist exhaustive", () => { + const expected = [ + "message", + "function_call", + "function_call_output", + "reasoning", + "item_reference", + "web_search_call", + "file_search_call", + "computer_call", + "computer_call_output", + "computer_tool_call", + "computer_tool_call_output", + "code_interpreter_call", + "image_generation_call", + "local_shell_call", + "local_shell_call_output", + "shell_call", + "shell_call_output", + "mcp_call", + "mcp_list_tools", + "mcp_approval_request", + "mcp_approval_response", + "custom_tool_call", + "custom_tool_call_output", + "apply_patch_call", + "apply_patch_call_output", + "program", + "program_output", + "tool_search_call", + "tool_search_output", + "additional_tools", + "compaction", + ]; + + expect(SUPPORTED_RESPONSES_OUTPUT_ITEM_TYPES).toEqual(expected); + expect(expected.every(isSupportedResponsesOutputItemType)).toBe(true); + expect(isSupportedResponsesOutputItemType("provider_specific_output")).toBe( + false, + ); + }); + + test("only permits sparse added fields to be extended by output_item.done", () => { + expect( + responsesDoneItemMatchesAdded( + { + type: "function_call", + id: "fc_sparse", + call_id: "call_sparse", + name: "lookup", + arguments: '{"value":1}', + }, + { + type: "function_call", + id: "fc_sparse", + call_id: "call_sparse", + name: "lookup", + arguments: "", + }, + ), + ).toBe(true); + expect( + responsesDoneItemMatchesAdded( + { + type: "image_generation_call", + id: "image_sparse", + result: "base64-result", + }, + { + type: "image_generation_call", + id: "image_sparse", + result: null, + }, + ), + ).toBe(true); + expect( + responsesDoneItemMatchesAdded( + { + type: "function_call", + id: "fc_changed", + call_id: "call_changed", + name: "lookup", + arguments: "evil", + }, + { + type: "function_call", + id: "fc_changed", + call_id: "call_changed", + name: "lookup", + arguments: "good", + }, + ), + ).toBe(false); + for (const [type, field, partType] of [ + ["message", "content", "output_text"], + ["reasoning", "summary", "summary_text"], + ] as const) { + expect( + responsesDoneItemMatchesAdded( + { + type, + id: `${type}_changed`, + [field]: [{ type: partType, text: "evil" }], + }, + { + type, + id: `${type}_changed`, + [field]: [{ type: partType, text: "good" }], + }, + ), + ).toBe(false); + } + }); + + test("rejects terminal null replacement but permits sparse enrichment", () => { + expect( + responsesTerminalItemMatches( + { + type: "image_generation_call", + id: "image_terminal_sparse", + status: "completed", + result: "base64-result", + details: { revised_prompt: "cat" }, + }, + { + type: "image_generation_call", + id: "image_terminal_sparse", + status: "completed", + result: null, + details: { revised_prompt: null }, + }, + ), + ).toBe(false); + expect( + responsesTerminalItemMatches( + { + type: "image_generation_call", + id: "image_terminal_sparse", + status: "completed", + result: "base64-result", + details: { revised_prompt: "cat" }, + }, + { + type: "image_generation_call", + id: "image_terminal_sparse", + status: "completed", + result: undefined, + details: {}, + }, + ), + ).toBe(true); + expect( + responsesTerminalItemMatches( + { + type: "image_generation_call", + id: "image_terminal_changed", + status: "completed", + result: "evil", + }, + { + type: "image_generation_call", + id: "image_terminal_changed", + status: "completed", + result: "good", + }, + ), + ).toBe(false); + }); + + test("validates type-specific hosted-tool statuses", () => { + expect( + isValidResponsesOutputItemStatus("web_search_call", "searching", "added"), + ).toBe(true); + expect( + isValidResponsesOutputItemStatus( + "code_interpreter_call", + "interpreting", + "added", + ), + ).toBe(true); + expect( + isValidResponsesOutputItemStatus( + "image_generation_call", + "generating", + "added", + ), + ).toBe(true); + expect( + isValidResponsesOutputItemStatus("web_search_call", "banana", "added"), + ).toBe(false); + expect( + isValidResponsesOutputItemStatus( + "image_generation_call", + "searching", + "added", + ), + ).toBe(false); + expect( + isValidResponsesOutputItemStatus( + "tool_search_call", + "searching", + "added", + ), + ).toBe(false); + expect( + isValidResponsesOutputItemStatus( + "function_call_output", + "failed", + "terminal", + ), + ).toBe(false); + expect( + isValidResponsesOutputItemStatus( + "function_call_output", + "completed", + "terminal", + ), + ).toBe(true); + expect( + isValidResponsesOutputItemStatus("program", "failed", "terminal"), + ).toBe(false); + expect( + isValidResponsesOutputItemStatus( + "image_generation_call", + "generating", + "done", + ), + ).toBe(false); + expect( + isValidResponsesOutputItemStatus("mcp_list_tools", "completed", "done"), + ).toBe(false); + expect( + isValidResponsesOutputItemStatus( + "file_search_call", + "incomplete", + "done", + ), + ).toBe(true); + expect( + isValidResponsesOutputItemStatus( + "code_interpreter_call", + "incomplete", + "done", + ), + ).toBe(true); + expect( + isValidResponsesOutputItemStatus( + "tool_search_call", + "incomplete", + "done", + ), + ).toBe(true); + expect( + isValidResponsesOutputItemStatus("tool_search_call", "failed", "done"), + ).toBe(false); + expect( + isValidResponsesOutputItemStatus("custom_tool_call", "failed", "done"), + ).toBe(false); + }); + + test.each([ + ["role", "user"], + ["status", "in_progress"], + ] as const)( + "public rejects a terminal message with changed %s", + async (field, value) => { + const itemId = `msg-changed-${field}`; + await expect( + accumulateResponsesSSEStream( + buildSSEResponse([ + { + event: "response.output_item.added", + data: { + output_index: 0, + item: { + type: "message", + id: itemId, + role: "assistant", + status: "in_progress", + }, + }, + }, + { + event: "response.output_item.done", + data: { + output_index: 0, + item: { + type: "message", + id: itemId, + role: "assistant", + status: "completed", + content: [], + }, + }, + }, + { + event: "response.completed", + data: { + response: { + status: "completed", + output: [ + { + type: "message", + id: itemId, + role: "assistant", + status: "completed", + content: [], + [field]: value, + }, + ], + }, + }, + }, + ]), + { validation: "public", stopAtTerminal: true }, + ), + ).rejects.toThrow("malformed Responses terminal event"); + }, + ); + test.each(["public", "codex"] as const)( "%s requires a present terminal output snapshot to be a one-to-one complete mapping", async (validation) => { @@ -1256,6 +1897,8 @@ describe("accumulateResponsesSSEStream", () => { [], [{ type: "message", id: "", content: [] }], [{ type: "item_reference", id: "unknown-item" }], + [{ type: "item_reference", id: itemId, content: [] }], + [{ type: "message", id: itemId }], [ { type: "item_reference", id: itemId }, { type: "item_reference", id: itemId }, @@ -2292,6 +2935,38 @@ test("Anthropic translator emits inclusive Responses cache usage", async () => { ).not.toThrow(); }); +test("Anthropic translator emits content_filter as response.incomplete", async () => { + const event = (type: string, data: Record) => + `event: ${type}\ndata: ${JSON.stringify({ type, ...data })}\n\n`; + const upstream = new Response( + event("message_start", { + message: { + id: "msg_filtered", + type: "message", + role: "assistant", + model: "claude-test", + content: [], + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 10, output_tokens: 0 }, + }, + }) + + event("message_delta", { + delta: { stop_reason: "refusal", stop_sequence: null }, + usage: { output_tokens: 1 }, + }) + + event("message_stop", {}), + ); + + const output = await translateAnthropicStreamToResponses(upstream, { + strict: true, + }).text(); + expect(output).toContain("event: response.incomplete"); + expect(output).toContain('"status":"incomplete"'); + expect(output).toContain('"reason":"content_filter"'); + expect(output).not.toContain("event: response.completed"); +}); + // --------------------------------------------------------------------------- // streamResponsesPassthrough — true streaming (Responses → Responses client) // --------------------------------------------------------------------------- @@ -2520,14 +3195,16 @@ describe("streamResponsesPassthrough", () => { }); test("does not forward malformed JSON", async () => { + const outcomes: boolean[] = []; const output = await streamResponsesPassthrough( new Response("event: response.created\ndata: {bad}\n\n"), - () => {}, + (_response, successful) => outcomes.push(successful), undefined, "public", ).text(); expect(output).not.toContain("{bad}"); expect(output).toContain("event: response.failed"); + expect(outcomes).toEqual([false]); }); test("rejects duplicate call_id before forwarding the conflicting event", async () => { @@ -2595,6 +3272,7 @@ describe("streamResponsesPassthrough", () => { }); test("validates and forwards a provider failure terminal", async () => { + const outcomes: boolean[] = []; const output = await streamResponsesPassthrough( buildSSEResponse([ { @@ -2608,12 +3286,33 @@ describe("streamResponsesPassthrough", () => { }, }, ]), - () => {}, + (_response, successful) => outcomes.push(successful), undefined, "public", ).text(); expect(output.match(/event: response\.failed/g)).toHaveLength(1); expect(output).toContain("provider failed"); + expect(outcomes).toEqual([false]); + }); + + test("reports a missing terminal as unsuccessful", async () => { + const outcomes: boolean[] = []; + const output = await streamResponsesPassthrough( + buildSSEResponse([ + { + event: "response.created", + data: { + type: "response.created", + response: { id: "missing-terminal", status: "in_progress" }, + }, + }, + ]), + (_response, successful) => outcomes.push(successful), + undefined, + "public", + ).text(); + expect(output).toContain("event: response.failed"); + expect(outcomes).toEqual([false]); }); test("counts comment-only wire bytes toward the aggregate cap", async () => { @@ -2623,7 +3322,7 @@ describe("streamResponsesPassthrough", () => { undefined, "public", ).text(); - expect(output).toContain("exceeded aggregate byte limit"); + expect(output).toContain("Upstream response stream failed"); }); test("fails the stream when aggregate retained event data exceeds its cap", async () => { @@ -2636,7 +3335,7 @@ describe("streamResponsesPassthrough", () => { const output = await streamResponsesPassthrough(upstream, () => {}).text(); expect(output).toContain("event: response.failed"); - expect(output).toContain("exceeded aggregate byte limit"); + expect(output).toContain("Upstream response stream failed"); }); test("stops pulling upstream while a downstream consumer is not reading", async () => { @@ -2745,6 +3444,46 @@ describe("streamResponsesPassthrough", () => { expect(done.stopReason).toBe("end_turn"); }); + test("finalizes when the client cancels immediately after the terminal event", async () => { + const upstream = controllableSSE(); + let completeCalls = 0; + const client = streamResponsesPassthrough(upstream.response, () => { + completeCalls++; + }); + if (!client.body) throw new Error("test response has no body"); + const reader = client.body.getReader(); + const decoder = new TextDecoder(); + + upstream.push("response.created", { + type: "response.created", + response: { + id: "resp_cancel_after_terminal", + model: "gpt-5.6-sol", + status: "in_progress", + }, + }); + upstream.push("response.completed", { + type: "response.completed", + response: { + id: "resp_cancel_after_terminal", + model: "gpt-5.6-sol", + status: "completed", + output: [], + usage: { input_tokens: 1, output_tokens: 0 }, + }, + }); + + let seen = ""; + while (!seen.includes("event: response.completed")) { + const { done, value } = await reader.read(); + if (done) throw new Error("stream closed before terminal event"); + if (value) seen += decoder.decode(value, { stream: true }); + } + await reader.cancel(); + + expect(completeCalls).toBe(1); + }); + test("forwards every upstream event verbatim, preserving non-accumulated fields", async () => { const upstream = controllableSSE(); const clientResp = streamResponsesPassthrough(upstream.response, () => {}); @@ -2848,6 +3587,7 @@ describe("streamResponsesPassthrough", () => { const out = await drainToString(clientResp); // Client is told the turn failed rather than hanging on a missing terminal. expect(out).toContain("response.failed"); + expect(out).not.toContain("upstream exploded"); // onComplete still ran (so postResponse/cost tracking is not skipped)… expect(completed).not.toBeNull(); // …and exactly once (the `completed` guard must not double-fire). diff --git a/packages/gateway/test/openai-responses.test.ts b/packages/gateway/test/openai-responses.test.ts index 99bbb396..b32d9bbe 100644 --- a/packages/gateway/test/openai-responses.test.ts +++ b/packages/gateway/test/openai-responses.test.ts @@ -1083,6 +1083,24 @@ describe("buildOpenAIResponsesResponse", () => { expect(output[1].arguments).toBe('{"query":"cats"}'); }); + test("non-streaming: preserves validated native output items", async () => { + const rawOutputItems = [ + { + type: "web_search_call", + id: "ws_abc", + status: "completed", + action: { type: "search", query: "cats" }, + }, + ]; + const response = buildOpenAIResponsesResponse( + { ...baseResponse, rawOutputItems }, + false, + ); + const body = (await response.json()) as Record; + + expect(body.output).toEqual(rawOutputItems); + }); + test("non-streaming: max_tokens maps to incomplete status", async () => { const resp: GatewayResponse = { ...baseResponse, @@ -1092,6 +1110,9 @@ describe("buildOpenAIResponsesResponse", () => { const response = buildOpenAIResponsesResponse(resp, false); const body = (await response.json()) as Record; expect(body.status).toBe("incomplete"); + expect(body.incomplete_details).toEqual({ + reason: "max_output_tokens", + }); }); test("non-streaming: id gets resp_ prefix if missing", async () => { @@ -1124,6 +1145,37 @@ describe("buildOpenAIResponsesResponse", () => { expect(text).toContain("Hello! How can I help?"); }); + test("streaming: max_tokens emits an incomplete terminal", async () => { + const response = buildOpenAIResponsesResponse( + { ...baseResponse, stopReason: "max_tokens" }, + true, + ); + const text = await response.text(); + + expect(text).toContain("event: response.incomplete"); + expect(text).toContain('"status":"incomplete"'); + expect(text).toContain('"reason":"max_output_tokens"'); + expect(text).not.toContain("event: response.completed"); + }); + + test.each([false, true])( + "stream=%s: content_filter remains an incomplete terminal", + async (streaming) => { + const response = buildOpenAIResponsesResponse( + { ...baseResponse, stopReason: "content_filter" }, + streaming, + ); + const text = await response.text(); + + expect(text).toContain('"status":"incomplete"'); + expect(text).toContain('"reason":"content_filter"'); + if (streaming) { + expect(text).toContain("event: response.incomplete"); + expect(text).not.toContain("event: response.completed"); + } + }, + ); + test("streaming: tool_use produces function_call events", async () => { const resp: GatewayResponse = { id: "test", diff --git a/packages/gateway/test/openrouter-provider-routing.test.ts b/packages/gateway/test/openrouter-provider-routing.test.ts index a50e6549..dcc8ca03 100644 --- a/packages/gateway/test/openrouter-provider-routing.test.ts +++ b/packages/gateway/test/openrouter-provider-routing.test.ts @@ -14,6 +14,7 @@ import { upstreamFetch } from "../src/fetch"; import { setSessionAuth } from "../src/auth"; import type { GatewayConfig } from "../src/config"; import { fetchArgUrl } from "./helpers/fetch-url"; +import { loopbackRequest } from "./helpers/loopback-request"; vi.mock("../src/fetch", () => ({ upstreamFetch: vi.fn() })); @@ -264,6 +265,19 @@ async function stop(): Promise { afterEach(stop); +function localRequest( + run: Started, + path: string, + headers: Record, + body: string, +): Promise { + return loopbackRequest(`${run.baseURL}${path}`, { + method: "POST", + headers, + body, + }); +} + function requestHeaders( providerID: string, upstreamUrl = SESSION_UPSTREAM, @@ -309,19 +323,17 @@ async function foreground( upstreamUrl?: string; } = {}, ): Promise { - return fetch(`${run.baseURL}/v1/chat/completions`, { - method: "POST", - headers: requestHeaders( - options.providerID ?? "openrouter", - options.upstreamUrl, - ), - body: JSON.stringify( + return localRequest( + run, + "/v1/chat/completions", + requestHeaders(options.providerID ?? "openrouter", options.upstreamUrl), + JSON.stringify( chatBody(options.provider, { includeProvider: options.includeProvider, message: options.message, }), ), - }); + ); } function workerCalls() { @@ -353,17 +365,22 @@ async function useRecallForegroundInterceptor(): Promise { ); } -function activeSession() { - // Fixed x-lore-session-id makes this deterministic; no polling or fuzzy match. +function sessionByHeader(headerSessionId: string) { return import("../src/pipeline").then(({ getActiveSessions }) => { const state = [...getActiveSessions().values()].find( - (candidate) => candidate.headerSessionId === SESSION_ID, + (candidate) => candidate.headerSessionId === headerSessionId, ); - if (!state) throw new Error("fixed test session was not identified"); + if (!state) + throw new Error(`test session ${headerSessionId} was not identified`); return state; }); } +function activeSession() { + // Fixed x-lore-session-id makes this deterministic; no polling or fuzzy match. + return sessionByHeader(SESSION_ID); +} + describe("OpenRouter provider routing", () => { test.each([ ["explicit null", null], @@ -394,14 +411,15 @@ describe("OpenRouter provider routing", () => { await useNormalForegroundInterceptor(); const minimaxHeaders = requestHeaders("minimax"); delete minimaxHeaders["x-lore-upstream-url"]; - const responses = await fetch(`${run.baseURL}/v1/responses`, { - method: "POST", - headers: minimaxHeaders, - body: JSON.stringify({ + const responses = await localRequest( + run, + "/v1/responses", + minimaxHeaders, + JSON.stringify({ model: "anthropic/claude-sonnet-4-6", input: "Responses ingress to Anthropic route", }), - }); + ); expect(responses.ok).toBe(true); await responses.text(); expect((await activeSession()).lastUpstream).toMatchObject({ @@ -414,11 +432,12 @@ describe("OpenRouter provider routing", () => { delete copilotHeaders["x-lore-provider"]; delete copilotHeaders["x-lore-upstream-url"]; copilotHeaders["copilot-integration-id"] = "copilot-cli"; - const copilot = await fetch(`${run.baseURL}/v1/chat/completions`, { - method: "POST", - headers: copilotHeaders, - body: JSON.stringify(chatBody(undefined, { includeProvider: false })), - }); + const copilot = await localRequest( + run, + "/v1/chat/completions", + copilotHeaders, + JSON.stringify(chatBody(undefined, { includeProvider: false })), + ); expect(copilot.ok).toBe(true); await copilot.text(); expect((await activeSession()).lastUpstream).toMatchObject({ @@ -639,7 +658,7 @@ describe("OpenRouter provider routing", () => { test("protocol-distinct Vertex aliases coexist, persist, and select compatibly", async () => { const run = await start(); await useNormalForegroundInterceptor(); - const gemini = await fetch( + const gemini = await loopbackRequest( `${run.baseURL}/v1beta/models/gemini-2.5-flash:generateContent`, { method: "POST", @@ -904,7 +923,7 @@ describe("OpenRouter provider routing", () => { } }); - test("captures failed tightening and rejects an older request that resumes before capture", async () => { + test("captures failed tightening and preserves newer policy after an older request resumes", async () => { const run = await start(); const { setBeforeUpstreamCaptureForTest, setUpstreamInterceptor } = await import("../src/pipeline"); @@ -949,11 +968,13 @@ describe("OpenRouter provider routing", () => { message: "old slow turn", }); await oldStarted; - await ( - await foreground(run, { provider: newPolicy, message: "new fast turn" }) - ).text(); + const newRequest = foreground(run, { + provider: newPolicy, + message: "new queued turn", + }); releaseOld(); await (await oldRequest).text(); + await (await newRequest).text(); const state = await activeSession(); expect(state.lastUpstream?.providerOptions).toEqual(newPolicy); @@ -972,6 +993,77 @@ describe("OpenRouter provider routing", () => { ); }); + test("publishes a successful provisional migration's exact route and policy", async () => { + const run = await start(); + await useNormalForegroundInterceptor(); + const alias = "provider-routing-migration-alias"; + const oldRoute = "https://old-route.invalid/openrouter"; + const newRoute = "https://new-route.invalid/openrouter"; + const oldPolicy = { only: ["old-provider"] }; + const newPolicy = { + only: ["new-provider"], + allow_fallbacks: false, + }; + const oldHeaders = requestHeaders("openrouter", oldRoute); + delete oldHeaders["x-lore-session-id"]; + oldHeaders["x-session-affinity"] = alias; + + const established = await localRequest( + run, + "/v1/chat/completions", + oldHeaders, + JSON.stringify(chatBody(oldPolicy)), + ); + expect(established.ok).toBe(true); + await established.text(); + const original = await sessionByHeader(alias); + + const migrated = await localRequest( + run, + "/v1/chat/completions", + { + ...requestHeaders("openrouter", newRoute), + "x-session-affinity": alias, + }, + JSON.stringify({ + ...chatBody(newPolicy), + model: "openai/gpt-5.6", + }), + ); + expect(migrated.ok).toBe(true); + await migrated.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const state = await activeSession(); + expect(state.sessionID).toBe(original.sessionID); + expect(state.lastUpstream).toMatchObject({ + url: newRoute, + protocol: "openai", + providerID: "openrouter", + model: "openai/gpt-5.6", + providerOptions: newPolicy, + }); + expect(state.upstreamByProvider.get("openrouter")).toBe(state.lastUpstream); + const persisted = loadSessionTracking(state.sessionID)?.lastUpstream; + if (!persisted) + throw new Error("migrated upstream state was not persisted"); + const envelope = JSON.parse(persisted) as { + lastUpstream: Record; + upstreamByProvider: Record>; + }; + expect(envelope.lastUpstream).toMatchObject({ + url: newRoute, + protocol: "openai", + providerID: "openrouter", + model: "openai/gpt-5.6", + providerOptions: newPolicy, + }); + expect(envelope.upstreamByProvider.openrouter).toEqual( + envelope.lastUpstream, + ); + }); + test("Codex neither forwards nor durably snapshots provider options", async () => { const run = await start(); const { setUpstreamInterceptor } = await import("../src/pipeline"); @@ -1014,6 +1106,7 @@ describe("OpenRouter provider routing", () => { test("bounds persisted provider history and rejects oversized policies", async () => { const run = await start(); await useNormalForegroundInterceptor(); + const { setUpstreamInterceptor } = await import("../src/pipeline"); for (let index = 0; index < 20; index++) { const response = await foreground(run, { @@ -1035,13 +1128,17 @@ describe("OpenRouter provider routing", () => { }; expect(Object.keys(envelope.upstreamByProvider)).toHaveLength(16); + let oversizedForwarded = false; + setUpstreamInterceptor(async () => { + oversizedForwarded = true; + return openAIResponse(); + }); const oversized = await foreground(run, { provider: { only: ["x".repeat(65 * 1024)] }, message: "oversized policy", }); expect(oversized.status).toBe(502); - expect(await oversized.text()).toContain( - "OpenRouter provider routing options exceed 65536 bytes", - ); + expect(await oversized.text()).toContain("Gateway request failed"); + expect(oversizedForwarded).toBe(false); }); }); diff --git a/packages/gateway/test/pipeline-slash.test.ts b/packages/gateway/test/pipeline-slash.test.ts index 11f7bb6a..3214454c 100644 --- a/packages/gateway/test/pipeline-slash.test.ts +++ b/packages/gateway/test/pipeline-slash.test.ts @@ -37,16 +37,16 @@ describe("Pipeline — /lore:* slash commands", () => { afterEach(() => harness?.teardown()); - it("intercepts /lore:amnesia:on and :off without forwarding upstream", async () => { + it("fails closed for session-less /lore:amnesia toggles", async () => { harness = await createHarness({ fixtures: [] }); const on = await harness.chat(slashBody("/lore:amnesia:on")); expect(on.status).toBe(200); - expect(await textOf(on)).toContain("Amnesia mode on"); + expect(await textOf(on)).toContain("Amnesia mode was not changed"); const off = await harness.chat(slashBody("/lore:amnesia:off")); expect(off.status).toBe(200); - expect(await textOf(off)).toContain("Amnesia mode off"); + expect(await textOf(off)).toContain("Amnesia mode was not changed"); }); it("handles /lore:warm:stop|keep|auto", async () => { @@ -63,19 +63,23 @@ describe("Pipeline — /lore:* slash commands", () => { ).toContain("Cache warming set to auto"); }); - it("handles global /lore:warm:off and /lore:warm:on (persisted toggle)", async () => { + it("rejects unauthenticated global warming controls", async () => { const { isWarmingEnabled } = await import("../src/cache-warmer"); harness = await createHarness({ fixtures: [] }); - - const off = await harness.chat(slashBody("/lore:warm:off")); - expect(off.status).toBe(200); - expect(await textOf(off)).toContain("Cache warming disabled globally"); - expect(isWarmingEnabled()).toBe(false); - - const on = await harness.chat(slashBody("/lore:warm:on")); - expect(on.status).toBe(200); - expect(await textOf(on)).toContain("Cache warming enabled globally"); - expect(isWarmingEnabled()).toBe(true); + const initial = isWarmingEnabled(); + + for (const command of [ + "/lore:warm:off", + "/lore:warm:on", + "/lore:warm:reset", + ]) { + const response = await harness.chat(slashBody(command)); + expect(response.status).toBe(200); + expect(await textOf(response)).toContain( + "Global cache warming was not changed", + ); + expect(isWarmingEnabled()).toBe(initial); + } }); it("returns a helpful error for an unknown /lore:* command", async () => { diff --git a/packages/gateway/test/pipeline-streaming.test.ts b/packages/gateway/test/pipeline-streaming.test.ts index e0875a2a..3f745b9c 100644 --- a/packages/gateway/test/pipeline-streaming.test.ts +++ b/packages/gateway/test/pipeline-streaming.test.ts @@ -8,20 +8,80 @@ * postResponse storage. */ import { describe, it, expect, afterEach, vi } from "vitest"; +import { + distillation, + db, + getDailyCostForDay, + ltm, + loadSessionTracking, + saveSessionTracking, + temporal, +} from "@loreai/core"; +import * as Sentry from "@sentry/bun"; import type { Harness } from "./helpers/harness"; import { createHarness } from "./helpers/harness"; import type { FixtureEntry } from "../src/recorder"; +import type { GatewayRequest } from "../src/translate/types"; + +vi.mock("@sentry/bun", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + startInactiveSpan: vi.fn(actual.startInactiveSpan), + }; +}); + +vi.mock("../src/worker-health", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getDegradationWarning: vi.fn(actual.getDegradationWarning), + }; +}); + import { + activePipelineRequestCountForTest, buildStreamingResponse, abortAwareDelay, completeBudgetThrottleDelay, createForegroundAbortScope, + detachedPipelineRequestCountForTest, + evictLiveSessionForTest, + getActiveSessions, + handleCompactEndpoint, handleRequest, + handleResponsesCompactEndpoint, + isPipelineSessionActiveForTest, + expireProvisionalHeaderMappingsForTest, mergeRecallUsage, + pendingPipelineSessionClaimCountForTest, + resetPipelineState, + scheduleStreamingPostResponseForTest, + setPipelinePreUpstreamPauseForTest, + setMaxActivePipelineRequestsForTest, + setMaxDetachedPipelineRequestsForTest, + setPipelineResetSettleTimeoutForTest, + setPipelineResetPauseForTest, + setBeforeUpstreamCaptureForTest, + setPostResponseStartObserverForTest, + setRecallPersistenceCommitObserverForTest, + setProvisionalFinalizerPauseForTest, + setStreamingPostResponseLimitsForTest, + setStreamingPostResponseWaitObserverForTest, setUpstreamInterceptor, + streamingPostResponsePendingForTest, validatedMetaStream, } from "../src/pipeline"; -import { loadConfig } from "../src/config"; +import { loadConfig as loadBaseConfig } from "../src/config"; +import { authFingerprint } from "../src/auth"; +import { getDegradationWarning } from "../src/worker-health"; +import { + clearAllCosts, + computeCallCost, + getCostRate, + getDailySpend, + getSessionCosts, +} from "../src/cost-tracker"; import { translateAnthropicStreamToOpenAI } from "../src/stream/openai"; import { translateAnthropicStreamToResponses } from "../src/stream/openai-responses"; import { translateAnthropicStreamToGemini } from "../src/stream/gemini"; @@ -33,7 +93,7 @@ import { } from "./helpers/fixtures"; function loadLocalConfig() { - const config = loadConfig(); + const config = loadBaseConfig(); config.remoteGateway = false; config.hostedMode = false; return config; @@ -93,6 +153,155 @@ function validAnthropicSSE(text = "meta ok"): Response { ); } +function responsesEvent(type: string, data: Record): string { + return `event: ${type}\ndata: ${JSON.stringify({ type, ...data })}\n\n`; +} + +function validResponsesSSE( + id: string, + text?: string, + usage: Record = { + input_tokens: 1, + output_tokens: text ? 1 : 0, + }, +): string { + const output = text + ? { + type: "message", + id: `msg_${id}`, + role: "assistant", + status: "completed", + content: [{ type: "output_text", text }], + } + : undefined; + return ( + responsesEvent("response.created", { + response: { id, model: "gpt-5.6-sol", status: "in_progress" }, + }) + + (output + ? responsesEvent("response.output_item.added", { + output_index: 0, + item: { + type: "message", + id: output.id, + role: "assistant", + status: "in_progress", + }, + }) + + responsesEvent("response.output_text.delta", { + output_index: 0, + item_id: output.id, + content_index: 0, + delta: text, + }) + + responsesEvent("response.output_text.done", { + output_index: 0, + item_id: output.id, + content_index: 0, + text, + }) + + responsesEvent("response.output_item.done", { + output_index: 0, + item: output, + }) + : "") + + responsesEvent("response.completed", { + response: { + id, + model: "gpt-5.6-sol", + status: "completed", + output: output ? [output] : [], + usage, + }, + }) + ); +} + +function recallResponsesSSE(id: string, query: string): string { + const item = { + type: "function_call", + id: `fc_${id}`, + call_id: `call_${id}`, + name: "recall", + arguments: JSON.stringify({ query }), + status: "completed", + }; + return ( + responsesEvent("response.created", { + response: { id, model: "gpt-5.6-sol", status: "in_progress" }, + }) + + responsesEvent("response.output_item.added", { + output_index: 0, + item: { ...item, arguments: "", status: "in_progress" }, + }) + + responsesEvent("response.function_call_arguments.done", { + output_index: 0, + item_id: item.id, + arguments: item.arguments, + }) + + responsesEvent("response.output_item.done", { + output_index: 0, + item, + }) + + responsesEvent("response.completed", { + response: { + id, + model: "gpt-5.6-sol", + status: "completed", + output: [item], + usage: { input_tokens: 10, output_tokens: 1 }, + }, + }) + ); +} + +function incompleteResponsesSSE(id: string): string { + return ( + responsesEvent("response.created", { + response: { id, model: "gpt-5.6-sol", status: "in_progress" }, + }) + + responsesEvent("response.incomplete", { + response: { + id, + model: "gpt-5.6-sol", + status: "incomplete", + incomplete_details: { reason: "max_output_tokens" }, + output: [], + usage: { input_tokens: 1, output_tokens: 0 }, + }, + }) + ); +} + +function makeResponsesRequest(input: { + sessionHeaders: Record; + messages?: GatewayRequest["messages"]; + tools?: GatewayRequest["tools"]; +}): GatewayRequest { + return { + protocol: "openai-responses", + model: "gpt-5.6-sol", + system: "You are a coding agent.", + messages: input.messages ?? [ + { role: "user", content: [{ type: "text", text: "continue" }] }, + ], + tools: input.tools ?? [ + { name: "read", description: "Read a file", inputSchema: {} }, + ], + stream: true, + maxTokens: 1024, + metadata: {}, + rawHeaders: { + authorization: "Bearer test-key", + "x-lore-agent": "coder", + "x-lore-project": process.cwd(), + "x-lore-provider": "openai", + "x-lore-upstream-url": "https://api.openai.com/v1", + ...input.sessionHeaders, + }, + }; +} + describe("non-stream recall usage aggregation", () => { it("rejects per-field and cross-component safe-integer overflow", () => { expect(() => @@ -108,6 +317,233 @@ describe("non-stream recall usage aggregation", () => { ), ).toThrow("recall usage token overflow"); }); + + it.each(["caller", "upstream"] as const)( + "does not run marker fallback post-processing after %s follow-up cancellation", + async (abortSource) => { + const caller = new AbortController(); + let postResponses = 0; + let call = 0; + let followUpStarted!: () => void; + const started = new Promise((resolve) => { + followUpStarted = resolve; + }); + setPostResponseStartObserverForTest(() => postResponses++); + setUpstreamInterceptor(async () => { + call++; + if (call === 1) { + return new Response( + JSON.stringify({ + id: "resp_recall_abort", + object: "response", + created_at: 0, + model: "gpt-5.6-sol", + status: "completed", + output: [ + { + type: "function_call", + id: "fc_recall_abort", + call_id: "call_recall_abort", + name: "recall", + arguments: JSON.stringify({ + query: + "one two three four five six seven eight nine technical terms", + }), + status: "completed", + }, + ], + usage: { input_tokens: 1, output_tokens: 1 }, + }), + { headers: { "content-type": "application/json" } }, + ); + } + followUpStarted(); + if (abortSource === "upstream") { + throw new DOMException("upstream cancelled", "AbortError"); + } + return new Promise(() => {}); + }); + + try { + const request = makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "recall-followup-abort" }, + }); + request.stream = false; + request.signal = caller.signal; + const pending = handleRequest(request, loadLocalConfig()); + await started; + if (abortSource === "caller") { + caller.abort(new DOMException("client disconnected", "AbortError")); + } + + const response = await pending; + expect(response.status).toBe(502); + expect(postResponses).toBe(0); + } finally { + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }, + ); + + it("retains no replay body or recall result for a no-store turn", async () => { + let call = 0; + setUpstreamInterceptor(async () => { + call++; + if (call === 1) { + return new Response( + JSON.stringify({ + id: "resp_no_store_recall", + object: "response", + created_at: 0, + model: "gpt-5.6-sol", + status: "completed", + output: [ + { + type: "function_call", + id: "fc_no_store_recall", + call_id: "call_no_store_recall", + name: "recall", + arguments: JSON.stringify({ + query: + "one two three four five six seven eight nine private terms", + }), + status: "completed", + }, + ], + usage: { input_tokens: 1, output_tokens: 1 }, + }), + { headers: { "content-type": "application/json" } }, + ); + } + return new Response( + JSON.stringify({ + id: "resp_no_store_final", + object: "response", + created_at: 0, + model: "gpt-5.6-sol", + status: "completed", + output: [ + { + type: "message", + id: "msg_no_store_final", + role: "assistant", + status: "completed", + content: [ + { + type: "output_text", + text: "private answer", + annotations: [], + }, + ], + }, + ], + usage: { input_tokens: 1, output_tokens: 1 }, + }), + { headers: { "content-type": "application/json" } }, + ); + }); + + try { + const request = makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "no-store-recall-session" }, + }); + request.stream = false; + request.rawHeaders["x-lore-no-store"] = "true"; + const response = await handleRequest(request, loadLocalConfig()); + expect(response.status).toBe(200); + await response.text(); + + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === "no-store-recall-session", + ); + expect(state).toBeDefined(); + expect(state?.cacheAnalytics.lastRequestBody).toBeNull(); + expect(state?.recallStore.size).toBe(0); + expect( + loadSessionTracking(state?.sessionID ?? "")?.recallStore, + ).toBeNull(); + expect( + loadSessionTracking(state?.sessionID ?? "")?.fingerprint, + ).toBeFalsy(); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("accounts typed failed JSON recall follow-up usage in the fallback turn", async () => { + clearAllCosts(); + let call = 0; + setUpstreamInterceptor(async () => { + call++; + if (call === 1) { + return new Response( + JSON.stringify({ + id: "resp_failed_json_recall", + object: "response", + created_at: 0, + model: "gpt-5.6-sol", + status: "completed", + output: [ + { + type: "function_call", + id: "fc_failed_json_recall", + call_id: "call_failed_json_recall", + name: "recall", + arguments: JSON.stringify({ + query: "failed json recall usage", + }), + status: "completed", + }, + ], + usage: { input_tokens: 10, output_tokens: 1 }, + }), + { headers: { "content-type": "application/json" } }, + ); + } + return new Response( + JSON.stringify({ + id: "resp_failed_json_recall_followup", + object: "response", + created_at: 0, + model: "gpt-5.6-sol", + status: "failed", + output: [], + usage: { input_tokens: 1_000, output_tokens: 100 }, + error: { type: "server_error", message: "provider failed" }, + }), + { headers: { "content-type": "application/json" } }, + ); + }); + + try { + const request = makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "failed-json-recall-usage" }, + }); + request.stream = false; + request.rawHeaders["x-lore-no-store"] = "true"; + const response = await handleRequest(request, loadLocalConfig()); + expect(response.status).toBe(200); + await response.text(); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === "failed-json-recall-usage", + ); + expect(state).toBeDefined(); + expect( + getSessionCosts(state?.sessionID ?? "")?.conversation, + ).toMatchObject({ + inputTokens: 1_010, + outputTokens: 101, + turns: 1, + }); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + clearAllCosts(); + } + }); }); describe("budget throttle cancellation", () => { @@ -239,6 +675,7 @@ describe("Pipeline — streaming responses", () => { let harness: Harness; afterEach(() => harness?.teardown()); + afterEach(() => vi.mocked(getDegradationWarning).mockReset()); it("does not deadlock when an OpenAI translator drops Anthropic lifecycle frames", async () => { const anthropic = buildStreamingResponse( @@ -441,97 +878,5393 @@ describe("Pipeline — streaming responses", () => { expect(metaUpstream.response.body?.locked).toBe(false); }); - it("pauses a filled build queue and resumes it when reads begin", async () => { - let pulls = 0; - const body = await validAnthropicSSE("resume").text(); - const chunks = body - .split(/(?=event: )/) - .filter(Boolean) - .map((chunk) => new TextEncoder().encode(chunk)); - let index = 0; - const upstream = new Response( - new ReadableStream({ - pull(controller) { - pulls++; - if (index < chunks.length) controller.enqueue(chunks[index++]); - else controller.close(); - }, - }), + it("closes a Responses tool continuation before post-response storage", async () => { + const order: string[] = []; + let postResponses = 0; + let upstreamCalls = 0; + let upstreamCancellations = 0; + setPostResponseStartObserverForTest(() => + order.push(`post${++postResponses}`), ); - const downstream = buildStreamingResponse(upstream, () => {}); - await new Promise((resolve) => setImmediate(resolve)); - const pullsBeforeRead = pulls; - expect(pullsBeforeRead).toBeLessThan(chunks.length + 1); - const text = await downstream.text(); - expect(text).toContain("resume"); - expect(pulls).toBeGreaterThan(pullsBeforeRead); - }); - - it("distinguishes external meta abort from silent downstream cancellation", async () => { - let cancelledBeforeAcquire = false; - const beforeAcquire = validatedMetaStream( + const wire = validResponsesSSE("resp_tool_continuation", "continued"); + const makeUpstream = () => new Response( new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(wire)); + }, + pull() { + return new Promise(() => {}); + }, cancel() { - cancelledBeforeAcquire = true; + upstreamCancellations++; }, }), - ), - "anthropic", - false, + { headers: { "content-type": "text/event-stream" } }, + ); + setUpstreamInterceptor(async () => { + upstreamCalls++; + if (upstreamCalls === 2) order.push("upstream2"); + return makeUpstream(); + }); + const toolOutput = "tool output line\n".repeat(512); + const request = (sessionHeaders: Record): GatewayRequest => + makeResponsesRequest({ + sessionHeaders, + messages: [ + { + role: "user", + content: [{ type: "text", text: "read the file" }], + }, + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "call_read", + name: "read", + input: { filePath: "large.txt" }, + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + toolUseId: "call_read", + content: [{ type: "text", text: toolOutput }], + }, + ], + }, + ], + }); + + try { + const response = await handleRequest( + request({ "x-session-affinity": "legacy-affinity-session" }), + loadLocalConfig(), + ); + + const body = await response.text(); + order.push("eof1"); + const secondResponse = await handleRequest( + request({ + "x-lore-session-id": "stable-lore-session", + "x-session-affinity": "legacy-affinity-session", + }), + loadLocalConfig(), + ); + expect(order).toEqual(["eof1"]); + const secondBody = await secondResponse.text(); + order.push("eof2"); + await new Promise((resolve) => setImmediate(resolve)); + + setStreamingPostResponseLimitsForTest(64, 0); + const perSessionSaturated = await handleRequest( + request({ + "x-lore-session-id": "responses-post-response-per-session", + }), + loadLocalConfig(), + ); + const perSessionSaturatedBody = await perSessionSaturated.text(); + order.push("eof3"); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + setStreamingPostResponseLimitsForTest(0, 2); + const globallySaturated = await handleRequest( + request({ "x-lore-session-id": "responses-post-response-global" }), + loadLocalConfig(), + ); + const globallySaturatedBody = await globallySaturated.text(); + order.push("eof4"); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(body).toContain("event: response.completed"); + expect(body).not.toContain("event: response.failed"); + expect(secondBody).toContain("event: response.completed"); + expect(secondBody).not.toContain("event: response.failed"); + expect(perSessionSaturatedBody).toContain("event: response.completed"); + expect(perSessionSaturatedBody).not.toContain("event: response.failed"); + expect(globallySaturatedBody).toContain("event: response.completed"); + expect(globallySaturatedBody).not.toContain("event: response.failed"); + expect(order).toEqual([ + "eof1", + "post1", + "upstream2", + "eof2", + "post2", + "eof3", + "post3", + "eof4", + "post4", + ]); + expect(streamingPostResponsePendingForTest()).toBe(0); + expect(upstreamCancellations).toBe(4); + } finally { + setStreamingPostResponseLimitsForTest(); + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("does not start Responses post-processing before the final EOF read", async () => { + let postResponses = 0; + let reader: ReadableStreamDefaultReader | undefined; + setPostResponseStartObserverForTest(() => postResponses++); + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE("resp_terminal_before_eof"), { + headers: { "content-type": "text/event-stream" }, + }), ); - await beforeAcquire.body?.cancel(); - expect(cancelledBeforeAcquire).toBe(true); - let externallyCancelled = false; - const abort = new AbortController(); - abort.abort(new DOMException("deadline", "TimeoutError")); - const removeAbortListener = vi.spyOn(abort.signal, "removeEventListener"); - const externallyAborted = validatedMetaStream( - new Response( - new ReadableStream({ - cancel() { - externallyCancelled = true; + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": "terminal-before-eof-session", }, }), - ), - "anthropic", - false, - abort.signal, - ); - await expect(externallyAborted.text()).rejects.toMatchObject({ - name: "TimeoutError", - }); - expect(externallyCancelled).toBe(true); - expect(removeAbortListener).toHaveBeenCalledWith( - "abort", - expect.any(Function), - ); + loadLocalConfig(), + ); + reader = response.body?.getReader(); + expect(reader).toBeDefined(); + if (!reader) throw new Error("missing response body"); + const decoder = new TextDecoder(); + let output = ""; + while (!output.includes("event: response.completed")) { + const chunk = await reader.read(); + expect(chunk.done).toBe(false); + if (chunk.value) { + output += decoder.decode(chunk.value, { stream: true }); + } + } + + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + expect(postResponses).toBe(0); + + for (;;) { + const finalChunk = await reader.read(); + if (finalChunk.done) break; + } + await vi.waitFor(() => expect(postResponses).toBe(1)); + } finally { + if (reader) await reader.cancel().catch(() => {}); + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } }); - it("meta downstream cancel does not await a hostile upstream cancel", async () => { - let sourceCancelled = false; - const upstream = new Response( - new ReadableStream({ - start(controller) { - controller.enqueue( - new TextEncoder().encode( - 'event: message_start\ndata: {"type":"message_start","message":{"id":"hostile","type":"message","role":"assistant","model":"test","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":1,"output_tokens":0}}}\n\n', - ), - ); - }, - pull() { - return new Promise(() => {}); - }, - cancel() { - sourceCancelled = true; - return new Promise(() => {}); - }, - }), + it("defers buffered warning-path storage until the Responses body closes", async () => { + const order: string[] = []; + vi.mocked(getDegradationWarning).mockReturnValueOnce("workers degraded"); + setPostResponseStartObserverForTest(() => order.push("post")); + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE("resp_warning"), { + headers: { "content-type": "text/event-stream" }, + }), ); - const downstreamBody = validatedMetaStream( - upstream, - "anthropic", + + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "warning-path-session" }, + }), + loadLocalConfig(), + ); + const body = await response.text(); + order.push("eof"); + + expect(body).toContain("workers degraded"); + expect(order).toEqual(["eof"]); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + expect(order).toEqual(["eof", "post"]); + } finally { + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("does not finalize a buffered warning-path incomplete Responses turn", async () => { + const alias = "warning-incomplete-alias"; + const canonical = "warning-incomplete-canonical"; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + return new Response( + upstreamCall === 1 + ? validResponsesSSE("resp_warning_incomplete_setup") + : incompleteResponsesSSE("resp_warning_incomplete"), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + const store = vi.spyOn(temporal, "store"); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === alias, + ); + expect(state).toBeDefined(); + + vi.mocked(getDegradationWarning).mockReturnValueOnce("workers degraded"); + clearAllCosts(); + store.mockClear(); + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }), + loadLocalConfig(), + ); + const body = await response.text(); + expect(body).toContain("event: response.incomplete"); + expect(body).not.toContain("event: response.completed"); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + expect(store).not.toHaveBeenCalled(); + expect( + getSessionCosts(state?.sessionID ?? "")?.conversation, + ).toMatchObject({ + inputTokens: 1, + outputTokens: 0, + turns: 1, + }); + + const compact = await handleCompactEndpoint( + new Request("http://gateway.test/v1/compact", { + method: "POST", + headers: { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-session-id": canonical, + }, + body: JSON.stringify({ project_path: process.cwd() }), + }), + loadLocalConfig(), + ); + expect(compact.status).toBe(404); + expect(loadSessionTracking(state?.sessionID ?? "")).toMatchObject({ + headerName: "x-session-affinity", + headerSessionId: alias, + }); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + clearAllCosts(); + } + }); + + it("uses reserved capacity for production finalizers", async () => { + const end = vi.fn(); + const span = { + end, + setAttribute: vi.fn(), + setAttributes: vi.fn(), + setStatus: vi.fn(), + updateName: vi.fn(), + } as unknown as Sentry.Span; + vi.mocked(Sentry.startInactiveSpan).mockReturnValueOnce(span); + let postResponses = 0; + setPostResponseStartObserverForTest(() => postResponses++); + setStreamingPostResponseLimitsForTest(0, 2); + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE("resp_dropped_span"), { + headers: { "content-type": "text/event-stream" }, + }), + ); + + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "dropped-span-session" }, + }), + loadLocalConfig(), + ); + await response.text(); + await vi.waitFor(() => expect(postResponses).toBe(1)); + + expect(end).toHaveBeenCalledOnce(); + expect(streamingPostResponsePendingForTest()).toBe(0); + } finally { + setStreamingPostResponseLimitsForTest(); + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("enforces and cleans up the real per-session and global queue limits", async () => { + setUpstreamInterceptor(async () => new Response("{}", { status: 200 })); + const admissionRequest = makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "queue-admission" }, + tools: [], + }); + admissionRequest.stream = false; + admissionRequest.rawHeaders["x-lore-agent"] = "title"; + + try { + await (await handleRequest(admissionRequest, loadLocalConfig())).text(); + const perSessionOrder: number[] = []; + let releasePerSession: (() => void) | undefined; + const perSessionGate = new Promise((resolve) => { + releasePerSession = resolve; + }); + let perSessionDrops = 0; + scheduleStreamingPostResponseForTest("real-limit-session", async () => { + await perSessionGate; + perSessionOrder.push(1); + }); + scheduleStreamingPostResponseForTest("real-limit-session", async () => { + await perSessionGate; + perSessionOrder.push(2); + }); + scheduleStreamingPostResponseForTest( + "real-limit-session", + () => { + perSessionOrder.push(3); + }, + () => perSessionDrops++, + ); + + expect(streamingPostResponsePendingForTest()).toBe(2); + expect(perSessionDrops).toBe(1); + releasePerSession?.(); + await vi.waitFor(() => + expect(streamingPostResponsePendingForTest()).toBe(0), + ); + expect(perSessionOrder).toEqual([1, 2]); + + let releaseGlobal: (() => void) | undefined; + const globalGate = new Promise((resolve) => { + releaseGlobal = resolve; + }); + let globalDrops = 0; + for (let index = 0; index < 64; index++) { + scheduleStreamingPostResponseForTest( + `real-global-limit-${index}`, + () => globalGate, + ); + } + scheduleStreamingPostResponseForTest( + "real-global-limit-overflow", + () => {}, + () => globalDrops++, + ); + + expect(streamingPostResponsePendingForTest()).toBe(64); + expect(globalDrops).toBe(1); + releaseGlobal?.(); + await vi.waitFor(() => + expect(streamingPostResponsePendingForTest()).toBe(0), + ); + + let releaseResetFinalizer: (() => void) | undefined; + const resetFinalizerGate = new Promise((resolve) => { + releaseResetFinalizer = resolve; + }); + scheduleStreamingPostResponseForTest( + "real-reset-limit", + () => resetFinalizerGate, + ); + const reset = resetPipelineState(); + let resetSettled = false; + void reset.then(() => { + resetSettled = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(resetSettled).toBe(false); + releaseResetFinalizer?.(); + await reset; + expect(streamingPostResponsePendingForTest()).toBe(0); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("ends the response span when a stream is cancelled before terminal", async () => { + const end = vi.fn(); + const setStatus = vi.fn(); + const span = { + end, + setAttribute: vi.fn(), + setAttributes: vi.fn(), + setStatus, + updateName: vi.fn(), + } as unknown as Sentry.Span; + vi.mocked(Sentry.startInactiveSpan).mockReturnValueOnce(span); + let upstreamStartedResolve: (() => void) | undefined; + const upstreamStarted = new Promise((resolve) => { + upstreamStartedResolve = resolve; + }); + let upstreamCancellations = 0; + setUpstreamInterceptor(async () => { + upstreamStartedResolve?.(); + return new Response( + new ReadableStream({ + cancel() { + upstreamCancellations++; + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "cancelled-span-session" }, + }), + loadLocalConfig(), + ); + const reader = response.body?.getReader(); + expect(reader).toBeDefined(); + await reader?.read(); + await upstreamStarted; + await reader?.cancel("client disconnected"); + await vi.waitFor(() => expect(end).toHaveBeenCalledOnce()); + + expect(setStatus).toHaveBeenCalledWith({ + code: 2, + message: "stream cancelled before terminal response", + }); + expect(upstreamCancellations).toBe(1); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("ends an unsuccessful Responses stream span exactly once", async () => { + const end = vi.fn(); + const span = { + end, + setAttribute: vi.fn(), + setAttributes: vi.fn(), + setStatus: vi.fn(), + updateName: vi.fn(), + } as unknown as Sentry.Span; + vi.mocked(Sentry.startInactiveSpan).mockReturnValueOnce(span); + setUpstreamInterceptor( + async () => + new Response(incompleteResponsesSSE("resp_incomplete_span"), { + headers: { "content-type": "text/event-stream" }, + }), + ); + + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "incomplete-span-session" }, + }), + loadLocalConfig(), + ); + expect(await response.text()).toContain("event: response.incomplete"); + await vi.waitFor(() => expect(end).toHaveBeenCalledOnce()); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it.each(["incomplete", "failed"] as const)( + "accounts validated usage from a %s Responses terminal without storing the turn", + async (terminal) => { + clearAllCosts(); + const today = new Date().toISOString().slice(0, 10); + const ledgerBefore = getDailyCostForDay(today); + const sessionHeader = `account-${terminal}-response-session`; + const wire = + responsesEvent("response.created", { + response: { + id: `resp_account_${terminal}`, + model: "gpt-5.6-sol", + status: "in_progress", + }, + }) + + responsesEvent(`response.${terminal}`, { + response: { + id: `resp_account_${terminal}`, + model: "gpt-5.6-sol", + status: terminal, + output: [], + usage: { input_tokens: 1_000, output_tokens: 100 }, + ...(terminal === "incomplete" + ? { incomplete_details: { reason: "max_output_tokens" } } + : { + error: { type: "server_error", message: "provider failed" }, + }), + }, + }); + setUpstreamInterceptor( + async () => + new Response(wire, { + headers: { "content-type": "text/event-stream" }, + }), + ); + const store = vi.spyOn(temporal, "store"); + + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": sessionHeader }, + }), + loadLocalConfig(), + ); + expect(await response.text()).toContain(`event: response.${terminal}`); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === sessionHeader, + ); + expect(state).toBeDefined(); + await vi.waitFor(() => { + expect(getSessionCosts(state?.sessionID ?? "")?.conversation).toEqual( + expect.objectContaining({ + inputTokens: 1_000, + outputTokens: 100, + turns: 1, + }), + ); + }); + expect(getDailySpend().spend).toBeGreaterThan(0); + expect(getCostRate()).toBeGreaterThan(0); + expect(getDailyCostForDay(today)).toBeGreaterThan(ledgerBefore); + expect(store).not.toHaveBeenCalled(); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + clearAllCosts(); + } + }, + ); + + it("accounts a failed recall-aware continuation without storing the turn", async () => { + clearAllCosts(); + const sessionHeader = "account-failed-recall-continuation"; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + if (upstreamCall === 1) { + const args = JSON.stringify({ + query: + "one two three four five six seven eight nine architecture terms", + }); + return new Response( + responsesEvent("response.created", { + response: { + id: "resp_recall_accounting", + model: "gpt-5.6-sol", + status: "in_progress", + }, + }) + + responsesEvent("response.output_item.added", { + output_index: 0, + item: { + type: "function_call", + id: "fc_recall_accounting", + call_id: "call_recall_accounting", + name: "recall", + }, + }) + + responsesEvent("response.function_call_arguments.done", { + output_index: 0, + item_id: "fc_recall_accounting", + arguments: args, + }) + + responsesEvent("response.output_item.done", { + output_index: 0, + item: { + type: "function_call", + id: "fc_recall_accounting", + call_id: "call_recall_accounting", + name: "recall", + arguments: args, + status: "completed", + }, + }) + + responsesEvent("response.completed", { + response: { + id: "resp_recall_accounting", + model: "gpt-5.6-sol", + status: "completed", + output: [ + { + type: "function_call", + id: "fc_recall_accounting", + call_id: "call_recall_accounting", + name: "recall", + arguments: args, + status: "completed", + }, + ], + usage: { input_tokens: 10, output_tokens: 1 }, + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + } + return new Response( + responsesEvent("response.created", { + response: { + id: "resp_recall_accounting_failed", + model: "gpt-5.6-sol", + status: "in_progress", + }, + }) + + responsesEvent("response.failed", { + response: { + id: "resp_recall_accounting_failed", + model: "gpt-5.6-sol", + status: "failed", + output: [], + usage: { input_tokens: 1_000, output_tokens: 100 }, + error: { type: "server_error", message: "provider failed" }, + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + const store = vi.spyOn(temporal, "store"); + + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": sessionHeader }, + }), + loadLocalConfig(), + ); + expect(await response.text()).toContain("event: response.failed"); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === sessionHeader, + ); + expect(state).toBeDefined(); + await vi.waitFor(() => { + expect( + getSessionCosts(state?.sessionID ?? "")?.conversation, + ).toMatchObject({ + inputTokens: 1_010, + outputTokens: 101, + turns: 1, + }); + }); + expect(store).not.toHaveBeenCalled(); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + clearAllCosts(); + } + }); + + it("defers non-stream incomplete accounting and span closure until EOF", async () => { + clearAllCosts(); + const end = vi.fn(); + const span = { + end, + setAttribute: vi.fn(), + setAttributes: vi.fn(), + setStatus: vi.fn(), + updateName: vi.fn(), + } as unknown as Sentry.Span; + vi.mocked(Sentry.startInactiveSpan).mockReturnValueOnce(span); + setUpstreamInterceptor( + async () => + new Response( + JSON.stringify({ + id: "resp_nonstream_incomplete_accounting", + object: "response", + created_at: 0, + model: "gpt-5.6-sol", + status: "incomplete", + output: [], + usage: { input_tokens: 1_000, output_tokens: 100 }, + }), + { headers: { "content-type": "application/json" } }, + ), + ); + + try { + const request = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": "nonstream-incomplete-accounting", + }, + }); + request.stream = false; + const response = await handleRequest(request, loadLocalConfig()); + expect(end).not.toHaveBeenCalled(); + expect(response.status).toBe(200); + const body = (await response.json()) as Record; + expect(body.status).toBe("incomplete"); + expect(body.incomplete_details).toEqual({ + reason: "max_output_tokens", + }); + await vi.waitFor(() => expect(end).toHaveBeenCalledOnce()); + const state = [...getActiveSessions().values()].find( + (candidate) => + candidate.headerSessionId === "nonstream-incomplete-accounting", + ); + expect( + getSessionCosts(state?.sessionID ?? "")?.conversation, + ).toMatchObject({ + inputTokens: 1_000, + outputTokens: 100, + turns: 1, + }); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + clearAllCosts(); + } + }); + + it("rejects an unknown public incomplete reason on an established conversation", async () => { + const sessionHeader = "malformed-incomplete-established"; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + if (upstreamCall === 1) { + return new Response(validResponsesSSE("resp_malformed_setup"), { + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response( + JSON.stringify({ + id: "resp_malformed_incomplete", + model: "gpt-5.6-sol", + status: "incomplete", + incomplete_details: { reason: "provider_specific" }, + output: [], + usage: { input_tokens: 10, output_tokens: 2 }, + }), + { headers: { "content-type": "application/json" } }, + ); + }); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": sessionHeader }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const request = makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": sessionHeader }, + }); + request.stream = false; + const response = await handleRequest(request, loadLocalConfig()); + expect(response.status).toBe(502); + expect(await response.text()).toContain("Gateway request failed"); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it.each([false, true])( + "rejects malformed completed JSON on an established conversation codex=%s", + async (codex) => { + const sessionHeader = `malformed-completed-${codex}`; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + if (upstreamCall === 1) { + return new Response(validResponsesSSE("resp_completed_setup"), { + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response(JSON.stringify({ status: "completed" }), { + headers: { "content-type": "application/json" }, + }); + }); + const store = vi.spyOn(temporal, "store"); + + try { + const setup = makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": sessionHeader }, + }); + setup.codex = codex; + await (await handleRequest(setup, loadLocalConfig())).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === sessionHeader, + ); + expect(state).toBeDefined(); + clearAllCosts(); + store.mockClear(); + + const request = makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": sessionHeader }, + }); + request.stream = false; + request.codex = codex; + const response = await handleRequest(request, loadLocalConfig()); + expect(response.status).toBe(502); + expect(await response.text()).toContain("Gateway request failed"); + expect(store).not.toHaveBeenCalled(); + expect(getSessionCosts(state?.sessionID ?? "")).toBeNull(); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + clearAllCosts(); + } + }, + ); + + it("rejects an unknown public incomplete reason on a provisional conversation", async () => { + setUpstreamInterceptor( + async () => + new Response( + JSON.stringify({ + id: "resp_provisional_unknown_incomplete", + model: "gpt-5.6-sol", + status: "incomplete", + incomplete_details: { reason: "provider_specific" }, + output: [], + usage: { input_tokens: 10, output_tokens: 2 }, + }), + { headers: { "content-type": "application/json" } }, + ), + ); + + try { + const request = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": "provisional-unknown-incomplete", + }, + }); + request.stream = false; + const response = await handleRequest(request, loadLocalConfig()); + expect(response.status).toBe(502); + expect(await response.text()).toContain("Gateway request failed"); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("does not expose recall from an incomplete buffered response", async () => { + const sessionHeader = "incomplete-private-recall"; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + if (upstreamCall === 1) { + return new Response(validResponsesSSE("resp_recall_setup"), { + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response( + JSON.stringify({ + id: "resp_incomplete_recall", + model: "gpt-5.6-sol", + status: "incomplete", + incomplete_details: { reason: "max_output_tokens" }, + output: [ + { + type: "function_call", + id: "fc_private_recall", + call_id: "call_private_recall", + name: "recall", + arguments: '{"query":"private context"}', + }, + ], + usage: { input_tokens: 10, output_tokens: 2 }, + }), + { headers: { "content-type": "application/json" } }, + ); + }); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": sessionHeader }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const request = makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": sessionHeader }, + }); + request.stream = false; + const response = await handleRequest(request, loadLocalConfig()); + expect(response.status).toBe(502); + const body = await response.text(); + expect(body).toContain("Gateway request failed"); + expect(body).not.toContain("private context"); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("preserves finalizer order for overlapping requests in one session", async () => { + const order: string[] = []; + const sources: Array< + ReadableStreamDefaultController | undefined + > = []; + const startedResolvers: Array<(() => void) | undefined> = []; + const started = [0, 1, 2].map( + (index) => + new Promise((resolve) => { + startedResolvers[index] = resolve; + }), + ); + let upstreamCall = 0; + let postResponses = 0; + setPostResponseStartObserverForTest(() => + order.push(`post${++postResponses}`), + ); + setUpstreamInterceptor(async () => { + const index = upstreamCall++; + return new Response( + new ReadableStream({ + start(controller) { + sources[index] = controller; + if (index === 2) order.push("upstream3"); + startedResolvers[index]?.(); + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + const request = () => + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "overlapping-session" }, + }); + + try { + const first = await handleRequest(request(), loadLocalConfig()); + const firstBody = first.text(); + await started[0]; + + const second = await handleRequest(request(), loadLocalConfig()); + const secondBody = second.text(); + await new Promise((resolve) => setImmediate(resolve)); + expect(upstreamCall).toBe(1); + + sources[0]?.enqueue( + new TextEncoder().encode(validResponsesSSE("resp_overlap_1")), + ); + sources[0]?.close(); + await firstBody; + order.push("eof1"); + await started[1]; + expect(order).toEqual(["eof1", "post1"]); + + const third = await handleRequest(request(), loadLocalConfig()); + const thirdBody = third.text(); + await new Promise((resolve) => setImmediate(resolve)); + expect(upstreamCall).toBe(2); + + sources[1]?.enqueue( + new TextEncoder().encode(validResponsesSSE("resp_overlap_2")), + ); + sources[1]?.close(); + await secondBody; + order.push("eof2"); + await started[2]; + + expect(order).toEqual(["eof1", "post1", "eof2", "post2", "upstream3"]); + + sources[2]?.enqueue( + new TextEncoder().encode(validResponsesSSE("resp_overlap_3")), + ); + sources[2]?.close(); + await thirdBody; + order.push("eof3"); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(order).toEqual([ + "eof1", + "post1", + "eof2", + "post2", + "upstream3", + "eof3", + "post3", + ]); + expect(streamingPostResponsePendingForTest()).toBe(0); + } finally { + setStreamingPostResponseWaitObserverForTest(undefined); + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("keeps one same-session waiter off global active capacity", async () => { + const ownerHeaders = { "x-lore-session-id": "fair-session-owner" }; + let ownerSource: ReadableStreamDefaultController | undefined; + let ownerStartedResolve!: () => void; + const ownerStarted = new Promise((resolve) => { + ownerStartedResolve = resolve; + }); + let upstreamCalls = 0; + setMaxActivePipelineRequestsForTest(2); + setUpstreamInterceptor(async () => { + upstreamCalls++; + if (upstreamCalls === 1) { + return new Response( + new ReadableStream({ + start(controller) { + ownerSource = controller; + ownerStartedResolve(); + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + } + return new Response(validResponsesSSE(`resp_fair_${upstreamCalls}`), { + headers: { "content-type": "text/event-stream" }, + }); + }); + + try { + const owner = await handleRequest( + makeResponsesRequest({ sessionHeaders: ownerHeaders }), + loadLocalConfig(), + ); + const ownerBody = owner.text(); + await ownerStarted; + + const waiting = await handleRequest( + makeResponsesRequest({ sessionHeaders: ownerHeaders }), + loadLocalConfig(), + ); + const waitingBody = waiting.text(); + await new Promise((resolve) => setImmediate(resolve)); + expect(activePipelineRequestCountForTest()).toBe(1); + + const overflow = await handleRequest( + makeResponsesRequest({ sessionHeaders: ownerHeaders }), + loadLocalConfig(), + ); + expect(await overflow.text()).toContain("event: response.failed"); + + const unrelated = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "fair-unrelated-session" }, + }), + loadLocalConfig(), + ); + expect(await unrelated.text()).toContain("event: response.completed"); + expect(upstreamCalls).toBe(2); + + ownerSource?.enqueue( + new TextEncoder().encode(validResponsesSSE("resp_fair_owner")), + ); + ownerSource?.close(); + ownerSource = undefined; + await ownerBody; + expect(await waitingBody).toContain("event: response.completed"); + expect(upstreamCalls).toBe(3); + } finally { + ownerSource?.close(); + setMaxActivePipelineRequestsForTest(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("rejects an unread early-flush response after pipeline reset", async () => { + let postResponses = 0; + setPostResponseStartObserverForTest(() => postResponses++); + setUpstreamInterceptor(async () => { + throw new Error("unread early-flush response must not start"); + }); + + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": "responses-post-response-reset", + }, + }), + loadLocalConfig(), + ); + await resetPipelineState(); + setPostResponseStartObserverForTest(() => postResponses++); + setUpstreamInterceptor(async () => new Response("{}", { status: 200 })); + const reopenedRequest = makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "post-reset-request" }, + tools: [], + }); + reopenedRequest.stream = false; + reopenedRequest.rawHeaders["x-lore-agent"] = "title"; + await (await handleRequest(reopenedRequest, loadLocalConfig())).text(); + + let staleUpstreamCalls = 0; + setUpstreamInterceptor(async () => { + staleUpstreamCalls++; + return new Response(validResponsesSSE("resp_after_reset"), { + headers: { "content-type": "text/event-stream" }, + }); + }); + const body = await response.text(); + + expect(body).toContain("event: response.failed"); + expect(body).toContain("Gateway request failed"); + expect(staleUpstreamCalls).toBe(0); + expect(postResponses).toBe(0); + expect(streamingPostResponsePendingForTest()).toBe(0); + } finally { + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("retains cancelled producers until abort-unaware work settles", async () => { + let releaseProducer: (() => void) | undefined; + const producerPause = new Promise((resolve) => { + releaseProducer = resolve; + }); + let producerWaitingResolve: (() => void) | undefined; + const producerWaiting = new Promise((resolve) => { + producerWaitingResolve = resolve; + }); + setPipelinePreUpstreamPauseForTest(producerPause, () => + producerWaitingResolve?.(), + ); + const sessionHeaders = { + "x-lore-session-id": "cancelled-preterminal-session", + }; + const initialActiveRequests = activePipelineRequestCountForTest(); + + try { + const response = await handleRequest( + makeResponsesRequest({ sessionHeaders }), + loadLocalConfig(), + ); + const reader = response.body?.getReader(); + expect(reader).toBeDefined(); + await reader?.read(); + await producerWaiting; + + const state = [...getActiveSessions().values()].find( + (candidate) => + candidate.headerSessionId === sessionHeaders["x-lore-session-id"], + ); + expect(state).toBeDefined(); + expect(activePipelineRequestCountForTest()).toBe( + initialActiveRequests + 1, + ); + expect(isPipelineSessionActiveForTest(state?.sessionID ?? "")).toBe(true); + + await reader?.cancel("client disconnected"); + expect(activePipelineRequestCountForTest()).toBe( + initialActiveRequests + 1, + ); + expect(isPipelineSessionActiveForTest(state?.sessionID ?? "")).toBe(true); + setMaxActivePipelineRequestsForTest(initialActiveRequests + 1); + const saturated = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "capacity-rejected-session" }, + }), + loadLocalConfig(), + ); + expect(saturated.status).toBe(503); + expect(await saturated.text()).toContain("Gateway is busy"); + + releaseProducer?.(); + await vi.waitFor(() => { + expect(activePipelineRequestCountForTest()).toBe(initialActiveRequests); + expect(isPipelineSessionActiveForTest(state?.sessionID ?? "")).toBe( + false, + ); + }); + } finally { + releaseProducer?.(); + setMaxActivePipelineRequestsForTest(); + setPipelinePreUpstreamPauseForTest(undefined); + await resetPipelineState(); + } + }); + + it("waits for an abort-unaware early-flush producer before reset clears state", async () => { + let releaseProducer: (() => void) | undefined; + const producerPause = new Promise((resolve) => { + releaseProducer = resolve; + }); + let producerWaitingResolve: (() => void) | undefined; + const producerWaiting = new Promise((resolve) => { + producerWaitingResolve = resolve; + }); + setPipelinePreUpstreamPauseForTest(producerPause, () => + producerWaitingResolve?.(), + ); + let upstreamCalls = 0; + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response(validResponsesSSE("resp_stale_producer"), { + headers: { "content-type": "text/event-stream" }, + }); + }); + let reset: Promise | undefined; + + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "stale-producer-session" }, + }), + loadLocalConfig(), + ); + const body = response.text(); + await producerWaiting; + reset = resetPipelineState(); + let resetSettled = false; + void reset.then(() => { + resetSettled = true; + }); + + await new Promise((resolve) => setImmediate(resolve)); + expect(resetSettled).toBe(false); + releaseProducer?.(); + await reset; + expect(resetSettled).toBe(true); + expect(upstreamCalls).toBe(0); + expect(await body).toContain("event: response.failed"); + } finally { + releaseProducer?.(); + await reset; + setPipelinePreUpstreamPauseForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("fences an early-flush producer that resumes after the reset timeout", async () => { + const sessionHeader = "late-stale-producer-session"; + const staleProjectPath = "/tmp"; + const freshProjectPath = process.cwd(); + const staleUpstream = "https://stale-reset.example"; + const freshUpstream = "https://fresh-reset.example"; + let releaseProducer: (() => void) | undefined; + const producerPause = new Promise((resolve) => { + releaseProducer = resolve; + }); + let producerWaitingResolve: (() => void) | undefined; + const producerWaiting = new Promise((resolve) => { + producerWaitingResolve = resolve; + }); + setPipelinePreUpstreamPauseForTest(producerPause, () => + producerWaitingResolve?.(), + ); + setPipelineResetSettleTimeoutForTest(0); + let upstreamCalls = 0; + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response(validResponsesSSE("resp_late_stale_producer"), { + headers: { "content-type": "text/event-stream" }, + }); + }); + + try { + const staleRequest = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": sessionHeader, + }, + }); + staleRequest.rawHeaders["x-lore-project"] = staleProjectPath; + staleRequest.rawHeaders["x-lore-upstream-url"] = staleUpstream; + const response = await handleRequest(staleRequest, loadLocalConfig()); + const body = response.text(); + await producerWaiting; + await resetPipelineState(); + + expect(activePipelineRequestCountForTest()).toBe(0); + expect(detachedPipelineRequestCountForTest()).toBe(1); + setMaxDetachedPipelineRequestsForTest(1); + const quarantineFull = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": "quarantine-saturation-session", + }, + }), + loadLocalConfig(), + ); + expect(quarantineFull.status).toBe(503); + expect(await quarantineFull.text()).toContain("Gateway is busy"); + setMaxDetachedPipelineRequestsForTest(); + setPipelinePreUpstreamPauseForTest(undefined); + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response(validResponsesSSE("resp_reopened_after_timeout"), { + headers: { "content-type": "text/event-stream" }, + }); + }); + const freshRequest = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": sessionHeader, + }, + }); + freshRequest.rawHeaders["x-lore-project"] = freshProjectPath; + freshRequest.rawHeaders["x-lore-upstream-url"] = freshUpstream; + const reopened = await handleRequest(freshRequest, loadLocalConfig()); + expect(await reopened.text()).toContain("event: response.completed"); + expect(upstreamCalls).toBe(1); + const freshState = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === sessionHeader, + ); + expect(freshState).toBeDefined(); + await vi.waitFor(() => { + expect(loadSessionTracking(freshState?.sessionID ?? "")).toMatchObject({ + projectPath: freshProjectPath, + projectPathProvisional: false, + lastUpstream: expect.stringContaining(freshUpstream), + }); + }); + const freshTracking = loadSessionTracking(freshState?.sessionID ?? ""); + + releaseProducer?.(); + expect(await body).toContain("event: response.failed"); + await vi.waitFor(() => + expect(detachedPipelineRequestCountForTest()).toBe(0), + ); + expect(upstreamCalls).toBe(1); + expect(freshState).toMatchObject({ + projectPath: freshProjectPath, + projectPathProvisional: false, + lastUpstream: expect.objectContaining({ url: freshUpstream }), + }); + expect(loadSessionTracking(freshState?.sessionID ?? "")).toMatchObject({ + projectPath: freshProjectPath, + projectPathProvisional: false, + lastUpstream: freshTracking?.lastUpstream, + }); + } finally { + releaseProducer?.(); + setMaxDetachedPipelineRequestsForTest(); + setPipelinePreUpstreamPauseForTest(undefined); + setPipelineResetSettleTimeoutForTest(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("aborts a started stream before reset reopens admission", async () => { + let upstreamStartedResolve: (() => void) | undefined; + const upstreamStarted = new Promise((resolve) => { + upstreamStartedResolve = resolve; + }); + let postResponses = 0; + let upstreamCancellations = 0; + setUpstreamInterceptor(async () => { + upstreamStartedResolve?.(); + return new Response( + new ReadableStream({ + cancel() { + upstreamCancellations++; + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": "started-response-before-reset", + }, + }), + loadLocalConfig(), + ); + const bodyResult = response.text(); + await upstreamStarted; + await resetPipelineState(); + + setPostResponseStartObserverForTest(() => postResponses++); + setUpstreamInterceptor(async () => new Response("{}", { status: 200 })); + const reopenedRequest = makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "started-post-reset-request" }, + tools: [], + }); + reopenedRequest.stream = false; + reopenedRequest.rawHeaders["x-lore-agent"] = "title"; + await (await handleRequest(reopenedRequest, loadLocalConfig())).text(); + const body = await bodyResult; + + expect(body).toContain("event: response.failed"); + expect(body).toContain("Gateway request failed"); + expect(upstreamCancellations).toBe(1); + expect(postResponses).toBe(0); + expect(streamingPostResponsePendingForTest()).toBe(0); + } finally { + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("does not start an unread structural compaction after reset", async () => { + let upstreamCalls = 0; + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response(validResponsesSSE("resp_before_unread_compact"), { + headers: { "content-type": "text/event-stream" }, + }); + }); + const sessionHeaders = { + "x-lore-session-id": "unread-compaction-reset-session", + }; + let compactionRead: { mockRestore: () => void } | undefined; + + try { + const established = await handleRequest( + makeResponsesRequest({ + sessionHeaders, + messages: Array.from({ length: 12 }, (_, index) => ({ + role: index % 2 === 0 ? ("user" as const) : ("assistant" as const), + content: [{ type: "text" as const, text: `turn ${index}` }], + })), + }), + loadLocalConfig(), + ); + await established.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + compactionRead = vi.spyOn(temporal, "undistilledCount"); + const compacted = await handleRequest( + makeResponsesRequest({ + sessionHeaders, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "Create an anchored summary from the conversation history above.", + }, + ], + }, + ], + tools: [], + }), + loadLocalConfig(), + ); + await resetPipelineState(); + + await expect(compacted.text()).rejects.toMatchObject({ + name: "AbortError", + message: "gateway pipeline reset", + }); + expect(compactionRead).not.toHaveBeenCalled(); + expect(upstreamCalls).toBe(1); + } finally { + compactionRead?.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("keeps cancelled streaming compaction active until abort-unaware distillation settles", async () => { + let upstreamCalls = 0; + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response(validResponsesSSE("resp_before_cancelled_compact"), { + headers: { "content-type": "text/event-stream" }, + }); + }); + const sessionHeaders = { + "x-lore-session-id": "cancelled-compaction-session", + }; + let distillationStarted!: () => void; + const started = new Promise((resolve) => { + distillationStarted = resolve; + }); + let distillationAborted!: () => void; + const aborted = new Promise((resolve) => { + distillationAborted = resolve; + }); + let releaseDistillation!: () => void; + const distillationResult = new Promise<{ + rounds: number; + distilled: number; + }>((resolve) => { + releaseDistillation = () => resolve({ rounds: 0, distilled: 0 }); + }); + let distillationSignal: AbortSignal | undefined; + const undistilledCount = vi + .spyOn(temporal, "undistilledCount") + .mockReturnValue(1); + const runDistillation = vi + .spyOn(distillation, "run") + .mockImplementation(async (input) => { + distillationStarted(); + const signal = input.signal; + if (!signal) throw new Error("compaction signal missing"); + distillationSignal = signal; + const onAbort = () => distillationAborted(); + signal.addEventListener("abort", onAbort, { once: true }); + if (signal.aborted) onAbort(); + // Deliberately ignore cancellation. The request must retain its active + // session claim until this underlying operation actually settles. + return distillationResult; + }); + + try { + const established = await handleRequest( + makeResponsesRequest({ + sessionHeaders, + messages: Array.from({ length: 12 }, (_, index) => ({ + role: index % 2 === 0 ? ("user" as const) : ("assistant" as const), + content: [{ type: "text" as const, text: `turn ${index}` }], + })), + }), + loadLocalConfig(), + ); + await established.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const sessionState = [...getActiveSessions().values()].find( + (candidate) => + candidate.headerSessionId === sessionHeaders["x-lore-session-id"], + ); + expect(sessionState).toBeDefined(); + + const compactRequest = makeResponsesRequest({ + sessionHeaders, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "Create an anchored summary from the conversation history above.", + }, + ], + }, + ], + tools: [], + }); + compactRequest.protocol = "anthropic"; + compactRequest.model = DEFAULT_MODEL; + compactRequest.rawHeaders["x-lore-provider"] = "anthropic"; + compactRequest.rawHeaders["x-lore-upstream-url"] = + "https://api.anthropic.com"; + const compacted = await handleRequest(compactRequest, loadLocalConfig()); + await started; + expect(distillationSignal).toBeDefined(); + expect(activePipelineRequestCountForTest()).toBe(1); + + await compacted.body?.cancel( + new DOMException("client disconnected", "AbortError"), + ); + await aborted; + await new Promise((resolve) => setImmediate(resolve)); + + expect(runDistillation).toHaveBeenCalledOnce(); + expect(distillationSignal?.aborted).toBe(true); + expect(activePipelineRequestCountForTest()).toBe(1); + expect( + isPipelineSessionActiveForTest(sessionState?.sessionID ?? ""), + ).toBe(true); + + releaseDistillation(); + await distillationResult; + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(activePipelineRequestCountForTest()).toBe(0); + expect( + isPipelineSessionActiveForTest(sessionState?.sessionID ?? ""), + ).toBe(false); + expect(upstreamCalls).toBe(1); + } finally { + releaseDistillation(); + runDistillation.mockRestore(); + undistilledCount.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("keeps cancelled streaming compaction active until abort-unaware LTM lookup settles", async () => { + let upstreamCalls = 0; + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response(validResponsesSSE("resp_before_cancelled_ltm"), { + headers: { "content-type": "text/event-stream" }, + }); + }); + const sessionHeaders = { + "x-lore-session-id": "cancelled-compaction-ltm-session", + }; + let releaseLookup!: () => void; + const lookupResult = new Promise< + Awaited> + >((resolve) => { + releaseLookup = () => resolve([]); + }); + let lookupStarted!: () => void; + const started = new Promise((resolve) => { + lookupStarted = resolve; + }); + let undistilledCount: { mockRestore(): void } | undefined; + let lookup: { mockRestore(): void } | undefined; + + try { + const established = await handleRequest( + makeResponsesRequest({ + sessionHeaders, + messages: Array.from({ length: 12 }, (_, index) => ({ + role: index % 2 === 0 ? ("user" as const) : ("assistant" as const), + content: [{ type: "text" as const, text: `turn ${index}` }], + })), + }), + loadLocalConfig(), + ); + await established.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const sessionState = [...getActiveSessions().values()].find( + (candidate) => + candidate.headerSessionId === sessionHeaders["x-lore-session-id"], + ); + expect(sessionState).toBeDefined(); + + undistilledCount = vi + .spyOn(temporal, "undistilledCount") + .mockReturnValue(0); + lookup = vi.spyOn(ltm, "forProjectOffloaded").mockImplementation(() => { + lookupStarted(); + return lookupResult; + }); + + const compactRequest = makeResponsesRequest({ + sessionHeaders, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "Create an anchored summary from the conversation history above.", + }, + ], + }, + ], + tools: [], + }); + compactRequest.protocol = "anthropic"; + compactRequest.model = DEFAULT_MODEL; + compactRequest.rawHeaders["x-lore-provider"] = "anthropic"; + compactRequest.rawHeaders["x-lore-upstream-url"] = + "https://api.anthropic.com"; + const compacted = await handleRequest(compactRequest, loadLocalConfig()); + await started; + + await compacted.body?.cancel( + new DOMException("client disconnected", "AbortError"), + ); + await new Promise((resolve) => setImmediate(resolve)); + + expect(lookup).toHaveBeenCalledOnce(); + expect(activePipelineRequestCountForTest()).toBe(1); + expect( + isPipelineSessionActiveForTest(sessionState?.sessionID ?? ""), + ).toBe(true); + + releaseLookup(); + await lookupResult; + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(activePipelineRequestCountForTest()).toBe(0); + expect( + isPipelineSessionActiveForTest(sessionState?.sessionID ?? ""), + ).toBe(false); + expect(upstreamCalls).toBe(1); + } finally { + releaseLookup(); + lookup?.mockRestore(); + undistilledCount?.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("rejects requests while pipeline reset is in progress", async () => { + let releaseReset: (() => void) | undefined; + const resetPause = new Promise((resolve) => { + releaseReset = resolve; + }); + let upstreamCalls = 0; + setPipelineResetPauseForTest(resetPause); + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response("{}", { status: 200 }); + }); + const reset = resetPipelineState(); + + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "request-during-reset" }, + }), + loadLocalConfig(), + ); + + expect(response.status).toBe(503); + expect(await response.text()).toContain("Gateway pipeline is resetting"); + expect(upstreamCalls).toBe(0); + } finally { + releaseReset?.(); + await reset; + setPipelineResetPauseForTest(undefined); + setUpstreamInterceptor(undefined); + } + }); + + it("makes concurrent reset callers await the same teardown", async () => { + let releaseReset: (() => void) | undefined; + const resetPause = new Promise((resolve) => { + releaseReset = resolve; + }); + setPipelineResetPauseForTest(resetPause); + const first = resetPipelineState(); + const second = resetPipelineState(); + let secondSettled = false; + void second.then(() => { + secondSettled = true; + }); + + try { + await Promise.resolve(); + expect(secondSettled).toBe(false); + releaseReset?.(); + await Promise.all([first, second]); + expect(secondSettled).toBe(true); + } finally { + releaseReset?.(); + await first; + setPipelineResetPauseForTest(undefined); + } + }); + + it("admits finalizers after a direct compact route initializes post-reset", async () => { + await resetPipelineState(); + setUpstreamInterceptor( + async () => + new Response(JSON.stringify({ output: [] }), { + headers: { "content-type": "application/json" }, + }), + ); + const compact = new Request("http://gateway.test/v1/responses/compact", { + method: "POST", + headers: { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-project": process.cwd(), + "x-lore-provider": "openai", + "x-lore-upstream-url": "https://api.openai.com/v1", + "x-lore-session-id": "direct-route-initializer", + }, + body: JSON.stringify({ + model: "gpt-5.6-sol", + instructions: "You are a coding agent.", + input: [ + { + role: "user", + content: [{ type: "input_text", text: "compact" }], + }, + ], + tools: [], + }), + }); + + try { + await ( + await handleResponsesCompactEndpoint(compact, loadLocalConfig()) + ).text(); + let postResponses = 0; + setPostResponseStartObserverForTest(() => postResponses++); + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE("resp_after_direct_compact"), { + headers: { "content-type": "text/event-stream" }, + }), + ); + const streamed = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "post-direct-route" }, + }), + loadLocalConfig(), + ); + await streamed.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(postResponses).toBe(1); + } finally { + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("allows authenticated sessions to use global warming controls", async () => { + const sessionHeaders = { + "x-lore-session-id": "authenticated-warming-admin", + }; + const { isWarmingEnabled } = await import("../src/cache-warmer"); + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE("resp_warming_admin"), { + headers: { "content-type": "text/event-stream" }, + }), + ); + const command = (text: string): GatewayRequest => { + const request = makeResponsesRequest({ + sessionHeaders, + messages: [{ role: "user", content: [{ type: "text", text }] }], + }); + request.stream = false; + return request; + }; + + try { + await ( + await handleRequest( + makeResponsesRequest({ sessionHeaders }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const off = await handleRequest( + command("/lore:warm:off"), + loadLocalConfig(), + ); + expect(await off.text()).toContain("Cache warming disabled globally"); + expect(isWarmingEnabled()).toBe(false); + + const on = await handleRequest( + command("/lore:warm:on"), + loadLocalConfig(), + ); + expect(await on.text()).toContain("Cache warming enabled globally"); + expect(isWarmingEnabled()).toBe(true); + + const reset = await handleRequest( + command("/lore:warm:reset"), + loadLocalConfig(), + ); + expect(await reset.text()).toContain( + "Cache warming circuit breaker reset", + ); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("persists a project binding made authoritative by curate", async () => { + const sessionHeaders = { + "x-lore-session-id": "curate-project-binding-session", + }; + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE("resp_curate_project_binding"), { + headers: { "content-type": "text/event-stream" }, + }), + ); + + try { + const provisional = makeResponsesRequest({ sessionHeaders }); + delete provisional.rawHeaders["x-lore-project"]; + await (await handleRequest(provisional, loadLocalConfig())).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const curate = makeResponsesRequest({ + sessionHeaders, + messages: [ + { + role: "user", + content: [{ type: "text", text: "/lore:curate" }], + }, + ], + }); + curate.stream = false; + curate.rawHeaders["x-lore-project"] = "/tmp"; + const response = await handleRequest(curate, loadLocalConfig()); + expect(response.status).toBe(200); + await response.text(); + + const state = [...getActiveSessions().values()].find( + (candidate) => + candidate.headerSessionId === sessionHeaders["x-lore-session-id"], + ); + expect(state).toBeDefined(); + expect(loadSessionTracking(state?.sessionID ?? "")).toMatchObject({ + projectPath: "/tmp", + projectPathProvisional: false, + }); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("rehydrates an evicted Tier-2 session for curate", async () => { + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE("resp_tier2_curate"), { + headers: { "content-type": "text/event-stream" }, + }), + ); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": "tier2-seed-session" }, + }), + loadLocalConfig(), + ) + ).text(); + await resetPipelineState(); + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE("resp_tier2_curate"), { + headers: { "content-type": "text/event-stream" }, + }), + ); + + saveSessionTracking("persisted-tier2-session", { + credentialFingerprint: authFingerprint({ + scheme: "bearer", + value: "test-key", + }), + headerName: "x-custom-session-affinity", + headerSessionId: "tier2-custom-value", + projectPath: process.cwd(), + projectPathProvisional: false, + }); + const curate = makeResponsesRequest({ + sessionHeaders: { "x-custom-session-affinity": "tier2-custom-value" }, + messages: [ + { + role: "user", + content: [{ type: "text", text: "/lore:curate" }], + }, + ], + }); + curate.stream = false; + const response = await handleRequest(curate, loadLocalConfig()); + expect(response.status).toBe(200); + expect(await response.text()).toContain("Curation complete"); + expect(getActiveSessions().has("persisted-tier2-session")).toBe(true); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("fails closed when an unknown canonical slash header conflicts with a known alias", async () => { + const legacyHeader = { "x-session-affinity": "slash-alias-session" }; + const store = vi.spyOn(temporal, "store"); + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE("resp_slash_alias"), { + headers: { "content-type": "text/event-stream" }, + }), + ); + + try { + const established = await handleRequest( + makeResponsesRequest({ sessionHeaders: legacyHeader }), + loadLocalConfig(), + ); + await established.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const slashRequest = (command: string): GatewayRequest => { + const request = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": "new-canonical-alias", + ...legacyHeader, + }, + messages: [ + { role: "user", content: [{ type: "text", text: command }] }, + ], + }); + request.stream = false; + return request; + }; + const curate = await handleRequest( + slashRequest("/lore:curate"), + loadLocalConfig(), + ); + expect(await curate.text()).toContain( + "No active session found for curation", + ); + await ( + await handleRequest(slashRequest("/lore:amnesia:on"), loadLocalConfig()) + ).text(); + + const order: string[] = []; + setPostResponseStartObserverForTest(() => order.push("post")); + store.mockClear(); + const sensitive = await handleRequest( + makeResponsesRequest({ + sessionHeaders: legacyHeader, + messages: [ + { + role: "user", + content: [{ type: "text", text: "sensitive turn" }], + }, + ], + }), + loadLocalConfig(), + ); + await sensitive.text(); + order.push("eof"); + + await ( + await handleRequest( + slashRequest("/lore:amnesia:off"), + loadLocalConfig(), + ) + ).text(); + order.push("slash"); + + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(order).toEqual(["eof", "slash", "post"]); + expect(store).toHaveBeenCalled(); + } finally { + store.mockRestore(); + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("does not confirm canonical migration from an aborted normal turn", async () => { + const alias = "aborted-migration-alias"; + const canonical = "aborted-migration-canonical"; + let upstreamCalls = 0; + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response(validResponsesSSE("resp_migration_setup"), { + headers: { "content-type": "text/event-stream" }, + }); + }); + let release!: () => void; + const paused = new Promise((resolve) => { + release = resolve; + }); + let waitingResolve!: () => void; + const waiting = new Promise((resolve) => { + waitingResolve = resolve; + }); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === alias, + ); + expect(state).toBeDefined(); + + setPipelinePreUpstreamPauseForTest(paused, waitingResolve); + const caller = new AbortController(); + const migration = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }); + migration.signal = caller.signal; + const failed = await handleRequest(migration, loadLocalConfig()); + const failedBody = failed.text(); + await waiting; + caller.abort(new DOMException("caller disconnected", "AbortError")); + release(); + expect(await failedBody).toContain("event: response.failed"); + setPipelinePreUpstreamPauseForTest(undefined); + + const compact = await handleCompactEndpoint( + new Request("http://gateway.test/v1/compact", { + method: "POST", + headers: { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-session-id": canonical, + }, + body: JSON.stringify({ project_path: process.cwd() }), + }), + loadLocalConfig(), + ); + expect(compact.status).toBe(404); + expect(loadSessionTracking(state?.sessionID ?? "")).toMatchObject({ + headerName: "x-session-affinity", + headerSessionId: alias, + }); + expect(upstreamCalls).toBe(1); + + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": canonical }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + expect(getActiveSessions().size).toBe(1); + expect(loadSessionTracking(state?.sessionID ?? "")).toMatchObject({ + headerName: "x-lore-session-id", + headerSessionId: canonical, + }); + expect(upstreamCalls).toBe(2); + } finally { + release(); + setPipelinePreUpstreamPauseForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("converges concurrent first turns with the same canonical header", async () => { + const canonical = "concurrent-first-canonical"; + let upstreamCalls = 0; + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response( + validResponsesSSE(`resp_concurrent_${upstreamCalls}`), + { + headers: { "content-type": "text/event-stream" }, + }, + ); + }); + + try { + const first = handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": canonical }, + }), + loadLocalConfig(), + ); + const second = handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": canonical }, + }), + loadLocalConfig(), + ); + const responses = await Promise.all([first, second]); + await Promise.all(responses.map((response) => response.text())); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const matching = [...getActiveSessions().values()].filter( + (state) => state.headerSessionId === canonical, + ); + expect(matching).toHaveLength(1); + expect(loadSessionTracking(matching[0]?.sessionID ?? "")).toMatchObject({ + headerName: "x-lore-session-id", + headerSessionId: canonical, + }); + expect(upstreamCalls).toBe(2); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("accounts a successful provisional canonical migration exactly once", async () => { + const alias = "accounted-migration-alias"; + const canonical = "accounted-migration-canonical"; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + return new Response( + upstreamCall === 1 + ? validResponsesSSE("resp_accounted_migration_setup") + : validResponsesSSE( + "resp_accounted_migration", + "migration accepted", + { + input_tokens: 1_100, + output_tokens: 100, + input_tokens_details: { cache_write_tokens: 1_000 }, + }, + ), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + const store = vi.spyOn(temporal, "store"); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === alias, + ); + expect(state).toBeDefined(); + clearAllCosts(); + const today = new Date().toISOString().slice(0, 10); + const ledgerBefore = getDailyCostForDay(today); + store.mockClear(); + + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + messages: [ + { + role: "user", + content: [ + { type: "text", text: "account this provisional migration" }, + ], + }, + ], + }), + loadLocalConfig(), + ); + expect(response.status).toBe(200); + await response.text(); + await vi.waitFor(() => { + expect( + getSessionCosts(state?.sessionID ?? "")?.conversation, + ).toMatchObject({ + inputTokens: 100, + outputTokens: 100, + cacheWriteTokens: 1_000, + turns: 1, + }); + }); + + expect(getDailySpend().spend).toBeGreaterThan(0); + expect(getDailyCostForDay(today)).toBeGreaterThan(ledgerBefore); + expect(store).toHaveBeenCalledTimes(2); + expect(loadSessionTracking(state?.sessionID ?? "")).toMatchObject({ + headerName: "x-lore-session-id", + headerSessionId: canonical, + }); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + clearAllCosts(); + } + }); + + it("does not confirm canonical migration when the caller aborts after response.completed", async () => { + const alias = "cancelled-complete-alias"; + const canonical = "cancelled-complete-canonical"; + const caller = new AbortController(); + const store = vi.spyOn(temporal, "store"); + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE("resp_cancelled_complete"), { + headers: { "content-type": "text/event-stream" }, + }), + ); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === alias, + ); + const original = loadSessionTracking(state?.sessionID ?? ""); + clearAllCosts(); + store.mockClear(); + + const request = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }); + request.signal = caller.signal; + const response = await handleRequest(request, loadLocalConfig()); + const reader = response.body?.getReader(); + expect(reader).toBeDefined(); + const decoder = new TextDecoder(); + let output = ""; + while (!output.includes("event: response.completed")) { + const chunk = await reader?.read(); + expect(chunk?.done).toBe(false); + if (chunk?.value) + output += decoder.decode(chunk.value, { stream: true }); + } + caller.abort(new DOMException("client disconnected", "AbortError")); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect( + getSessionCosts(state?.sessionID ?? "")?.conversation, + ).toMatchObject({ inputTokens: 1, outputTokens: 0, turns: 1 }); + expect(store).not.toHaveBeenCalled(); + expect(loadSessionTracking(state?.sessionID ?? "")).toEqual(original); + const compact = await handleCompactEndpoint( + new Request("http://gateway.test/v1/compact", { + method: "POST", + headers: { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-session-id": canonical, + }, + body: JSON.stringify({ project_path: process.cwd() }), + }), + loadLocalConfig(), + ); + expect(compact.status).toBe(404); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + clearAllCosts(); + } + }); + + it("rolls back recall persistence when the caller aborts after the continuation terminal", async () => { + const canonical = "cancelled-recall-canonical"; + const caller = new AbortController(); + const store = vi.spyOn(temporal, "store"); + let upstreamCall = 0; + const args = JSON.stringify({ + query: "one two three four five six seven eight nine recall terms", + }); + setUpstreamInterceptor(async () => { + upstreamCall++; + if (upstreamCall === 1) { + return new Response(validResponsesSSE("resp_recall_abort_setup"), { + headers: { "content-type": "text/event-stream" }, + }); + } + if (upstreamCall === 2) { + return new Response( + responsesEvent("response.created", { + response: { + id: "resp_recall_abort", + model: "gpt-5.6-sol", + status: "in_progress", + }, + }) + + responsesEvent("response.output_item.added", { + output_index: 0, + item: { + type: "function_call", + id: "fc_recall_abort", + call_id: "call_recall_abort", + name: "recall", + arguments: "", + }, + }) + + responsesEvent("response.function_call_arguments.done", { + output_index: 0, + item_id: "fc_recall_abort", + arguments: args, + }) + + responsesEvent("response.output_item.done", { + output_index: 0, + item: { + type: "function_call", + id: "fc_recall_abort", + call_id: "call_recall_abort", + name: "recall", + arguments: args, + status: "completed", + }, + }) + + responsesEvent("response.completed", { + response: { + id: "resp_recall_abort", + model: "gpt-5.6-sol", + status: "completed", + output: [ + { + type: "function_call", + id: "fc_recall_abort", + call_id: "call_recall_abort", + name: "recall", + arguments: args, + status: "completed", + }, + ], + usage: { input_tokens: 10, output_tokens: 1 }, + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + } + return new Response( + validResponsesSSE("resp_recall_abort_final", "final answer"), + { + headers: { "content-type": "text/event-stream" }, + }, + ); + }); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": canonical }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === canonical, + ); + expect(state).toBeDefined(); + const originalTracking = loadSessionTracking(state?.sessionID ?? ""); + store.mockClear(); + + const request = makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": canonical }, + }); + request.signal = caller.signal; + const response = await handleRequest(request, loadLocalConfig()); + const reader = response.body?.getReader(); + expect(reader).toBeDefined(); + const decoder = new TextDecoder(); + let output = ""; + while (!output.includes("event: response.completed")) { + const chunk = await reader?.read(); + expect(chunk?.done).toBe(false); + if (chunk?.value) + output += decoder.decode(chunk.value, { stream: true }); + } + caller.abort(new DOMException("caller disconnected", "AbortError")); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(upstreamCall).toBe(3); + expect(store).not.toHaveBeenCalled(); + expect(state?.recallStore.size).toBe(0); + expect(loadSessionTracking(state?.sessionID ?? "")).toMatchObject({ + headerName: originalTracking?.headerName, + headerSessionId: originalTracking?.headerSessionId, + recallStore: originalTracking?.recallStore, + }); + const compact = await handleCompactEndpoint( + new Request("http://gateway.test/v1/compact", { + method: "POST", + headers: { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-session-id": canonical, + }, + body: JSON.stringify({ project_path: process.cwd() }), + }), + loadLocalConfig(), + ); + expect(compact.status).toBe(200); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("rolls back recall persistence when post-response storage fails", async () => { + const alias = "failed-recall-storage-alias"; + const store = vi.spyOn(temporal, "store").mockImplementation(() => { + throw new Error("temporal storage failed"); + }); + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + return new Response( + upstreamCall === 1 + ? recallResponsesSSE( + "resp_failed_recall_storage", + "one two three four five six seven eight nine storage terms", + ) + : validResponsesSSE( + "resp_failed_recall_storage_final", + "final answer", + ), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ); + expect(await response.text()).toContain("event: response.completed"); + await vi.waitFor(() => expect(store).toHaveBeenCalled()); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === alias, + ); + + expect(upstreamCall).toBe(2); + expect(state?.recallStore.size).toBe(0); + expect( + loadSessionTracking(state?.sessionID ?? "")?.recallStore, + ).toBeNull(); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("rolls back all DB effects when recall commit fails", async () => { + const alias = "failed-recall-commit-atomicity-alias"; + const knowledgeId = ltm.create({ + projectPath: "/test/responses-recall-atomicity/failure-origin", + category: "gotcha", + title: "Failed recall commit terms", + content: + "one two three four five six seven eight nine atomic failure terms", + scope: "project", + crossProject: true, + }); + let commitAttempts = 0; + setRecallPersistenceCommitObserverForTest(() => { + commitAttempts++; + throw new Error("injected recall commit failure"); + }); + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + return new Response( + upstreamCall === 1 + ? recallResponsesSSE( + "resp_failed_recall_commit_atomicity", + "one two three four five six seven eight nine atomic failure terms", + ) + : validResponsesSSE( + "resp_failed_recall_commit_atomicity_final", + "final answer", + ), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ); + const reader = response.body?.getReader(); + expect(reader).toBeDefined(); + const decoder = new TextDecoder(); + let output = ""; + while (!output.includes("event: response.completed")) { + const chunk = await reader?.read(); + expect(chunk?.done).toBe(false); + if (chunk?.value) + output += decoder.decode(chunk.value, { stream: true }); + } + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === alias, + ); + expect(state).toBeDefined(); + const trackingBeforeEof = loadSessionTracking(state?.sessionID ?? ""); + const temporalBeforeEof = db() + .query( + "SELECT COUNT(*) AS count FROM temporal_messages WHERE session_id = ?", + ) + .get(state?.sessionID ?? "") as { count: number }; + + for (;;) { + const chunk = await reader?.read(); + if (chunk?.done) break; + } + await vi.waitFor(() => expect(commitAttempts).toBe(1)); + + const temporalAfterFailure = db() + .query( + "SELECT COUNT(*) AS count FROM temporal_messages WHERE session_id = ?", + ) + .get(state?.sessionID ?? "") as { count: number }; + const trackingAfterFailure = loadSessionTracking(state?.sessionID ?? ""); + expect(temporalAfterFailure.count).toBe(temporalBeforeEof.count); + expect(trackingAfterFailure?.messageCount).toBe( + trackingBeforeEof?.messageCount, + ); + expect(trackingAfterFailure?.turnsSinceCuration).toBe( + trackingBeforeEof?.turnsSinceCuration, + ); + expect(trackingAfterFailure?.recallStore).toBe( + trackingBeforeEof?.recallStore ?? null, + ); + expect(state?.recallStore.size).toBe(0); + expect(ltm.transferCount(knowledgeId)).toBe(0); + } finally { + setRecallPersistenceCommitObserverForTest(undefined); + ltm.remove(knowledgeId); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("commits recall persistence only after successful downstream EOF", async () => { + const alias = "successful-recall-persistence-alias"; + const knowledgeId = ltm.create({ + projectPath: "/test/responses-recall-atomicity/success-origin", + category: "gotcha", + title: "Successful recall persistence terms", + content: "one two three four five six seven eight nine success terms", + scope: "project", + crossProject: true, + }); + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + return new Response( + upstreamCall === 1 + ? recallResponsesSSE( + "resp_successful_recall_persistence", + "one two three four five six seven eight nine success terms", + ) + : validResponsesSSE( + "resp_successful_recall_persistence_final", + "final answer", + ), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + + try { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ); + const reader = response.body?.getReader(); + expect(reader).toBeDefined(); + const decoder = new TextDecoder(); + let output = ""; + while (!output.includes("event: response.completed")) { + const chunk = await reader?.read(); + expect(chunk?.done).toBe(false); + if (chunk?.value) + output += decoder.decode(chunk.value, { stream: true }); + } + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === alias, + ); + expect(state).toBeDefined(); + expect(state?.recallStore.size).toBe(0); + expect( + loadSessionTracking(state?.sessionID ?? "")?.recallStore, + ).toBeNull(); + const temporalBeforeEof = db() + .query( + "SELECT COUNT(*) AS count FROM temporal_messages WHERE session_id = ?", + ) + .get(state?.sessionID ?? "") as { count: number }; + expect(temporalBeforeEof.count).toBe(0); + expect(ltm.transferCount(knowledgeId)).toBe(0); + + for (;;) { + const chunk = await reader?.read(); + if (chunk?.done) break; + } + await vi.waitFor(() => expect(state?.recallStore.size).toBe(1)); + + expect(upstreamCall).toBe(2); + expect( + loadSessionTracking(state?.sessionID ?? "")?.recallStore, + ).not.toBeNull(); + const temporalCount = db() + .query( + "SELECT COUNT(*) AS count FROM temporal_messages WHERE session_id = ?", + ) + .get(state?.sessionID ?? "") as { count: number }; + expect(temporalCount.count).toBeGreaterThan(0); + expect( + loadSessionTracking(state?.sessionID ?? "")?.messageCount, + ).toBeGreaterThan(0); + expect(ltm.transferCount(knowledgeId)).toBeGreaterThan(0); + } finally { + ltm.remove(knowledgeId); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("accounts but does not persist a provisional incomplete response cancelled after its terminal", async () => { + const alias = "cancelled-incomplete-alias"; + const canonical = "cancelled-incomplete-canonical"; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + return new Response( + upstreamCall === 1 + ? validResponsesSSE("resp_cancelled_incomplete_setup") + : incompleteResponsesSSE("resp_cancelled_incomplete"), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + const store = vi.spyOn(temporal, "store"); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === alias, + ); + expect(state).toBeDefined(); + const original = loadSessionTracking(state?.sessionID ?? ""); + clearAllCosts(); + store.mockClear(); + + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }), + loadLocalConfig(), + ); + const reader = response.body?.getReader(); + expect(reader).toBeDefined(); + const decoder = new TextDecoder(); + let output = ""; + while (!output.includes("event: response.incomplete")) { + const chunk = await reader?.read(); + expect(chunk?.done).toBe(false); + if (chunk?.value) + output += decoder.decode(chunk.value, { stream: true }); + } + await reader?.cancel("client disconnected after incomplete terminal"); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect( + getSessionCosts(state?.sessionID ?? "")?.conversation, + ).toMatchObject({ inputTokens: 1, outputTokens: 0, turns: 1 }); + expect(store).not.toHaveBeenCalled(); + expect(loadSessionTracking(state?.sessionID ?? "")).toEqual(original); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + clearAllCosts(); + } + }); + + it("does not persist an established streaming turn cancelled after its terminal", async () => { + const sessionHeader = "cancelled-established-terminal"; + const store = vi.spyOn(temporal, "store"); + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE("resp_cancelled_established"), { + headers: { "content-type": "text/event-stream" }, + }), + ); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": sessionHeader }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === sessionHeader, + ); + expect(state).toBeDefined(); + clearAllCosts(); + store.mockClear(); + + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": sessionHeader }, + }), + loadLocalConfig(), + ); + const reader = response.body?.getReader(); + expect(reader).toBeDefined(); + const decoder = new TextDecoder(); + let output = ""; + while (!output.includes("event: response.completed")) { + const chunk = await reader?.read(); + expect(chunk?.done).toBe(false); + if (chunk?.value) + output += decoder.decode(chunk.value, { stream: true }); + } + await reader?.cancel("client disconnected after terminal event"); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect( + getSessionCosts(state?.sessionID ?? "")?.conversation, + ).toMatchObject({ inputTokens: 1, outputTokens: 0, turns: 1 }); + expect(store).not.toHaveBeenCalled(); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + clearAllCosts(); + } + }); + + it.each(["completed", "incomplete"] as const)( + "drops stale provisional %s accounting after a bounded reset drain", + async (status) => { + const alias = `stale-${status}-accounting-alias`; + const canonical = `stale-${status}-accounting-canonical`; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + return new Response( + upstreamCall === 1 || status === "completed" + ? validResponsesSSE(`resp_stale_${status}_${upstreamCall}`) + : incompleteResponsesSSE("resp_stale_incomplete_accounting"), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + let releaseFinalizer!: () => void; + const finalizerPause = new Promise((resolve) => { + releaseFinalizer = resolve; + }); + let finalizerWaitingResolve!: () => void; + const finalizerWaiting = new Promise((resolve) => { + finalizerWaitingResolve = resolve; + }); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === alias, + ); + expect(state).toBeDefined(); + clearAllCosts(); + + setProvisionalFinalizerPauseForTest( + finalizerPause, + finalizerWaitingResolve, + ); + setPipelineResetSettleTimeoutForTest(0); + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }), + loadLocalConfig(), + ); + expect(response.status).toBe(200); + const body = await response.text(); + expect(body).toContain( + status === "completed" + ? "event: response.completed" + : "event: response.incomplete", + ); + await finalizerWaiting; + await resetPipelineState(); + + clearAllCosts(); + const today = new Date().toISOString().slice(0, 10); + const ledgerBefore = getDailyCostForDay(today); + releaseFinalizer(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(getSessionCosts(state?.sessionID ?? "")).toBeNull(); + expect(getDailyCostForDay(today)).toBe(ledgerBefore); + } finally { + releaseFinalizer(); + setPipelineResetSettleTimeoutForTest(); + setProvisionalFinalizerPauseForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + clearAllCosts(); + } + }, + ); + + it("strips context markers before provisional migration reaches upstream", async () => { + const alias = "marker-provisional-alias"; + let upstreamCall = 0; + const forwardedBodies: string[] = []; + setUpstreamInterceptor(async (body) => { + upstreamCall++; + forwardedBodies.push(JSON.stringify(body)); + return new Response(validResponsesSSE(`resp_marker_${upstreamCall}`), { + headers: { "content-type": "text/event-stream" }, + }); + }); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": "marker-provisional-canonical", + "x-session-affinity": alias, + }, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "continue\n[lore:session-id=secret-marker]\n[lore:project=/secret/path]", + }, + ], + }, + ], + }), + loadLocalConfig(), + ); + await response.text(); + expect(forwardedBodies.at(-1)).not.toContain("lore:session-id"); + expect(forwardedBodies.at(-1)).not.toContain("lore:project"); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it.each(["provider", "malformed", "missing-terminal", "transport"] as const)( + "does not confirm canonical migration after a %s Responses failure", + async (failure) => { + const alias = `failed-${failure}-alias`; + const canonical = `failed-${failure}-canonical`; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + if (upstreamCall === 1) { + return new Response(validResponsesSSE(`resp_${failure}_setup`), { + headers: { "content-type": "text/event-stream" }, + }); + } + if (failure === "provider") { + return new Response( + responsesEvent("response.failed", { + response: { + id: `resp_${failure}`, + model: "gpt-5.6-sol", + status: "failed", + error: { type: "server_error", message: "provider failed" }, + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + } + if (failure === "malformed") { + return new Response("event: response.created\ndata: {bad}\n\n", { + headers: { "content-type": "text/event-stream" }, + }); + } + if (failure === "missing-terminal") { + return new Response( + responsesEvent("response.created", { + response: { id: `resp_${failure}`, status: "in_progress" }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + } + return new Response( + new ReadableStream({ + pull(controller) { + controller.error(new Error("transport failed")); + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + const store = vi.spyOn(temporal, "store"); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === alias, + ); + expect(state).toBeDefined(); + const originalTracking = loadSessionTracking(state?.sessionID ?? ""); + store.mockClear(); + + const migration = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }); + migration.rawHeaders["x-lore-project"] = + "/tmp/untrusted-provisional-project"; + const failed = await handleRequest(migration, loadLocalConfig()); + expect(await failed.text()).toContain("event: response.failed"); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const compact = await handleCompactEndpoint( + new Request("http://gateway.test/v1/compact", { + method: "POST", + headers: { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-session-id": canonical, + }, + body: JSON.stringify({ project_path: process.cwd() }), + }), + loadLocalConfig(), + ); + expect(compact.status).toBe(404); + expect(store).not.toHaveBeenCalled(); + expect(loadSessionTracking(state?.sessionID ?? "")).toEqual( + originalTracking, + ); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }, + ); + + it.each(["failed", "cancelled", "incomplete", "in_progress"] as const)( + "does not confirm canonical migration after a non-stream %s Responses body", + async (status) => { + const alias = `nonstream-${status}-alias`; + const canonical = `nonstream-${status}-canonical`; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + if (upstreamCall === 1) { + return new Response(validResponsesSSE(`resp_${status}_setup`), { + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response( + JSON.stringify({ + id: `resp_nonstream_${status}`, + object: "response", + created_at: 0, + model: "__test_fake_model__", + status, + output: [ + { + type: "message", + id: `msg_nonstream_${status}`, + role: "assistant", + status, + content: [ + { + type: "output_text", + text: "must not persist", + annotations: [], + }, + ], + }, + ], + usage: { + input_tokens: 1_100, + output_tokens: 100, + input_tokens_details: { cache_write_tokens: 1_000 }, + }, + }), + { headers: { "content-type": "application/json" } }, + ); + }); + const store = vi.spyOn(temporal, "store"); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === alias, + ); + expect(state).toBeDefined(); + saveSessionTracking(state?.sessionID ?? "", { + resolvedConversationTTL: "1h", + }); + expect( + evictLiveSessionForTest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + ), + ).toBe(true); + clearAllCosts(); + store.mockClear(); + + const failed = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }); + failed.stream = false; + const response = await handleRequest(failed, loadLocalConfig()); + if (status === "incomplete") { + expect(response.status).toBe(200); + const body = (await response.json()) as Record; + expect(body.status).toBe("incomplete"); + expect(body.incomplete_details).toEqual({ + reason: "max_output_tokens", + }); + } else { + expect(response.status).toBe(502); + expect(await response.text()).toContain("Gateway request failed"); + } + await vi.waitFor(() => { + expect( + getSessionCosts(state?.sessionID ?? "")?.conversation, + ).toMatchObject({ + inputTokens: 100, + outputTokens: 100, + cacheWriteTokens: 1_000, + turns: 1, + }); + }); + expect( + getSessionCosts(state?.sessionID ?? "")?.conversation.cost, + ).toBeCloseTo( + computeCallCost( + "__test_fake_model__", + { + input_tokens: 100, + output_tokens: 100, + cache_creation_input_tokens: 1_000, + }, + "conversation", + "1h", + ).total, + ); + expect(store).not.toHaveBeenCalled(); + + const compact = await handleCompactEndpoint( + new Request("http://gateway.test/v1/compact", { + method: "POST", + headers: { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-session-id": canonical, + }, + body: JSON.stringify({ project_path: process.cwd() }), + }), + loadLocalConfig(), + ); + expect(compact.status).toBe(404); + expect(loadSessionTracking(state?.sessionID ?? "")).toMatchObject({ + headerName: "x-session-affinity", + headerSessionId: alias, + }); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + clearAllCosts(); + } + }, + ); + + it.each( + ( + [ + { + protocol: "anthropic", + provider: "anthropic", + upstream: "https://api.anthropic.com", + malformed: {}, + }, + { + protocol: "openai", + provider: "openai", + upstream: "https://api.openai.com", + malformed: { choices: [] }, + }, + { + protocol: "openai-responses", + provider: "openai", + upstream: "https://api.openai.com", + malformed: { status: "completed" }, + }, + { + protocol: "gemini", + provider: "google", + upstream: "https://generativelanguage.googleapis.com", + malformed: {}, + }, + ] as const + ).flatMap((entry) => [ + { ...entry, shape: "malformed", body: entry.malformed }, + { + ...entry, + shape: "error-envelope", + body: { error: { type: "server_error", message: "provider failed" } }, + }, + ]), + )( + "does not confirm canonical migration after a $protocol 2xx $shape body", + async ({ protocol, provider, upstream, body }) => { + const alias = `nonstream-${protocol}-alias`; + const canonical = `nonstream-${protocol}-canonical`; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + if (upstreamCall === 1) { + return new Response(validResponsesSSE(`resp_${protocol}_setup`), { + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }); + const store = vi.spyOn(temporal, "store"); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === alias, + ); + expect(state).toBeDefined(); + const originalTracking = loadSessionTracking(state?.sessionID ?? ""); + store.mockClear(); + + const migration = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }); + migration.protocol = protocol; + migration.stream = false; + migration.rawHeaders["x-lore-provider"] = provider; + migration.rawHeaders["x-lore-upstream-url"] = upstream; + const response = await handleRequest(migration, loadLocalConfig()); + expect(response.status).toBe(502); + expect(await response.text()).toContain("Gateway request failed"); + await new Promise((resolve) => setImmediate(resolve)); + + expect(store).not.toHaveBeenCalled(); + expect(loadSessionTracking(state?.sessionID ?? "")).toEqual( + originalTracking, + ); + const compact = await handleCompactEndpoint( + new Request("http://gateway.test/v1/compact", { + method: "POST", + headers: { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-session-id": canonical, + }, + body: JSON.stringify({ project_path: process.cwd() }), + }), + loadLocalConfig(), + ); + expect(compact.status).toBe(404); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }, + ); + + it("does not migrate a confirmed alias across confident projects", async () => { + const alias = "cross-project-migration-alias"; + const canonical = "cross-project-migration-canonical"; + const projectA = "/tmp/lore-cross-header-project-a"; + const projectB = "/tmp/lore-cross-header-project-b"; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + return new Response(validResponsesSSE(`resp_cross_${upstreamCall}`), { + headers: { "content-type": "text/event-stream" }, + }); + }); + + try { + const established = makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }); + established.rawHeaders["x-lore-project"] = projectA; + await (await handleRequest(established, loadLocalConfig())).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const original = [...getActiveSessions().values()].find( + (state) => state.headerSessionId === alias, + ); + expect(original).toMatchObject({ + projectPath: projectA, + projectPathProvisional: false, + }); + + const migration = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }); + migration.rawHeaders["x-lore-project"] = projectB; + expect( + await (await handleRequest(migration, loadLocalConfig())).text(), + ).toContain("event: response.completed"); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(original).toMatchObject({ + headerName: "x-session-affinity", + headerSessionId: alias, + projectPath: projectA, + projectPathProvisional: false, + }); + const independent = [...getActiveSessions().values()].find( + (state) => state.headerSessionId === canonical, + ); + expect(independent).toMatchObject({ + headerName: "x-lore-session-id", + projectPath: projectB, + projectPathProvisional: false, + }); + expect(independent?.sessionID).not.toBe(original?.sessionID); + expect(upstreamCall).toBe(2); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("rechecks project binding after a concurrent first turn claims the fallback session", async () => { + const alias = "concurrent-project-migration-alias"; + const canonical = "concurrent-project-migration-canonical"; + const projectA = "/tmp/lore-concurrent-project-a"; + const projectB = "/tmp/lore-concurrent-project-b"; + let releaseFirst!: () => void; + const firstPause = new Promise((resolve) => { + releaseFirst = resolve; + }); + let firstWaitingResolve!: () => void; + const firstWaiting = new Promise((resolve) => { + firstWaitingResolve = resolve; + }); + let upstreamCalls = 0; + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response( + validResponsesSSE(`resp_project_race_${upstreamCalls}`), + { + headers: { "content-type": "text/event-stream" }, + }, + ); + }); + setBeforeUpstreamCaptureForTest(async (request) => { + if ( + request.rawHeaders["x-session-affinity"] === alias && + !request.rawHeaders["x-lore-session-id"] + ) { + firstWaitingResolve(); + await firstPause; + } + }); + + try { + const first = makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }); + first.rawHeaders["x-lore-project"] = projectA; + const firstResponse = await handleRequest(first, loadLocalConfig()); + const firstBody = firstResponse.text(); + await firstWaiting; + + const migration = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }); + migration.rawHeaders["x-lore-project"] = projectB; + const migrationResponse = await handleRequest( + migration, + loadLocalConfig(), + ); + const migrationBody = migrationResponse.text(); + await vi.waitFor(() => + expect(pendingPipelineSessionClaimCountForTest()).toBe(1), + ); + + releaseFirst(); + expect(await firstBody).toContain("event: response.completed"); + const boundAfterFirst = [...getActiveSessions().values()].find( + (state) => state.headerSessionId === alias, + ); + expect(boundAfterFirst).toMatchObject({ + projectPath: projectA, + projectPathProvisional: false, + }); + expect(await migrationBody).toContain("event: response.failed"); + expect(upstreamCalls).toBe(1); + + setBeforeUpstreamCaptureForTest(undefined); + const retry = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }); + retry.rawHeaders["x-lore-project"] = projectB; + expect( + await (await handleRequest(retry, loadLocalConfig())).text(), + ).toContain("event: response.completed"); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const original = [...getActiveSessions().values()].find( + (state) => state.headerSessionId === alias, + ); + const independent = [...getActiveSessions().values()].find( + (state) => state.headerSessionId === canonical, + ); + expect(original).toMatchObject({ + projectPath: projectA, + projectPathProvisional: false, + }); + expect(independent).toMatchObject({ + projectPath: projectB, + projectPathProvisional: false, + }); + expect(independent?.sessionID).not.toBe(original?.sessionID); + expect(upstreamCalls).toBe(2); + } finally { + releaseFirst(); + setBeforeUpstreamCaptureForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("does not migrate a confirmed alias across confidently inferred projects", async () => { + const alias = "inferred-project-migration-alias"; + const canonical = "inferred-project-migration-canonical"; + const projectA = "/tmp/lore-inferred-migration-a"; + const projectB = "/tmp/lore-inferred-migration-b"; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + return new Response(validResponsesSSE(`resp_inferred_${upstreamCall}`), { + headers: { "content-type": "text/event-stream" }, + }); + }); + + try { + const established = makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }); + established.rawHeaders["x-lore-project"] = projectA; + await (await handleRequest(established, loadLocalConfig())).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const original = [...getActiveSessions().values()].find( + (state) => state.headerSessionId === alias, + ); + + const migration = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }); + delete migration.rawHeaders["x-lore-project"]; + migration.system = `You are a coding agent.\nWorking directory: ${projectB}`; + await (await handleRequest(migration, loadLocalConfig())).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(original).toMatchObject({ + headerName: "x-session-affinity", + headerSessionId: alias, + projectPath: projectA, + }); + const independent = [...getActiveSessions().values()].find( + (state) => state.headerSessionId === canonical, + ); + expect(independent).toMatchObject({ + headerName: "x-lore-session-id", + projectPath: projectB, + }); + expect(independent?.sessionID).not.toBe(original?.sessionID); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it.each([false, true])( + "does not confirm canonical migration after a blocked Gemini response (stream=%s)", + async (stream) => { + const alias = `blocked-gemini-${stream}-alias`; + const canonical = `blocked-gemini-${stream}-canonical`; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + if (upstreamCall === 1) { + return new Response(validResponsesSSE("resp_gemini_block_setup"), { + headers: { "content-type": "text/event-stream" }, + }); + } + const blocked = { + responseId: "gemini-blocked", + modelVersion: "gemini-test", + promptFeedback: { blockReason: "SAFETY" }, + usageMetadata: { + promptTokenCount: 1, + candidatesTokenCount: 0, + totalTokenCount: 1, + }, + }; + return new Response( + stream + ? `data: ${JSON.stringify(blocked)}\n\n` + : JSON.stringify(blocked), + { + headers: { + "content-type": stream ? "text/event-stream" : "application/json", + }, + }, + ); + }); + const store = vi.spyOn(temporal, "store"); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => candidate.headerSessionId === alias, + ); + const original = loadSessionTracking(state?.sessionID ?? ""); + store.mockClear(); + + const migration = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }); + migration.protocol = "gemini"; + migration.stream = stream; + migration.rawHeaders["x-lore-provider"] = "google"; + migration.rawHeaders["x-lore-upstream-url"] = + "https://generativelanguage.googleapis.com"; + await (await handleRequest(migration, loadLocalConfig())).text(); + await new Promise((resolve) => setImmediate(resolve)); + + expect(store).not.toHaveBeenCalled(); + expect(loadSessionTracking(state?.sessionID ?? "")).toEqual(original); + expect(state).toMatchObject({ + headerName: "x-session-affinity", + headerSessionId: alias, + }); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }, + ); + + it("rejects a provisional canonical migration that conflicts with a confirmed alias", async () => { + const aliasA = "provisional-conflict-alias-a"; + const aliasB = "provisional-conflict-alias-b"; + const canonical = "provisional-conflict-canonical"; + const projectA = "/tmp/lore-provisional-conflict-a"; + const projectB = "/tmp/lore-provisional-conflict-b"; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + return new Response(validResponsesSSE(`resp_conflict_${upstreamCall}`), { + headers: { "content-type": "text/event-stream" }, + }); + }); + const establish = async (alias: string, project: string): Promise => { + const request = makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }); + request.rawHeaders["x-lore-project"] = project; + await (await handleRequest(request, loadLocalConfig())).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + }; + let release!: () => void; + const paused = new Promise((resolve) => { + release = resolve; + }); + let waitingResolve!: () => void; + const waiting = new Promise((resolve) => { + waitingResolve = resolve; + }); + + try { + await establish(aliasA, projectA); + await establish(aliasB, projectB); + const stateA = [...getActiveSessions().values()].find( + (state) => state.headerSessionId === aliasA, + ); + const stateB = [...getActiveSessions().values()].find( + (state) => state.headerSessionId === aliasB, + ); + expect(stateA?.sessionID).not.toBe(stateB?.sessionID); + + setPipelinePreUpstreamPauseForTest(paused, waitingResolve); + const first = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": aliasA, + }, + }); + first.rawHeaders["x-lore-project"] = projectA; + const firstResponse = await handleRequest(first, loadLocalConfig()); + const firstBody = firstResponse.text(); + await waiting; + setPipelinePreUpstreamPauseForTest(undefined); + + const conflicting = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": aliasB, + }, + }); + conflicting.rawHeaders["x-lore-project"] = projectB; + const conflictingResponse = await handleRequest( + conflicting, + loadLocalConfig(), + ); + const conflictingBody = conflictingResponse.text(); + release(); + + expect(await firstBody).toContain("event: response.completed"); + expect(await conflictingBody).toContain("event: response.failed"); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(stateA).toMatchObject({ + headerName: "x-lore-session-id", + headerSessionId: canonical, + projectPath: projectA, + }); + expect(stateB).toMatchObject({ + headerName: "x-session-affinity", + headerSessionId: aliasB, + projectPath: projectB, + }); + expect(upstreamCall).toBe(3); + } finally { + release(); + setPipelinePreUpstreamPauseForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("does not confirm expired provisional ownership after another session claims the canonical header", async () => { + const aliasA = "expired-owner-alias-a"; + const aliasB = "expired-owner-alias-b"; + const canonical = "expired-owner-canonical"; + const projectA = "/tmp/lore-expired-owner-a"; + const projectB = "/tmp/lore-expired-owner-b"; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + if (upstreamCall === 3) { + return new Response(JSON.stringify({ error: "validation failed" }), { + status: 500, + headers: { "content-type": "application/json" }, + }); + } + return new Response(validResponsesSSE(`resp_expired_${upstreamCall}`), { + headers: { "content-type": "text/event-stream" }, + }); + }); + const establish = async (alias: string, project: string): Promise => { + const request = makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": alias }, + }); + request.rawHeaders["x-lore-project"] = project; + await (await handleRequest(request, loadLocalConfig())).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + }; + let release!: () => void; + const paused = new Promise((resolve) => { + release = resolve; + }); + let waitingResolve!: () => void; + const waiting = new Promise((resolve) => { + waitingResolve = resolve; + }); + + try { + await establish(aliasA, projectA); + await establish(aliasB, projectB); + const stateA = [...getActiveSessions().values()].find( + (state) => state.headerSessionId === aliasA, + ); + const stateB = [...getActiveSessions().values()].find( + (state) => state.headerSessionId === aliasB, + ); + const request = (alias: string, project: string): GatewayRequest => { + const result = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": canonical, + "x-session-affinity": alias, + }, + }); + result.rawHeaders["x-lore-project"] = project; + return result; + }; + + await ( + await handleRequest(request(aliasA, projectA), loadLocalConfig()) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const costsBeforeExpiredRetry = structuredClone( + getSessionCosts(stateA?.sessionID ?? "")?.conversation, + ); + setPipelinePreUpstreamPauseForTest(paused, waitingResolve); + const retryA = await handleRequest( + request(aliasA, projectA), + loadLocalConfig(), + ); + const retryABody = retryA.text(); + await waiting; + setPipelinePreUpstreamPauseForTest(undefined); + expireProvisionalHeaderMappingsForTest(); + + await ( + await handleRequest(request(aliasB, projectB), loadLocalConfig()) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + release(); + await retryABody; + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(stateA).toMatchObject({ + headerName: "x-session-affinity", + headerSessionId: aliasA, + projectPath: projectA, + }); + expect(stateB).toMatchObject({ + headerName: "x-lore-session-id", + headerSessionId: canonical, + projectPath: projectB, + }); + expect( + [...getActiveSessions().values()].filter( + (state) => state.headerSessionId === canonical, + ), + ).toHaveLength(1); + expect(getSessionCosts(stateA?.sessionID ?? "")?.conversation).toEqual( + costsBeforeExpiredRetry, + ); + } finally { + release(); + setPipelinePreUpstreamPauseForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("does not learn Tier-2 header evidence from failed requests", async () => { + const candidateHeader = "x-candidate-session"; + const globalHeader = "x-global-session"; + const candidateValue = "candidate-session-a"; + const otherCandidateValue = "candidate-session-b"; + const globalValue = "global-session-a"; + const failedGlobalValue = "global-session-b"; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + if (upstreamCall === 3 || upstreamCall === 4) { + return new Response(JSON.stringify({ error: "upstream failed" }), { + status: 500, + headers: { "content-type": "application/json" }, + }); + } + return new Response(validResponsesSSE(`resp_learning_${upstreamCall}`), { + headers: { "content-type": "text/event-stream" }, + }); + }); + const messages = (seed: string, turn: number): GatewayRequest["messages"] => + Array.from({ length: turn * 2 - 1 }, (_, index) => ({ + role: index % 2 === 0 ? ("user" as const) : ("assistant" as const), + content: [ + { + type: "text" as const, + text: index === 0 ? seed : `${seed} turn ${index}`, + }, + ], + })); + const turn = async ( + sessionHeaders: Record, + seed: string, + number: number, + succeeds: boolean, + ): Promise => { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders, + messages: messages(seed, number), + }), + loadLocalConfig(), + ); + const body = await response.text(); + expect(body).toContain( + succeeds ? "event: response.completed" : "event: response.failed", + ); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + }; + + try { + // Establish legitimate uniqueness only for candidateHeader. + await turn( + { [candidateHeader]: otherCandidateValue }, + "other successful session", + 1, + true, + ); + await turn( + { + [candidateHeader]: candidateValue, + [globalHeader]: globalValue, + }, + "primary session", + 1, + true, + ); + const primary = [...getActiveSessions().values()].find( + (state) => + state.candidateHeaders?.get(globalHeader)?.value === globalValue, + ); + expect(primary?.candidateHeaders?.get(candidateHeader)?.seenCount).toBe( + 1, + ); + expect(primary?.candidateHeaders?.get(globalHeader)?.seenCount).toBe(1); + + // A failed new session must not add global uniqueness. + await turn( + { [globalHeader]: failedGlobalValue }, + "failed distinct session", + 1, + false, + ); + expect( + [...getActiveSessions().values()].some( + (state) => + state.candidateHeaders?.get(globalHeader)?.value === + failedGlobalValue, + ), + ).toBe(false); + + // A failed matched turn must not advance the primary candidates. + await turn( + { + [candidateHeader]: candidateValue, + [globalHeader]: globalValue, + }, + "primary session", + 2, + false, + ); + expect(primary?.candidateHeaders?.get(candidateHeader)?.seenCount).toBe( + 1, + ); + expect(primary?.candidateHeaders?.get(globalHeader)?.seenCount).toBe(1); + + // The retry is only the second successful observation. If the failed + // matched turn counted, candidateHeader would promote here. + await turn( + { + [candidateHeader]: candidateValue, + [globalHeader]: globalValue, + }, + "primary session", + 2, + true, + ); + expect(primary?.headerSessionId).toBeUndefined(); + expect(primary?.candidateHeaders?.get(candidateHeader)?.seenCount).toBe( + 2, + ); + expect(primary?.candidateHeaders?.get(globalHeader)?.seenCount).toBe(2); + + // The third successful globalHeader observation remains non-unique. If + // the failed new session counted globally, it would promote here. + await turn({ [globalHeader]: globalValue }, "primary session", 3, true); + expect(primary?.headerSessionId).toBeUndefined(); + expect(primary?.candidateHeaders?.get(globalHeader)?.seenCount).toBe(3); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("keeps failed Tier-2 promotion retries provisional until success", async () => { + const headerName = "x-retry-session"; + const targetValue = "retry-session-target"; + const distinctValue = "retry-session-other"; + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + if (upstreamCall === 4 || upstreamCall === 5) { + return new Response(JSON.stringify({ error: "upstream failed" }), { + status: 500, + headers: { "content-type": "application/json" }, + }); + } + return new Response(validResponsesSSE(`resp_retry_${upstreamCall}`), { + headers: { "content-type": "text/event-stream" }, + }); + }); + const messages = (seed: string, turn: number): GatewayRequest["messages"] => + Array.from({ length: turn * 2 - 1 }, (_, index) => ({ + role: index % 2 === 0 ? ("user" as const) : ("assistant" as const), + content: [ + { + type: "text" as const, + text: index === 0 ? seed : `${seed} turn ${index}`, + }, + ], + })); + const turn = async ( + headerValue: string, + seed: string, + number: number, + succeeds: boolean, + ): Promise => { + const response = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { [headerName]: headerValue }, + messages: messages(seed, number), + }), + loadLocalConfig(), + ); + const body = await response.text(); + expect(body).toContain( + succeeds ? "event: response.completed" : "event: response.failed", + ); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + }; + const store = vi.spyOn(temporal, "store"); + + try { + await turn(distinctValue, "other session", 1, true); + await turn(targetValue, "target session", 1, true); + await turn(targetValue, "target session", 2, true); + const target = [...getActiveSessions().values()].find( + (state) => + state.candidateHeaders?.get(headerName)?.value === targetValue, + ); + expect(target).toBeDefined(); + expect(target?.messageCount).toBe(3); + expect(target?.candidateHeaders?.get(headerName)?.seenCount).toBe(2); + store.mockClear(); + + // The third observation promotes only provisionally, and provider failure + // must leave all session-owned state unchanged. + await turn(targetValue, "target session", 3, false); + expect(target?.messageCount).toBe(3); + expect(target?.candidateHeaders?.get(headerName)?.seenCount).toBe(2); + expect(target?.headerName).toBeUndefined(); + expect(target?.headerSessionId).toBeUndefined(); + expect(store).not.toHaveBeenCalled(); + + // A retry resolved from the provisional index must remain on the same + // validation-only path rather than entering the full pipeline early. + await turn(targetValue, "target session", 3, false); + expect(target?.messageCount).toBe(3); + expect(target?.candidateHeaders?.get(headerName)?.seenCount).toBe(2); + expect(target?.headerName).toBeUndefined(); + expect(target?.headerSessionId).toBeUndefined(); + expect(store).not.toHaveBeenCalled(); + + const slash = makeResponsesRequest({ + sessionHeaders: { [headerName]: targetValue }, + messages: [ + { + role: "user", + content: [{ type: "text", text: "/lore:amnesia:on" }], + }, + ], + }); + const slashResponse = await handleRequest(slash, loadLocalConfig()); + expect(await slashResponse.text()).toContain( + "Amnesia mode was not changed", + ); + expect(target?.amnesia).toBe(false); + + // Even a validated upstream completion does not publish until the client + // consumes EOF and the post-response finalizer commits the turn. + const validation = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { [headerName]: targetValue }, + messages: messages("target session", 3), + }), + loadLocalConfig(), + ); + expect(target?.headerSessionId).toBeUndefined(); + expect(await validation.text()).toContain("event: response.completed"); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + expect(target?.headerName).toBe(headerName); + expect(target?.headerSessionId).toBe(targetValue); + expect(target?.messageCount).toBe(5); + + await turn(targetValue, "target session", 4, true); + expect(target?.messageCount).toBe(7); + expect(upstreamCall).toBe(7); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("rejects slash commands with ambiguous promoted Tier-2 headers", async () => { + let upstreamCall = 0; + setUpstreamInterceptor(async () => { + upstreamCall++; + return new Response(validResponsesSSE(`resp_tier2_${upstreamCall}`), { + headers: { "content-type": "text/event-stream" }, + }); + }); + const messages = (seed: string, turn: number): GatewayRequest["messages"] => + Array.from({ length: turn * 2 - 1 }, (_, index) => ({ + role: index % 2 === 0 ? ("user" as const) : ("assistant" as const), + content: [ + { + type: "text" as const, + text: index === 0 ? seed : `${seed} turn ${index}`, + }, + ], + })); + const turn = async ( + headerName: string, + headerValue: string, + seed: string, + number: number, + ): Promise => { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { [headerName]: headerValue }, + messages: messages(seed, number), + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + }; + + try { + await turn("x-alpha-session", "alpha-session-value", "alpha seed", 1); + await turn("x-alpha-session", "alpha-other-value", "alpha other", 1); + await turn("x-alpha-session", "alpha-session-value", "alpha seed", 2); + await turn("x-alpha-session", "alpha-session-value", "alpha seed", 3); + + await turn("x-beta-session", "beta-session-value", "beta seed", 1); + await turn("x-beta-session", "beta-other-value", "beta other", 1); + await turn("x-beta-session", "beta-session-value", "beta seed", 2); + await turn("x-beta-session", "beta-session-value", "beta seed", 3); + + const alpha = [...getActiveSessions().values()].find( + (state) => + state.headerName === "x-alpha-session" && + state.headerSessionId === "alpha-session-value", + ); + const beta = [...getActiveSessions().values()].find( + (state) => + state.headerName === "x-beta-session" && + state.headerSessionId === "beta-session-value", + ); + expect(alpha).toBeDefined(); + expect(beta).toBeDefined(); + expect(alpha?.sessionID).not.toBe(beta?.sessionID); + + const ambiguous = makeResponsesRequest({ + sessionHeaders: { + "x-alpha-session": "alpha-session-value", + "x-beta-session": "beta-session-value", + }, + messages: [ + { + role: "user", + content: [{ type: "text", text: "/lore:amnesia:on" }], + }, + ], + }); + ambiguous.stream = false; + const response = await handleRequest(ambiguous, loadLocalConfig()); + expect(await response.text()).toContain("Amnesia mode was not changed"); + expect(alpha?.amnesia).toBe(false); + expect(beta?.amnesia).toBe(false); + expect(upstreamCall).toBe(8); + + const normalAmbiguous = makeResponsesRequest({ + sessionHeaders: { + "x-alpha-session": "alpha-session-value", + "x-beta-session": "beta-session-value", + }, + messages: messages("alpha seed", 4), + }); + const normal = await handleRequest(normalAmbiguous, loadLocalConfig()); + expect(await normal.text()).toContain("event: response.failed"); + expect(upstreamCall).toBe(8); + expect(alpha?.messageCount).toBe(messages("alpha seed", 3).length); + expect(beta?.messageCount).toBe(messages("beta seed", 3).length); + } finally { + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("rejects unbound structural compaction before reading project memory", async () => { + const victimAlias = "structural-compaction-victim-alias"; + let upstreamCalls = 0; + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response(validResponsesSSE("resp_structural_victim"), { + headers: { "content-type": "text/event-stream" }, + }); + }); + const undistilled = vi.spyOn(temporal, "undistilled"); + + try { + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": victimAlias }, + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const provisional = makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": "provisional-structural-session", + }, + }); + delete provisional.rawHeaders["x-lore-project"]; + await (await handleRequest(provisional, loadLocalConfig())).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + undistilled.mockClear(); + + const attack = async ( + sessionHeaders: Record, + credential: string | null, + projectPath = process.cwd(), + ): Promise => { + const request = makeResponsesRequest({ + sessionHeaders, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "Create an anchored summary from the conversation history above.", + }, + ], + }, + ], + tools: [], + }); + request.stream = false; + request.rawHeaders["x-lore-project"] = projectPath; + if (credential) request.rawHeaders.authorization = credential; + else delete request.rawHeaders.authorization; + return handleRequest(request, loadLocalConfig()); + }; + + const fresh = await attack( + { "x-lore-session-id": "new-structural-attacker" }, + "Bearer test-key", + ); + expect(fresh.status).toBe(404); + expect(await fresh.text()).not.toContain("structural victim"); + + const conflictingAlias = await attack( + { + "x-lore-session-id": "unknown-structural-canonical", + "x-session-affinity": victimAlias, + }, + "Bearer test-key", + ); + expect(conflictingAlias.status).toBe(404); + expect(await conflictingAlias.text()).not.toContain("structural victim"); + + const missingCredential = await attack( + { "x-session-affinity": victimAlias }, + null, + ); + expect(missingCredential.status).toBe(400); + + const wrongCredential = await attack( + { "x-session-affinity": victimAlias }, + "Bearer wrong-tenant-key", + ); + expect(wrongCredential.status).toBe(404); + + const provisionalRebind = await attack( + { "x-lore-session-id": "provisional-structural-session" }, + "Bearer test-key", + "/tmp", + ); + expect(provisionalRebind.status).toBe(403); + + expect(undistilled).not.toHaveBeenCalled(); + expect(upstreamCalls).toBe(2); + } finally { + undistilled.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("does not fall through an indexed canonical session to a conflicting alias", async () => { + let upstreamCalls = 0; + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response( + validResponsesSSE(`resp_alias_conflict_${upstreamCalls}`), + { + headers: { "content-type": "text/event-stream" }, + }, + ); + }); + const canonicalHeaders = { + "x-lore-session-id": "authoritative-canonical-session", + }; + const fallbackHeaders = { + "x-session-affinity": "conflicting-fallback-session", + }; + const canonicalRequest = makeResponsesRequest({ + sessionHeaders: canonicalHeaders, + }); + const store = vi.spyOn(temporal, "store"); + + try { + await (await handleRequest(canonicalRequest, loadLocalConfig())).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + await ( + await handleRequest( + makeResponsesRequest({ sessionHeaders: fallbackHeaders }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + expect(evictLiveSessionForTest(canonicalRequest)).toBe(true); + + const slash = makeResponsesRequest({ + sessionHeaders: { ...canonicalHeaders, ...fallbackHeaders }, + messages: [ + { + role: "user", + content: [{ type: "text", text: "/lore:amnesia:on" }], + }, + ], + }); + slash.stream = false; + await (await handleRequest(slash, loadLocalConfig())).text(); + + store.mockClear(); + await ( + await handleRequest( + makeResponsesRequest({ + sessionHeaders: canonicalHeaders, + messages: [ + { + role: "user", + content: [{ type: "text", text: "canonical sensitive turn" }], + }, + ], + }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + expect(store).not.toHaveBeenCalled(); + + store.mockClear(); + await ( + await handleRequest( + makeResponsesRequest({ sessionHeaders: fallbackHeaders }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(store).toHaveBeenCalled(); + } finally { + store.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("cancels a request waiting behind an unrelated downstream finalizer", async () => { + const sessionHeaders = { + "x-lore-session-id": "cancel-finalizer-wait-session", + }; + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE("resp_waiter_setup"), { + headers: { "content-type": "text/event-stream" }, + }), + ); + let release!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + let finalizerStarted!: () => void; + const started = new Promise((resolve) => { + finalizerStarted = resolve; + }); + let pending: Promise | undefined; + + try { + await ( + await handleRequest( + makeResponsesRequest({ sessionHeaders }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => + candidate.headerSessionId === sessionHeaders["x-lore-session-id"], + ); + expect(state).toBeDefined(); + scheduleStreamingPostResponseForTest(state?.sessionID ?? "", async () => { + finalizerStarted(); + await blocked; + }); + await started; + + const caller = new AbortController(); + const slash = makeResponsesRequest({ + sessionHeaders, + messages: [ + { + role: "user", + content: [{ type: "text", text: "/lore:amnesia:on" }], + }, + ], + }); + slash.stream = false; + slash.signal = caller.signal; + pending = handleRequest(slash, loadLocalConfig()); + caller.abort(new DOMException("caller disconnected", "AbortError")); + + const outcome = await Promise.race([ + pending.then((response) => response.status), + new Promise<"pending">((resolve) => + setImmediate(() => resolve("pending")), + ), + ]); + expect(outcome).toBe(502); + } finally { + release(); + await pending; + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it.each(["curate", "compact", "responses-compact"] as const)( + "holds %s behind preterminal session work", + async (endpoint) => { + const sessionHeaders = { + "x-lore-session-id": `preterminal-${endpoint}-session`, + }; + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE(`resp_${endpoint}_setup`), { + headers: { "content-type": "text/event-stream" }, + }), + ); + let release!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + let finalizerStarted!: () => void; + const started = new Promise((resolve) => { + finalizerStarted = resolve; + }); + let pending: Promise | undefined; + + try { + await ( + await handleRequest( + makeResponsesRequest({ sessionHeaders }), + loadLocalConfig(), + ) + ).text(); + await new Promise((resolve) => setImmediate(resolve)); + const state = [...getActiveSessions().values()].find( + (candidate) => + candidate.headerSessionId === sessionHeaders["x-lore-session-id"], + ); + expect(state).toBeDefined(); + scheduleStreamingPostResponseForTest( + state?.sessionID ?? "", + async () => { + finalizerStarted(); + await blocked; + }, + ); + await started; + + const headers = { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-project": process.cwd(), + "x-lore-provider": "openai", + "x-lore-upstream-url": "https://api.openai.com/v1", + ...sessionHeaders, + }; + if (endpoint === "curate") { + const curate = makeResponsesRequest({ + sessionHeaders, + messages: [ + { + role: "user", + content: [{ type: "text", text: "/lore:curate" }], + }, + ], + }); + curate.stream = false; + pending = handleRequest(curate, loadLocalConfig()); + } else { + pending = + endpoint === "compact" + ? handleCompactEndpoint( + new Request("http://gateway.test/v1/compact", { + method: "POST", + headers, + body: JSON.stringify({ + project_path: process.cwd(), + tokens_before: 1, + }), + }), + loadLocalConfig(), + ) + : handleResponsesCompactEndpoint( + new Request("http://gateway.test/v1/responses/compact", { + method: "POST", + headers, + body: JSON.stringify({ + model: "gpt-5.6-sol", + instructions: "You are a coding agent.", + input: [ + { + role: "user", + content: [{ type: "input_text", text: "compact" }], + }, + ], + tools: [], + }), + }), + loadLocalConfig(), + ); + } + + const outcome = await Promise.race([ + pending.then(() => "settled" as const), + new Promise<"pending">((resolve) => + setImmediate(() => resolve("pending")), + ), + ]); + expect(outcome).toBe("pending"); + expect(isPipelineSessionActiveForTest(state?.sessionID ?? "")).toBe( + true, + ); + + release(); + const response = await pending; + expect(response.status).toBe(200); + await response.text(); + await vi.waitFor(() => + expect(isPipelineSessionActiveForTest(state?.sessionID ?? "")).toBe( + false, + ), + ); + } finally { + release(); + if (pending) { + const response = await pending; + if (!response.bodyUsed) await response.text(); + } + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }, + ); + + it.each(["structural", "compact", "responses-compact"] as const)( + "rechecks %s project authorization after a queued session claim", + async (endpoint) => { + const sessionHeaders = { + "x-lore-session-id": `queued-project-${endpoint}-session`, + }; + const projectA = `/tmp/lore-queued-${endpoint}-a`; + const projectB = `/tmp/lore-queued-${endpoint}-b`; + let upstreamCalls = 0; + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response(validResponsesSSE(`resp_queued_${endpoint}`), { + headers: { "content-type": "text/event-stream" }, + }); + }); + let releaseRebind!: () => void; + const rebindPause = new Promise((resolve) => { + releaseRebind = resolve; + }); + let rebindWaitingResolve!: () => void; + const rebindWaiting = new Promise((resolve) => { + rebindWaitingResolve = resolve; + }); + const undistilled = vi.spyOn(temporal, "undistilled"); + + try { + const setup = makeResponsesRequest({ sessionHeaders }); + setup.rawHeaders["x-lore-project"] = projectA; + await (await handleRequest(setup, loadLocalConfig())).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + undistilled.mockClear(); + + setPipelinePreUpstreamPauseForTest(rebindPause, rebindWaitingResolve); + const rebind = makeResponsesRequest({ sessionHeaders }); + rebind.rawHeaders["x-lore-project"] = projectB; + const rebindResponse = await handleRequest(rebind, loadLocalConfig()); + const rebindBody = rebindResponse.text(); + await rebindWaiting; + setPipelinePreUpstreamPauseForTest(undefined); + + const headers = { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-project": projectA, + "x-lore-provider": "openai", + "x-lore-upstream-url": "https://api.openai.com/v1", + ...sessionHeaders, + }; + let pending: Promise; + if (endpoint === "structural") { + const structural = makeResponsesRequest({ + sessionHeaders, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "Create an anchored summary from the conversation history above.", + }, + ], + }, + ], + tools: [], + }); + structural.stream = false; + structural.rawHeaders["x-lore-project"] = projectA; + pending = handleRequest(structural, loadLocalConfig()); + } else if (endpoint === "compact") { + pending = handleCompactEndpoint( + new Request("http://gateway.test/v1/compact", { + method: "POST", + headers, + body: JSON.stringify({ project_path: projectA }), + }), + loadLocalConfig(), + ); + } else { + pending = handleResponsesCompactEndpoint( + new Request("http://gateway.test/v1/responses/compact", { + method: "POST", + headers, + body: JSON.stringify({ + model: "gpt-5.6-sol", + instructions: "You are a coding agent.", + input: [ + { + role: "user", + content: [{ type: "input_text", text: "compact" }], + }, + ], + tools: [], + }), + }), + loadLocalConfig(), + ); + } + await vi.waitFor(() => + expect(pendingPipelineSessionClaimCountForTest()).toBe(1), + ); + + releaseRebind(); + expect(await rebindBody).toContain("event: response.completed"); + const response = await pending; + expect(response.status).toBe(403); + expect(await response.text()).toMatch(/project[_ ]path/i); + expect(undistilled).not.toHaveBeenCalled(); + expect(upstreamCalls).toBe(2); + const state = [...getActiveSessions().values()].find( + (candidate) => + candidate.headerSessionId === sessionHeaders["x-lore-session-id"], + ); + expect(state).toMatchObject({ + projectPath: projectB, + projectPathProvisional: false, + }); + } finally { + releaseRebind(); + undistilled.mockRestore(); + setPipelinePreUpstreamPauseForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }, + ); + + it.each([ + "regular", + "structural", + "compact", + "responses-compact", + "slash", + ] as const)( + "rejects a queued %s request after affinity rotation revokes its identity", + async (route) => { + const oldAffinity = `queued-revoked-${route}-old`; + const newAffinity = `queued-revoked-${route}-new`; + const history: GatewayRequest["messages"] = Array.from( + { length: 12 }, + (_, index) => ({ + role: + index === 0 || index === 10 + ? ("user" as const) + : ("assistant" as const), + content: [ + { + type: "text" as const, + text: `${route} rotation history ${index}`, + }, + ], + }), + ); + let upstreamCalls = 0; + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response( + validResponsesSSE(`resp_queued_revoked_${upstreamCalls}`), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + let releaseRotation!: () => void; + const rotationPause = new Promise((resolve) => { + releaseRotation = resolve; + }); + let rotationWaitingResolve!: () => void; + const rotationWaiting = new Promise((resolve) => { + rotationWaitingResolve = resolve; + }); + let queued: Promise | undefined; + let rotationBody: Promise | undefined; + const summaryRead = vi.spyOn(distillation, "loadForSession"); + + try { + const seed = makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": oldAffinity }, + messages: [history[0]], + }); + await (await handleRequest(seed, loadLocalConfig())).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const setup = makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": oldAffinity }, + messages: history, + }); + await (await handleRequest(setup, loadLocalConfig())).text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + setPipelinePreUpstreamPauseForTest( + rotationPause, + rotationWaitingResolve, + ); + const rotation = makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": newAffinity }, + messages: [ + ...history, + { + role: "user", + content: [{ type: "text", text: "continue after restart" }], + }, + ], + }); + const rotationResponse = await handleRequest( + rotation, + loadLocalConfig(), + ); + rotationBody = rotationResponse.text(); + await rotationWaiting; + const oldState = [...getActiveSessions().values()].find( + (state) => state.headerSessionId === oldAffinity, + ); + expect(oldState).toBeDefined(); + expect(isPipelineSessionActiveForTest(oldState?.sessionID ?? "")).toBe( + true, + ); + + const headers = { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-project": process.cwd(), + "x-lore-provider": "openai", + "x-lore-upstream-url": "https://api.openai.com/v1", + "x-session-affinity": oldAffinity, + }; + if (route === "regular") { + const request = makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": oldAffinity }, + }); + request.stream = false; + queued = handleRequest(request, loadLocalConfig()); + } else if (route === "structural") { + const request = makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": oldAffinity }, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "Create an anchored summary from the conversation history above.", + }, + ], + }, + ], + tools: [], + }); + request.stream = false; + queued = handleRequest(request, loadLocalConfig()); + } else if (route === "slash") { + const request = makeResponsesRequest({ + sessionHeaders: { "x-session-affinity": oldAffinity }, + messages: [ + { + role: "user", + content: [{ type: "text", text: "/lore:amnesia:on" }], + }, + ], + }); + request.stream = false; + queued = handleRequest(request, loadLocalConfig()); + } else if (route === "compact") { + queued = handleCompactEndpoint( + new Request("http://gateway.test/v1/compact", { + method: "POST", + headers, + body: JSON.stringify({ project_path: process.cwd() }), + }), + loadLocalConfig(), + ); + } else { + queued = handleResponsesCompactEndpoint( + new Request("http://gateway.test/v1/responses/compact", { + method: "POST", + headers, + body: JSON.stringify({ + model: "gpt-5.6-sol", + instructions: "You are a coding agent.", + input: [ + { + role: "user", + content: [{ type: "input_text", text: "compact" }], + }, + ], + tools: [], + }), + }), + loadLocalConfig(), + ); + } + await vi.waitFor(() => + expect(pendingPipelineSessionClaimCountForTest()).toBe(1), + ); + summaryRead.mockClear(); + + releaseRotation(); + expect(await rotationBody).toContain("event: response.completed"); + const response = await queued; + expect(response.status).toBe(route === "slash" ? 200 : 404); + expect(await response.text()).toMatch(/authenticated.*session/i); + expect(upstreamCalls).toBe(3); + expect(summaryRead).not.toHaveBeenCalled(); + } finally { + releaseRotation(); + if (rotationBody) await rotationBody.catch(() => ""); + if (queued) { + const response = await queued.catch(() => undefined); + if (response && !response.bodyUsed) await response.text(); + } + summaryRead.mockRestore(); + setPipelinePreUpstreamPauseForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }, + ); + + it("drops a captured post-response finalizer after session eviction", async () => { + const sessionHeaders = { + "x-lore-session-id": "evicted-finalizer-session", + }; + const request = makeResponsesRequest({ + sessionHeaders, + messages: [ + { + role: "user", + content: [{ type: "text", text: "must not store after eviction" }], + }, + ], + }); + const store = vi.spyOn(temporal, "store"); + let postResponses = 0; + setPostResponseStartObserverForTest(() => postResponses++); + setUpstreamInterceptor( + async () => + new Response(validResponsesSSE("resp_evicted_finalizer"), { + headers: { "content-type": "text/event-stream" }, + }), + ); + + try { + const response = await handleRequest(request, loadLocalConfig()); + const reader = response.body?.getReader(); + expect(reader).toBeDefined(); + const decoder = new TextDecoder(); + let output = ""; + while (!output.includes("event: response.completed")) { + const chunk = await reader?.read(); + expect(chunk?.done).toBe(false); + if (chunk?.value) + output += decoder.decode(chunk.value, { stream: true }); + } + expect(streamingPostResponsePendingForTest()).toBe(1); + expect(evictLiveSessionForTest(request)).toBe(true); + + for (;;) { + const chunk = await reader?.read(); + if (!chunk || chunk.done) break; + } + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(postResponses).toBe(0); + expect(store).not.toHaveBeenCalled(); + expect(streamingPostResponsePendingForTest()).toBe(0); + } finally { + store.mockRestore(); + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("keeps a preterminal turn private when amnesia is disabled concurrently", async () => { + const legacyHeader = { "x-session-affinity": "amnesia-snapshot-session" }; + let upstreamCalls = 0; + let sensitiveSource: + | ReadableStreamDefaultController + | undefined; + let sensitiveStartedResolve: (() => void) | undefined; + const sensitiveStarted = new Promise((resolve) => { + sensitiveStartedResolve = resolve; + }); + setUpstreamInterceptor(async () => { + upstreamCalls++; + if (upstreamCalls === 1) { + return new Response(validResponsesSSE("resp_amnesia_setup"), { + headers: { "content-type": "text/event-stream" }, + }); + } + sensitiveStartedResolve?.(); + return new Response( + new ReadableStream({ + start(controller) { + sensitiveSource = controller; + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + }); + const slashRequest = (command: string): GatewayRequest => { + const request = makeResponsesRequest({ + sessionHeaders: legacyHeader, + messages: [ + { role: "user", content: [{ type: "text", text: command }] }, + ], + }); + request.stream = false; + return request; + }; + const store = vi.spyOn(temporal, "store"); + + try { + const established = await handleRequest( + makeResponsesRequest({ sessionHeaders: legacyHeader }), + loadLocalConfig(), + ); + await established.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + await ( + await handleRequest(slashRequest("/lore:amnesia:on"), loadLocalConfig()) + ).text(); + store.mockClear(); + + const order: string[] = []; + setPostResponseStartObserverForTest(() => order.push("post")); + const sensitive = await handleRequest( + makeResponsesRequest({ + sessionHeaders: legacyHeader, + messages: [ + { + role: "user", + content: [{ type: "text", text: "preterminal secret" }], + }, + ], + }), + loadLocalConfig(), + ); + const sensitiveBody = sensitive.text(); + await sensitiveStarted; + + const disableAmnesia = handleRequest( + slashRequest("/lore:amnesia:off"), + loadLocalConfig(), + ); + const beforeTerminal = await Promise.race([ + disableAmnesia.then(() => "settled" as const), + new Promise<"pending">((resolve) => + setImmediate(() => resolve("pending")), + ), + ]); + expect(beforeTerminal).toBe("pending"); + + sensitiveSource?.enqueue( + new TextEncoder().encode(validResponsesSSE("resp_amnesia_secret")), + ); + sensitiveSource?.close(); + await sensitiveBody; + order.push("eof"); + await (await disableAmnesia).text(); + order.push("slash"); + + expect(order).toEqual(["eof", "post", "slash"]); + expect(store).not.toHaveBeenCalled(); + } finally { + store.mockRestore(); + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it("waits for a canonical finalizer before compaction reads temporal state", async () => { + const order: string[] = []; + let upstreamCalls = 0; + const originalStore = temporal.store.bind(temporal); + const store = vi.spyOn(temporal, "store").mockImplementation((input) => { + const result = originalStore(input); + if (!order.includes("stored")) order.push("stored"); + return result; + }); + const undistilledCount = vi + .spyOn(temporal, "undistilledCount") + .mockImplementation(() => { + if (!order.includes("compaction-read")) order.push("compaction-read"); + return 0; + }); + const wire = validResponsesSSE("resp_before_compaction"); + setUpstreamInterceptor(async () => { + upstreamCalls++; + return new Response(wire, { + headers: { "content-type": "text/event-stream" }, + }); + }); + try { + const first = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": "compaction-stable-session", + }, + messages: Array.from({ length: 12 }, (_, index) => ({ + role: index % 2 === 0 ? ("user" as const) : ("assistant" as const), + content: [ + { type: "text" as const, text: `remember this turn ${index}` }, + ], + })), + }), + loadLocalConfig(), + ); + await first.text(); + + const compacted = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { + "x-lore-session-id": "compaction-stable-session", + }, + messages: [ + { + role: "user", + content: [{ type: "text", text: "Summarize this conversation." }], + }, + ], + tools: [], + }), + loadLocalConfig(), + ); + expect(order).not.toContain("stored"); + const compactedBody = await compacted.text(); + + expect(compactedBody).toContain("remember this turn"); + expect(order.indexOf("stored")).toBeLessThan( + order.indexOf("compaction-read"), + ); + expect(upstreamCalls).toBe(1); + } finally { + store.mockRestore(); + undistilledCount.mockRestore(); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + + it.each(["compact", "responses-compact"] as const)( + "waits for deferred storage in the explicit %s endpoint", + async (endpoint) => { + const order: string[] = []; + let upstreamCalls = 0; + setPostResponseStartObserverForTest(() => order.push("post")); + setUpstreamInterceptor(async () => { + upstreamCalls++; + if (upstreamCalls === 1) { + return new Response(validResponsesSSE("resp_explicit_compact"), { + headers: { "content-type": "text/event-stream" }, + }); + } + return new Response(JSON.stringify({ output: [] }), { + headers: { "content-type": "application/json" }, + }); + }); + const sessionID = `explicit-${endpoint}-session`; + + try { + const streamed = await handleRequest( + makeResponsesRequest({ + sessionHeaders: { "x-lore-session-id": sessionID }, + }), + loadLocalConfig(), + ); + await streamed.text(); + order.push("eof"); + + const headers = { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-lore-project": process.cwd(), + "x-lore-provider": "openai", + "x-lore-upstream-url": "https://api.openai.com/v1", + "x-lore-session-id": sessionID, + }; + const response = + endpoint === "compact" + ? await handleCompactEndpoint( + new Request("http://gateway.test/v1/compact", { + method: "POST", + headers, + body: JSON.stringify({ + project_path: process.cwd(), + tokens_before: 1, + }), + }), + loadLocalConfig(), + ) + : await handleResponsesCompactEndpoint( + new Request("http://gateway.test/v1/responses/compact", { + method: "POST", + headers, + body: JSON.stringify({ + model: "gpt-5.6-sol", + instructions: "You are a coding agent.", + input: [ + { + role: "user", + content: [{ type: "input_text", text: "continue" }], + }, + ], + tools: [], + }), + }), + loadLocalConfig(), + ); + await response.text(); + order.push("endpoint"); + + expect(order.slice(0, 3)).toEqual(["eof", "post", "endpoint"]); + } finally { + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }, + ); + + it("pauses a filled build queue and resumes it when reads begin", async () => { + let pulls = 0; + const body = await validAnthropicSSE("resume").text(); + const chunks = body + .split(/(?=event: )/) + .filter(Boolean) + .map((chunk) => new TextEncoder().encode(chunk)); + let index = 0; + const upstream = new Response( + new ReadableStream({ + pull(controller) { + pulls++; + if (index < chunks.length) controller.enqueue(chunks[index++]); + else controller.close(); + }, + }), + ); + const downstream = buildStreamingResponse(upstream, () => {}); + await new Promise((resolve) => setImmediate(resolve)); + const pullsBeforeRead = pulls; + expect(pullsBeforeRead).toBeLessThan(chunks.length + 1); + const text = await downstream.text(); + expect(text).toContain("resume"); + expect(pulls).toBeGreaterThan(pullsBeforeRead); + }); + + it("distinguishes external meta abort from silent downstream cancellation", async () => { + let cancelledBeforeAcquire = false; + const beforeAcquire = validatedMetaStream( + new Response( + new ReadableStream({ + cancel() { + cancelledBeforeAcquire = true; + }, + }), + ), + "anthropic", + false, + ); + await beforeAcquire.body?.cancel(); + expect(cancelledBeforeAcquire).toBe(true); + + let externallyCancelled = false; + const abort = new AbortController(); + abort.abort(new DOMException("deadline", "TimeoutError")); + const removeAbortListener = vi.spyOn(abort.signal, "removeEventListener"); + const externallyAborted = validatedMetaStream( + new Response( + new ReadableStream({ + cancel() { + externallyCancelled = true; + }, + }), + ), + "anthropic", + false, + abort.signal, + ); + await expect(externallyAborted.text()).rejects.toMatchObject({ + name: "TimeoutError", + }); + expect(externallyCancelled).toBe(true); + expect(removeAbortListener).toHaveBeenCalledWith( + "abort", + expect.any(Function), + ); + }); + + it("meta downstream cancel does not await a hostile upstream cancel", async () => { + let sourceCancelled = false; + const upstream = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode( + 'event: message_start\ndata: {"type":"message_start","message":{"id":"hostile","type":"message","role":"assistant","model":"test","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":1,"output_tokens":0}}}\n\n', + ), + ); + }, + pull() { + return new Promise(() => {}); + }, + cancel() { + sourceCancelled = true; + return new Promise(() => {}); + }, + }), + ); + const downstreamBody = validatedMetaStream( + upstream, + "anthropic", false, ).body; if (!downstreamBody) throw new Error("test stream has no body"); @@ -602,7 +6335,9 @@ describe("Pipeline — streaming responses", () => { controller.abort(new DOMException("client disconnected", "AbortError")); const response = await pending; expect(response.status).toBe(502); - await expect(response.text()).resolves.toContain("client disconnected"); + await expect(response.text()).resolves.toContain( + "Gateway request failed", + ); } finally { setUpstreamInterceptor(undefined); } @@ -631,7 +6366,9 @@ describe("Pipeline — streaming responses", () => { await vi.advanceTimersByTimeAsync(300_000); const response = await pending; expect(response.status).toBe(502); - await expect(response.text()).resolves.toContain("timed out"); + await expect(response.text()).resolves.toContain( + "Gateway request failed", + ); } finally { setUpstreamInterceptor(undefined); vi.useRealTimers(); @@ -802,7 +6539,7 @@ describe("Pipeline — streaming responses", () => { for (const testCase of cases) { nextStatus = testCase.status; - const response = await fetch(`${harness.baseURL}${testCase.path}`, { + const response = await harness.request(testCase.path, { method: "POST", headers: { "content-type": "application/json", @@ -821,6 +6558,151 @@ describe("Pipeline — streaming responses", () => { } }); + it.each([true, false])( + "preserves a valid incomplete Responses terminal for cross-protocol meta stream=%s", + async (stream) => { + setUpstreamInterceptor(async () => + stream + ? new Response(incompleteResponsesSSE("resp_meta_incomplete"), { + headers: { "content-type": "text/event-stream" }, + }) + : new Response( + JSON.stringify({ + id: "resp_meta_incomplete", + model: "gpt-5.6-sol", + status: "incomplete", + output: [], + usage: { input_tokens: 10, output_tokens: 2 }, + }), + { headers: { "content-type": "application/json" } }, + ), + ); + + try { + const response = await handleRequest( + { + protocol: "anthropic", + model: "gpt-5.6-sol", + system: "title this", + messages: [ + { role: "user", content: [{ type: "text", text: "title" }] }, + ], + tools: [], + stream, + maxTokens: 32, + metadata: {}, + rawHeaders: { + authorization: "Bearer test-key", + "x-lore-agent": "title", + "x-lore-provider": "openai", + }, + }, + loadLocalConfig(), + ); + expect(response.status).toBe(200); + const body = await response.text(); + expect(body).toContain('"stop_reason":"max_tokens"'); + expect(body).not.toContain("Gateway request failed"); + } finally { + setUpstreamInterceptor(undefined); + } + }, + ); + + it("rejects an unknown public incomplete reason on cross-protocol meta passthrough", async () => { + setUpstreamInterceptor( + async () => + new Response( + JSON.stringify({ + id: "resp_meta_malformed", + model: "gpt-5.6-sol", + status: "incomplete", + incomplete_details: { reason: "provider_specific" }, + output: [], + usage: { input_tokens: 10, output_tokens: 2 }, + }), + { headers: { "content-type": "application/json" } }, + ), + ); + + try { + const response = await handleRequest( + { + protocol: "anthropic", + model: "gpt-5.6-sol", + system: "title this", + messages: [ + { role: "user", content: [{ type: "text", text: "title" }] }, + ], + tools: [], + stream: false, + maxTokens: 32, + metadata: {}, + rawHeaders: { + authorization: "Bearer test-key", + "x-lore-agent": "title", + "x-lore-provider": "openai", + }, + }, + loadLocalConfig(), + ); + expect(response.status).toBe(502); + expect(await response.text()).toContain("Gateway request failed"); + } finally { + setUpstreamInterceptor(undefined); + } + }); + + it("rejects malformed completed JSON on same-protocol Responses meta passthrough", async () => { + setUpstreamInterceptor( + async () => + new Response( + JSON.stringify({ + id: "resp_meta_malformed_function", + model: "gpt-5.6-sol", + status: "completed", + output: [ + { + type: "function_call", + id: "fc_meta_malformed", + call_id: "call_meta_malformed", + arguments: "{}", + }, + ], + usage: { input_tokens: 10, output_tokens: 2 }, + }), + { headers: { "content-type": "application/json" } }, + ), + ); + + try { + const response = await handleRequest( + { + protocol: "openai-responses", + model: "gpt-5.6-sol", + system: "title this", + messages: [ + { role: "user", content: [{ type: "text", text: "title" }] }, + ], + tools: [], + stream: false, + maxTokens: 32, + metadata: {}, + rawHeaders: { + authorization: "Bearer test-key", + "x-lore-agent": "title", + "x-lore-provider": "openai", + }, + }, + loadLocalConfig(), + ); + expect(response.status).toBe(502); + expect(await response.text()).toContain("Gateway request failed"); + } finally { + setUpstreamInterceptor(undefined); + } + }); + it.each(["openai-responses", "gemini"] as const)( "strictly translates an open-tail Anthropic meta stream to %s", async (protocol) => { @@ -1186,7 +7068,7 @@ describe("Pipeline — streaming responses", () => { headers: { "content-type": "text/event-stream" }, }), ); - const response = await fetch(`${harness.baseURL}${path}`, { + const response = await harness.request(path, { method: "POST", headers: { "content-type": "application/json", diff --git a/packages/gateway/test/project-path.test.ts b/packages/gateway/test/project-path.test.ts index b38db9c9..3adeb024 100644 --- a/packages/gateway/test/project-path.test.ts +++ b/packages/gateway/test/project-path.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect } from "vitest"; +import { DatabaseSync } from "node:sqlite"; import { inferProjectPath, inferProjectPathDetailed, @@ -18,6 +19,7 @@ import type { SessionState } from "../src/translate/types"; import type { ResolveProjectResult } from "../src/synthetic-tools"; import { ensureProject, + db, projectId, ltm, saveSessionTracking, @@ -759,6 +761,43 @@ describe("resolveSessionProjectPath", () => { expect(moved.some((e) => e.title === `bucket-finding-${sid}`)).toBe(true); }); + test("keeps the old provisional binding when re-attribution is transiently busy", () => { + const sid = `busyMerge-${crypto.randomUUID()}`; + const bucket = unattributedBucketPath(sid); + const realPath = `/test/merge/busy-${crypto.randomUUID()}`; + ensureProject(bucket); + const blocker = new DatabaseSync(process.env.LORE_DB_PATH as string); + db().exec("PRAGMA busy_timeout = 0"); + try { + blocker.exec("BEGIN IMMEDIATE"); + const state = provisionalState(sid, bucket); + const result = resolveSessionProjectPath( + { path: realPath, source: "header" }, + state, + localCfg, + ); + + expect(result).toBe(bucket); + expect(state.projectPath).toBe(bucket); + expect(state.projectPathProvisional).toBe(true); + } finally { + if (blocker.isTransaction) blocker.exec("ROLLBACK"); + blocker.close(); + db().exec("PRAGMA busy_timeout = 5000"); + } + + // The next confident turn retries after contention clears and heals. + const state = provisionalState(sid, bucket); + expect( + resolveSessionProjectPath( + { path: realPath, source: "header" }, + state, + localCfg, + ), + ).toBe(realPath); + expect(state.projectPathProvisional).toBe(false); + }); + // --- confidentlyWrong: re-bind an already-confident session bound to a // stale header path, WITHOUT merging (cross-project safety) --- @@ -1099,4 +1138,35 @@ describe("applySyntheticResolution gitHead binding (#627 Phase 1)", () => { expect(path).toBe("/tmp/provisional"); expect(state.gitHead).toBeUndefined(); }); + + test("keeps the old provisional binding when synthetic re-attribution is busy", () => { + const sid = `synthetic-busy-${crypto.randomUUID()}`; + const bucket = unattributedBucketPath(sid); + const realPath = `/test/synthetic/busy-${crypto.randomUUID()}`; + ensureProject(bucket); + const state = { + ...freshState(sid), + projectPath: bucket, + }; + const blocker = new DatabaseSync(process.env.LORE_DB_PATH as string); + db().exec("PRAGMA busy_timeout = 0"); + try { + blocker.exec("BEGIN IMMEDIATE"); + expect(applySyntheticResolution(state, { root: realPath }, bucket)).toBe( + bucket, + ); + expect(state.projectPath).toBe(bucket); + expect(state.projectPathProvisional).toBe(true); + } finally { + if (blocker.isTransaction) blocker.exec("ROLLBACK"); + blocker.close(); + db().exec("PRAGMA busy_timeout = 5000"); + } + + expect(applySyntheticResolution(state, { root: realPath }, bucket)).toBe( + realPath, + ); + expect(state.projectPath).toBe(realPath); + expect(state.projectPathProvisional).toBe(false); + }); }); diff --git a/packages/gateway/test/recall-codex-stream.test.ts b/packages/gateway/test/recall-codex-stream.test.ts index 54312a20..b482fd42 100644 --- a/packages/gateway/test/recall-codex-stream.test.ts +++ b/packages/gateway/test/recall-codex-stream.test.ts @@ -29,6 +29,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { loopbackRequest } from "./helpers/loopback-request"; /** One Responses-API SSE event (`event:` + `data:` framing). */ function sseEvent(event: string, data: unknown): string { @@ -224,7 +225,7 @@ describe("recall follow-up — openai-codex (ChatGPT) path", () => { } }; - const resp = await fetch(`${baseURL}/v1/codex/responses`, { + const resp = await loopbackRequest(`${baseURL}/v1/codex/responses`, { method: "POST", headers: { "content-type": "application/json", @@ -272,7 +273,7 @@ describe("recall follow-up — openai-codex (ChatGPT) path", () => { upstream: Response, ): Promise => { setUpstreamInterceptor(async () => upstream); - const response = await fetch(`${baseURL}${path}`, { + const response = await loopbackRequest(`${baseURL}${path}`, { method: "POST", headers: { "content-type": "application/json", @@ -447,7 +448,7 @@ describe("recall follow-up — openai-codex (ChatGPT) path", () => { } }; - const resp = await fetch(`${baseURL}/v1/responses`, { + const resp = await loopbackRequest(`${baseURL}/v1/responses`, { method: "POST", headers: { "content-type": "application/json", diff --git a/packages/gateway/test/recall-marker-split.e2e.test.ts b/packages/gateway/test/recall-marker-split.e2e.test.ts index 056e29dd..7cfec1c4 100644 --- a/packages/gateway/test/recall-marker-split.e2e.test.ts +++ b/packages/gateway/test/recall-marker-split.e2e.test.ts @@ -30,6 +30,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { loopbackRequest } from "./helpers/loopback-request"; // --------------------------------------------------------------------------- // Helpers: build Anthropic SSE events for fixtures @@ -237,7 +238,7 @@ describe("Streaming recall marker — Anthropic native (split envelope)", () => teardownFn = () => teardownAll(dbPath, projectDir, server, closeDB, setUpstreamInterceptor); - const resp = await fetch(`${baseURL}/v1/messages`, { + const resp = await loopbackRequest(`${baseURL}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", @@ -315,7 +316,7 @@ describe("Streaming recall marker — Anthropic native (split envelope)", () => teardownFn = () => teardownAll(dbPath, projectDir, server, closeDB, setUpstreamInterceptor); - const resp = await fetch(`${baseURL}/v1/messages`, { + const resp = await loopbackRequest(`${baseURL}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", @@ -378,7 +379,7 @@ describe("Streaming recall marker — Anthropic native (split envelope)", () => teardownFn = () => teardownAll(dbPath, projectDir, server, closeDB, setUpstreamInterceptor); - const resp = await fetch(`${baseURL}/v1/messages`, { + const resp = await loopbackRequest(`${baseURL}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", @@ -494,7 +495,7 @@ describe("Streaming recall marker — non-Anthropic (inline + translated)", () = // stream/openai.ts converts the Anthropic SSE to OpenAI Chat Completions // chunks — including the inline marker, which arrives as a // delta.content chunk. - const resp = await fetch(`${baseURL}/v1/chat/completions`, { + const resp = await loopbackRequest(`${baseURL}/v1/chat/completions`, { method: "POST", headers: { "content-type": "application/json", @@ -683,7 +684,7 @@ describe("Streaming recall marker — mixed tools (recall + Read)", () => { teardownFn = () => teardownAll(dbPath, projectDir, server, closeDB, setUpstreamInterceptor); - const resp = await fetch(`${baseURL}/v1/messages`, { + const resp = await loopbackRequest(`${baseURL}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", @@ -873,7 +874,7 @@ describe("Streaming recall marker — non-Anthropic mixed tools (recall + Read)" // pipeline.ts:4824 takeHeldBackEvents() path which forwards the // preamble's message_delta + message_stop to the OpenAI translator // so it can emit finish_reason="tool_calls" + [DONE]. - const resp = await fetch(`${baseURL}/v1/chat/completions`, { + const resp = await loopbackRequest(`${baseURL}/v1/chat/completions`, { method: "POST", headers: { "content-type": "application/json", @@ -983,7 +984,7 @@ describe("Streaming recall marker — multi-recall drill-down", () => { teardownFn = () => teardownAll(dbPath, projectDir, server, closeDB, setUpstreamInterceptor); - const resp = await fetch(`${baseURL}/v1/messages`, { + const resp = await loopbackRequest(`${baseURL}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", diff --git a/packages/gateway/test/recall-openai-stream.test.ts b/packages/gateway/test/recall-openai-stream.test.ts index 9e99372c..4c8ed159 100644 --- a/packages/gateway/test/recall-openai-stream.test.ts +++ b/packages/gateway/test/recall-openai-stream.test.ts @@ -23,6 +23,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { loopbackRequest } from "./helpers/loopback-request"; // SSE chunk for OpenAI chat-completions streaming. function sseChunk(obj: unknown): string { @@ -156,7 +157,7 @@ describe("recall interception — OpenAI streaming path", () => { } }; - const resp = await fetch(`${baseURL}/v1/messages`, { + const resp = await loopbackRequest(`${baseURL}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", diff --git a/packages/gateway/test/recompress-scope.test.ts b/packages/gateway/test/recompress-scope.test.ts index fb79d87a..86cada64 100644 --- a/packages/gateway/test/recompress-scope.test.ts +++ b/packages/gateway/test/recompress-scope.test.ts @@ -114,7 +114,7 @@ describe("upstream re-compression scoping (#1032 follow-up, wire-level)", () => }), ), ); - await fetch(`${harness.baseURL}/v1/chat/completions`, { + await harness.request("/v1/chat/completions", { method: "POST", headers: { "content-type": "application/json", diff --git a/packages/gateway/test/remote-attribution.test.ts b/packages/gateway/test/remote-attribution.test.ts index 34725d94..b4341f78 100644 --- a/packages/gateway/test/remote-attribution.test.ts +++ b/packages/gateway/test/remote-attribution.test.ts @@ -33,6 +33,33 @@ function pathlessBody(userMessage: string): Record { }; } +function pathlessBodyWithoutTools( + userMessage: string, +): Record { + return { + ...pathlessBody(userMessage), + // Keep enough tool definitions for normal-turn classification, but none of + // the read/shell tools that trigger the synthetic project-resolution probe. + tools: [ + { + name: "write_a", + description: "Write A", + input_schema: { type: "object" }, + }, + { + name: "write_b", + description: "Write B", + input_schema: { type: "object" }, + }, + { + name: "write_c", + description: "Write C", + input_schema: { type: "object" }, + }, + ], + }; +} + describe("remote gateway: path-less session attribution", () => { let harness: Harness; @@ -99,6 +126,398 @@ describe("remote gateway: path-less session attribution", () => { // The gateway's own cwd must NOT have become a project. expect(projects.some((p) => p.path === process.cwd())).toBe(false); }); + + it("re-attributes a provisional bucket before publishing a rotated session header", async () => { + const first = "bucket turn that must follow the session"; + const second = "second stable bucket turn"; + const third = "confident turn after client restart"; + const realPath = "/client/projects/remote-self-heal"; + harness = await createHarness({ + fixtures: makeConversationFixtures([ + { userMessage: first, assistantText: "First response." }, + { userMessage: second, assistantText: "Second response." }, + { userMessage: third, assistantText: "Third response." }, + ]), + configOverrides: { + remoteGateway: true, + gatewayAuthToken: TEST_GATEWAY_AUTH_TOKEN, + }, + }); + + let response = await harness.chat( + pathlessBodyWithoutTools(first), + "test-key", + { + "x-lore-project": "", + "x-session-affinity": "remote-affinity-before-restart", + "x-lore-gateway-token": TEST_GATEWAY_AUTH_TOKEN, + }, + ); + expect(response.status).toBe(200); + await response.text(); + response = await harness.chat( + { + ...pathlessBodyWithoutTools(second), + messages: [ + { role: "user", content: first }, + { role: "assistant", content: "First response." }, + { role: "user", content: second }, + ], + }, + "test-key", + { + "x-lore-project": "", + "x-session-affinity": "remote-affinity-before-restart", + "x-lore-gateway-token": TEST_GATEWAY_AUTH_TOKEN, + }, + ); + expect(response.status).toBe(200); + await response.text(); + // A real client restart may coincide with a gateway restart. This also + // deterministically drains the old affinity's deferred finalizer before the + // persisted binding is inspected and the affinity value rotates. + await harness.restartPipeline(); + + const before = harness.queryDB<{ + session_id: string; + project_path: string; + project_path_provisional: number; + }>( + `SELECT session_id, project_path, project_path_provisional + FROM session_state + WHERE header_session_id = 'remote-affinity-before-restart'`, + ); + expect(before).toHaveLength(1); + expect(before[0].project_path).toMatch(/^\/__lore_unattributed__\//); + expect(before[0].project_path_provisional).toBe(1); + + // OpenCode restarted and rotated its affinity value. Fingerprint adoption + // proves continuity with both leading user messages in the provisional + // bucket; the successful turn must self-heal that bucket before publishing + // the new header and confident path. + response = await harness.chat( + { + ...pathlessBodyWithoutTools(third), + messages: [ + { role: "user", content: first }, + { role: "assistant", content: "First response." }, + { role: "user", content: second }, + { role: "assistant", content: "Second response." }, + { role: "user", content: third }, + ], + }, + "test-key", + { + "x-lore-project": realPath, + "x-session-affinity": "remote-affinity-after-restart", + "x-lore-gateway-token": TEST_GATEWAY_AUTH_TOKEN, + }, + ); + expect(response.status).toBe(200); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const after = harness.queryDB<{ + session_id: string; + header_session_id: string; + project_path: string; + project_path_provisional: number; + }>( + `SELECT session_id, header_session_id, project_path, + project_path_provisional + FROM session_state + WHERE session_id = ?`, + [before[0].session_id], + ); + expect(after).toEqual([ + { + session_id: before[0].session_id, + header_session_id: "remote-affinity-after-restart", + project_path: realPath, + project_path_provisional: 0, + }, + ]); + + const attribution = harness.queryDB<{ + content: string; + project_path: string; + }>( + `SELECT tm.content, p.path AS project_path + FROM temporal_messages tm + JOIN projects p ON p.id = tm.project_id + WHERE tm.session_id = ?`, + [before[0].session_id], + ); + expect(attribution.some((row) => row.content.includes(first))).toBe(true); + expect(attribution.some((row) => row.content.includes(second))).toBe(true); + expect(attribution.some((row) => row.content.includes(third))).toBe(true); + expect(new Set(attribution.map((row) => row.project_path))).toEqual( + new Set([realPath]), + ); + }); + + it("rolls back project re-attribution when the provisional turn commit fails", async () => { + const first = "atomic bucket turn"; + const second = "atomic stable bucket turn"; + const third = "atomic confident retry"; + const realPath = "/client/projects/atomic-self-heal"; + const oldRoute = "https://old-anthropic-route.invalid"; + const newRoute = "https://new-anthropic-route.invalid"; + const oldAffinity = "atomic-affinity-before-restart"; + const newAffinity = "atomic-affinity-after-restart"; + harness = await createHarness({ + fixtures: makeConversationFixtures([ + { userMessage: first, assistantText: "First response." }, + { userMessage: second, assistantText: "Second response." }, + { userMessage: third, assistantText: "Failed local commit." }, + { userMessage: third, assistantText: "Failed project merge." }, + { userMessage: third, assistantText: "Successful retry." }, + ]), + configOverrides: { + remoteGateway: true, + gatewayAuthToken: TEST_GATEWAY_AUTH_TOKEN, + callerUpstreamAllowlist: [ + new URL(oldRoute).origin, + new URL(newRoute).origin, + ], + }, + }); + + let response = await harness.chat( + pathlessBodyWithoutTools(first), + "test-key", + { + "x-lore-project": "", + "x-session-affinity": oldAffinity, + "x-lore-provider": "anthropic", + "x-lore-upstream-url": oldRoute, + "x-lore-gateway-token": TEST_GATEWAY_AUTH_TOKEN, + }, + ); + expect(response.status).toBe(200); + await response.text(); + response = await harness.chat( + { + ...pathlessBodyWithoutTools(second), + messages: [ + { role: "user", content: first }, + { role: "assistant", content: "First response." }, + { role: "user", content: second }, + ], + }, + "test-key", + { + "x-lore-project": "", + "x-session-affinity": oldAffinity, + "x-lore-provider": "anthropic", + "x-lore-upstream-url": oldRoute, + "x-lore-gateway-token": TEST_GATEWAY_AUTH_TOKEN, + }, + ); + expect(response.status).toBe(200); + await response.text(); + await harness.restartPipeline(); + + const before = harness.queryDB<{ + session_id: string; + project_path: string; + last_upstream: string; + }>( + `SELECT session_id, project_path, last_upstream + FROM session_state + WHERE header_session_id = ?`, + [oldAffinity], + ); + expect(before).toHaveLength(1); + const bucketPath = before[0].project_path; + expect(bucketPath).toMatch(/^\/__lore_unattributed__\//); + expect(before[0].last_upstream).toContain(oldRoute); + + const { db } = await import("@loreai/core"); + db().exec(` + CREATE TEMP TRIGGER fail_atomic_provisional_turn + BEFORE INSERT ON temporal_messages + WHEN NEW.content LIKE '%${third}%' + BEGIN + SELECT RAISE(ABORT, 'forced provisional temporal failure'); + END; + `); + + const migratedBody = { + ...pathlessBodyWithoutTools(third), + model: "claude-opus-4-1", + messages: [ + { role: "user", content: first }, + { role: "assistant", content: "First response." }, + { role: "user", content: second }, + { role: "assistant", content: "Second response." }, + { role: "user", content: third }, + ], + }; + response = await harness.chat(migratedBody, "test-key", { + "x-lore-project": realPath, + "x-session-affinity": newAffinity, + "x-lore-provider": "anthropic", + "x-lore-upstream-url": newRoute, + "x-lore-gateway-token": TEST_GATEWAY_AUTH_TOKEN, + }); + expect(response.status).toBe(200); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const afterFailure = harness.queryDB<{ + header_session_id: string; + project_path: string; + project_path_provisional: number; + last_upstream: string; + }>( + `SELECT header_session_id, project_path, project_path_provisional, + last_upstream + FROM session_state + WHERE session_id = ?`, + [before[0].session_id], + ); + expect(afterFailure).toEqual([ + { + header_session_id: oldAffinity, + project_path: bucketPath, + project_path_provisional: 1, + last_upstream: before[0].last_upstream, + }, + ]); + expect( + harness.queryDB("SELECT id FROM projects WHERE path = ?", [realPath]), + ).toHaveLength(0); + expect( + harness.queryDB("SELECT id FROM projects WHERE path = ?", [bucketPath]), + ).toHaveLength(1); + expect( + harness.queryDB( + "SELECT project_id FROM project_path_aliases WHERE path = ?", + [bucketPath], + ), + ).toHaveLength(0); + const failedAttribution = harness.queryDB<{ + content: string; + project_path: string; + }>( + `SELECT tm.content, p.path AS project_path + FROM temporal_messages tm + JOIN projects p ON p.id = tm.project_id + WHERE tm.session_id = ?`, + [before[0].session_id], + ); + expect(failedAttribution.some((row) => row.content.includes(first))).toBe( + true, + ); + expect(failedAttribution.some((row) => row.content.includes(second))).toBe( + true, + ); + expect(failedAttribution.some((row) => row.content.includes(third))).toBe( + false, + ); + expect(new Set(failedAttribution.map((row) => row.project_path))).toEqual( + new Set([bucketPath]), + ); + + db().exec("DROP TRIGGER fail_atomic_provisional_turn"); + db().exec(` + CREATE TEMP TRIGGER fail_atomic_project_merge + BEFORE DELETE ON projects + WHEN OLD.path LIKE '/__lore_unattributed__/%' + BEGIN + SELECT RAISE(ABORT, 'forced provisional project merge failure'); + END; + `); + response = await harness.chat(migratedBody, "test-key", { + "x-lore-project": realPath, + "x-session-affinity": newAffinity, + "x-lore-provider": "anthropic", + "x-lore-upstream-url": newRoute, + "x-lore-gateway-token": TEST_GATEWAY_AUTH_TOKEN, + }); + expect(response.status).toBe(200); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect( + harness.queryDB( + `SELECT header_session_id, project_path, project_path_provisional, + last_upstream + FROM session_state + WHERE session_id = ?`, + [before[0].session_id], + ), + ).toEqual(afterFailure); + expect( + harness.queryDB("SELECT id FROM projects WHERE path = ?", [realPath]), + ).toHaveLength(0); + expect( + harness.queryDB("SELECT id FROM projects WHERE path = ?", [bucketPath]), + ).toHaveLength(1); + + db().exec("DROP TRIGGER fail_atomic_project_merge"); + response = await harness.chat(migratedBody, "test-key", { + "x-lore-project": realPath, + "x-session-affinity": newAffinity, + "x-lore-provider": "anthropic", + "x-lore-upstream-url": newRoute, + "x-lore-gateway-token": TEST_GATEWAY_AUTH_TOKEN, + }); + expect(response.status).toBe(200); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const afterRetry = harness.queryDB<{ + header_session_id: string; + project_path: string; + project_path_provisional: number; + last_upstream: string; + }>( + `SELECT header_session_id, project_path, project_path_provisional, + last_upstream + FROM session_state + WHERE session_id = ?`, + [before[0].session_id], + ); + expect(afterRetry).toHaveLength(1); + expect(afterRetry[0]).toMatchObject({ + header_session_id: newAffinity, + project_path: realPath, + project_path_provisional: 0, + }); + expect(afterRetry[0].last_upstream).toContain(newRoute); + expect(afterRetry[0].last_upstream).toContain("claude-opus-4-1"); + expect( + harness.queryDB("SELECT id FROM projects WHERE path = ?", [bucketPath]), + ).toHaveLength(0); + expect( + harness.queryDB( + "SELECT project_id FROM project_path_aliases WHERE path = ?", + [bucketPath], + ), + ).toHaveLength(1); + const attribution = harness.queryDB<{ + content: string; + project_path: string; + }>( + `SELECT tm.content, p.path AS project_path + FROM temporal_messages tm + JOIN projects p ON p.id = tm.project_id + WHERE tm.session_id = ?`, + [before[0].session_id], + ); + expect(attribution.some((row) => row.content.includes(first))).toBe(true); + expect(attribution.some((row) => row.content.includes(second))).toBe(true); + expect(attribution.some((row) => row.content.includes(third))).toBe(true); + expect(new Set(attribution.map((row) => row.project_path))).toEqual( + new Set([realPath]), + ); + }); }); describe("lore data consolidate", () => { diff --git a/packages/gateway/test/remote-session-tenant-binding.e2e.test.ts b/packages/gateway/test/remote-session-tenant-binding.e2e.test.ts index 0f3bf271..0c8ebd84 100644 --- a/packages/gateway/test/remote-session-tenant-binding.e2e.test.ts +++ b/packages/gateway/test/remote-session-tenant-binding.e2e.test.ts @@ -258,8 +258,42 @@ describe("remote authenticated tenant session binding", () => { ); test("preserves provider-specific credentials inside one remote tenant session", async () => { + const openAIResponse = { + ...makeFixtureEntry({ + seq: 1, + requestMessages: [], + responseText: "tenant A OpenAI response", + model: "gpt-4o-mini", + }), + response: { + id: "resp_tenant_a", + object: "response", + model: "gpt-4o-mini", + status: "completed", + output: [ + { + type: "message", + id: "msg_tenant_a", + role: "assistant", + status: "completed", + content: [ + { + type: "output_text", + text: "tenant A OpenAI response", + annotations: [], + }, + ], + }, + ], + usage: { + input_tokens: 100, + output_tokens: 10, + total_tokens: 110, + }, + }, + }; harness = await createHarness({ - fixtures: fixtures(2), + fixtures: [fixtures(1)[0], openAIResponse], configOverrides: { remoteGateway: true, gatewayAuthToken: TEST_GATEWAY_AUTH_TOKEN, @@ -280,11 +314,14 @@ describe("remote authenticated tenant session binding", () => { await response.text(); response = await harness.chat( - body([ - { role: "user", content: PRIVATE_A }, - { role: "assistant", content: "tenant A response" }, - { role: "user", content: FOLLOW_UP_A }, - ]), + { + ...body([ + { role: "user", content: PRIVATE_A }, + { role: "assistant", content: "tenant A response" }, + { role: "user", content: FOLLOW_UP_A }, + ]), + model: "gpt-4o-mini", + }, AUTH_CASES[0].credentialA, { "x-lore-session-id": SHARED_SESSION, @@ -333,6 +370,8 @@ describe("remote authenticated tenant session binding", () => { expect(response.status).toBe(200); await response.text(); } + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); const rows = harness.queryDB<{ content: string; @@ -456,6 +495,8 @@ describe("remote authenticated tenant session binding", () => { ); expect(response.status).toBe(200); await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); const resumed = harness.queryDB<{ session_id: string; diff --git a/packages/gateway/test/replay.test.ts b/packages/gateway/test/replay.test.ts index 8b05741c..8baf65e9 100644 --- a/packages/gateway/test/replay.test.ts +++ b/packages/gateway/test/replay.test.ts @@ -328,28 +328,14 @@ describe("Compaction interception", () => { afterEach(() => harness?.teardown()); - it("compaction request falls back to upstream when worker model is unavailable", async () => { - // In test, the worker model has no auth credentials, so llm.prompt() - // returns null. The gateway should fall back to forwarding the original - // compaction request to the upstream API (like handlePassthrough). - // Provide one fixture for the upstream fallback response. + it("rejects compaction without an authenticated prior session", async () => { const compactionSystem = "You are an anchored context summarization assistant for coding sessions. " + "Your job is to produce a structured summary of the conversation history."; const compactionUserMessage = "Please create an anchored summary from the conversation history above."; - harness = await createHarness({ - fixtures: [ - makeFixtureEntry({ - seq: 0, - system: compactionSystem, - requestMessages: [{ role: "user", content: compactionUserMessage }], - responseText: - "## Summary\n\nThis is a compaction summary from upstream.", - }), - ], - }); + harness = await createHarness({ fixtures: [] }); // Build a request that matches isCompactionRequest() via the system prompt pattern const resp = await harness.chat({ @@ -366,24 +352,14 @@ describe("Compaction interception", () => { // No tools — compaction agents typically have no tools }); - // The gateway must return a 200 — either from Lore's own summary or - // from the upstream fallback when the worker model is unavailable. - expect(resp.status).toBe(200); - - const body = (await resp.json()) as Record; - - // Response must have content (the synthesized summary text) - const content = body.content as Array>; - expect(Array.isArray(content)).toBe(true); - expect(content.length).toBeGreaterThanOrEqual(1); - - // There must be at least one text block - const textBlock = content.find((b) => b.type === "text"); - expect(textBlock).toBeDefined(); - expect(typeof (textBlock as Record).text).toBe("string"); + expect(resp.status).toBe(404); - // The response should have the standard Anthropic shape - expect(typeof body.id).toBe("string"); - expect(typeof body.stop_reason).toBe("string"); + const body = (await resp.json()) as { + type: string; + error: { message: string }; + }; + expect(body.type).toBe("error"); + expect(body.error.message).toContain("No authenticated session found"); + expect(harness.upstreamBodies()).toHaveLength(0); }); }); diff --git a/packages/gateway/test/run-command-adoption.test.ts b/packages/gateway/test/run-command-adoption.test.ts new file mode 100644 index 00000000..5c068f33 --- /dev/null +++ b/packages/gateway/test/run-command-adoption.test.ts @@ -0,0 +1,110 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + safeExit: vi.fn(), + forcedExit: vi.fn(), + spawn: vi.fn(), + startGateway: vi.fn(), + probeGateway: vi.fn(), +})); + +vi.mock("node:child_process", () => ({ spawn: mocks.spawn })); +vi.mock("@loreai/core", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + discoverWorkspaceRoot: () => process.cwd(), + getGitRemote: () => null, + log: { ...actual.log, silenceStderr: vi.fn() }, + }; +}); +vi.mock("../src/cli/exit", () => ({ + safeExit: mocks.safeExit, + forcedExit: mocks.forcedExit, +})); +vi.mock("../src/cli/import-auto", () => ({ maybeAutoImport: vi.fn() })); +vi.mock("../src/cli/start", () => ({ + startGateway: mocks.startGateway, + probeGateway: mocks.probeGateway, +})); +vi.mock("../src/config", () => ({ + loadConfig: () => ({ port: 3207, hosts: ["127.0.0.1"], debug: false }), + providerForUpstreamOrigin: () => undefined, +})); + +import { commandRun } from "../src/cli/run"; + +describe("lore run adoption with a reused gateway", () => { + const priorOpenAIBaseUrl = process.env.OPENAI_BASE_URL; + const priorLoreUpstream = process.env.LORE_UPSTREAM_OPENAI; + const priorAnthropicBaseUrl = process.env.ANTHROPIC_BASE_URL; + const priorLoreAnthropic = process.env.LORE_UPSTREAM_ANTHROPIC; + const priorAnthropicToken = process.env.ANTHROPIC_AUTH_TOKEN; + + beforeEach(() => { + vi.clearAllMocks(); + process.env.OPENAI_BASE_URL = "https://proxy.example.com/v1"; + delete process.env.LORE_UPSTREAM_OPENAI; + delete process.env.ANTHROPIC_BASE_URL; + delete process.env.LORE_UPSTREAM_ANTHROPIC; + delete process.env.ANTHROPIC_AUTH_TOKEN; + mocks.startGateway.mockResolvedValue({ + port: 3207, + owned: false, + config: { port: 3207, hosts: ["127.0.0.1"], debug: false }, + shutdown: vi.fn(), + }); + }); + + afterEach(() => { + if (priorOpenAIBaseUrl === undefined) delete process.env.OPENAI_BASE_URL; + else process.env.OPENAI_BASE_URL = priorOpenAIBaseUrl; + if (priorLoreUpstream === undefined) + delete process.env.LORE_UPSTREAM_OPENAI; + else process.env.LORE_UPSTREAM_OPENAI = priorLoreUpstream; + if (priorAnthropicBaseUrl === undefined) + delete process.env.ANTHROPIC_BASE_URL; + else process.env.ANTHROPIC_BASE_URL = priorAnthropicBaseUrl; + if (priorLoreAnthropic === undefined) + delete process.env.LORE_UPSTREAM_ANTHROPIC; + else process.env.LORE_UPSTREAM_ANTHROPIC = priorLoreAnthropic; + if (priorAnthropicToken === undefined) + delete process.env.ANTHROPIC_AUTH_TOKEN; + else process.env.ANTHROPIC_AUTH_TOKEN = priorAnthropicToken; + }); + + test("does not launch an env-routed agent against an incompatible reused gateway", async () => { + await commandRun({}, ["codex"]); + + expect(mocks.safeExit).toHaveBeenCalledWith(1); + expect(mocks.spawn).not.toHaveBeenCalled(); + }); + + test("launches a header-routed agent against a reused gateway", async () => { + delete process.env.OPENAI_BASE_URL; + process.env.ANTHROPIC_BASE_URL = "https://proxy.example.com/v1"; + mocks.spawn.mockImplementationOnce(() => { + throw new Error("__agent_launched__"); + }); + + await expect(commandRun({}, ["claude"])).rejects.toThrow( + "__agent_launched__", + ); + expect(mocks.safeExit).not.toHaveBeenCalled(); + }); + + test.each([ + "https://user:secret@proxy.example.com/v1", + "https://proxy.example.com/v1?api_key=secret", + ])("fails before gateway startup for unsafe upstream %s", async (url) => { + delete process.env.OPENAI_BASE_URL; + process.env.ANTHROPIC_BASE_URL = url; + process.env.ANTHROPIC_AUTH_TOKEN = "private-proxy-token"; + + await commandRun({}, ["claude"]); + + expect(mocks.safeExit).toHaveBeenCalledWith(1); + expect(mocks.startGateway).not.toHaveBeenCalled(); + expect(mocks.spawn).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/gateway/test/run-upstream-adoption.test.ts b/packages/gateway/test/run-upstream-adoption.test.ts index fdd86c04..797179c0 100644 --- a/packages/gateway/test/run-upstream-adoption.test.ts +++ b/packages/gateway/test/run-upstream-adoption.test.ts @@ -3,6 +3,7 @@ import { AGENTS } from "../src/cli/agents"; import { applyUpstreamAdoption, adoptForRemote, + formatUpstreamForLog, injectAdoptionHeaders, resolveAgentSelection, resolveLaunchTarget, @@ -35,6 +36,7 @@ const TOUCHED_ENV = [ describe("lore run upstream adoption", () => { const claude = AGENTS.find((a) => a.name === "claude-code")!; + const codex = AGENTS.find((a) => a.name === "codex")!; const gemini = AGENTS.find((a) => a.name === "gemini")!; const opencode = AGENTS.find((a) => a.name === "opencode")!; @@ -90,8 +92,7 @@ describe("lore run upstream adoption", () => { }); test("returns null and touches no env when the user set nothing", () => { - const adopted = applyUpstreamAdoption(claude, GATEWAY); - expect(adopted).toBeNull(); + expect(applyUpstreamAdoption(claude, GATEWAY)).toBeNull(); expect(process.env.LORE_UPSTREAM_ANTHROPIC).toBeUndefined(); }); @@ -111,17 +112,11 @@ describe("lore run upstream adoption", () => { expect(headers).not.toContain("X-Lore-Provider:"); }); - test("Gemini (no header mechanism) adopts via env only — no anthropic header injected", () => { + test("Gemini custom upstream fails closed because no routing mechanism exists", () => { process.env.GOOGLE_GEMINI_BASE_URL = "https://gemini-proxy.example.com"; - const adopted = applyUpstreamAdoption(gemini, GATEWAY); - // gemini wire protocol has no LORE_UPSTREAM_ knob, so gateway - // env is not set — but the adoption object is still returned. - expect(adopted).not.toBeNull(); - expect(adopted!.url).toBe("https://gemini-proxy.example.com"); - // injectAdoptionHeaders is a no-op for non-anthropic agents. - const env: Record = {}; - injectAdoptionHeaders(gemini, env, adopted!); - expect(env.ANTHROPIC_CUSTOM_HEADERS).toBeUndefined(); + expect(() => applyUpstreamAdoption(gemini, GATEWAY)).toThrow( + /cannot safely route/i, + ); }); test("opencode (no adoptable base-URL var) returns null even with envs set", () => { @@ -140,14 +135,41 @@ describe("lore run upstream adoption", () => { // adopted at all — no header can be injected from it. process.env.ANTHROPIC_BASE_URL = "https://good.example.com/\nX-Api-Key: stolen"; - const adopted = applyUpstreamAdoption(claude, GATEWAY); - expect(adopted).toBeNull(); + expect(() => applyUpstreamAdoption(claude, GATEWAY)).toThrow( + /unsafe or invalid upstream URL/, + ); expect(process.env.LORE_UPSTREAM_ANTHROPIC).toBeUndefined(); }); test("a base URL with an interior space is rejected (not adopted)", () => { process.env.ANTHROPIC_BASE_URL = "https://good.example.com/ evil"; - expect(applyUpstreamAdoption(claude, GATEWAY)).toBeNull(); + expect(() => applyUpstreamAdoption(claude, GATEWAY)).toThrow( + /unsafe or invalid upstream URL/, + ); + }); + + test("a base URL containing userinfo credentials is rejected", () => { + process.env.ANTHROPIC_BASE_URL = + "https://api-user:secret@proxy.example.com/v1"; + expect(() => applyUpstreamAdoption(claude, GATEWAY)).toThrow( + /unsafe or invalid upstream URL/, + ); + }); + + test("query-bearing base URLs are rejected because routing drops the query", () => { + process.env.ANTHROPIC_BASE_URL = + "https://proxy.example.com/v1?api_key=super-secret"; + expect(() => applyUpstreamAdoption(claude, GATEWAY)).toThrow( + /unsafe or invalid upstream URL/, + ); + }); + + test("query strings are still redacted by defensive log formatting", () => { + expect( + formatUpstreamForLog( + "https://proxy.example.com/v1?api_key=super-secret#fragment", + ), + ).toBe("https://proxy.example.com/v1?"); }); // --- Remote mode: adopt via header only, never touch gateway env ---------- @@ -166,6 +188,13 @@ describe("lore run upstream adoption", () => { test("adoptForRemote returns null when the user set no upstream", () => { expect(adoptForRemote(claude, "https://remote-gw.example.com")).toBeNull(); }); + + test("remote adoption fails closed for agents without request headers", () => { + process.env.OPENAI_BASE_URL = "https://openai-proxy.example.com/v1"; + expect(() => + adoptForRemote(codex, "https://remote-gw.example.com"), + ).toThrow(/cannot safely route/i); + }); }); // --------------------------------------------------------------------------- diff --git a/packages/gateway/test/server-node-bridge.test.ts b/packages/gateway/test/server-node-bridge.test.ts index 579cfc3a..73e7daf2 100644 --- a/packages/gateway/test/server-node-bridge.test.ts +++ b/packages/gateway/test/server-node-bridge.test.ts @@ -3,6 +3,14 @@ import type { IncomingMessage, ServerResponse } from "node:http"; import { describe, expect, test, vi } from "vitest"; import { handleForegroundBodyRoute, handleNodeRequest } from "../src/server"; import { decodeRequestBody } from "../src/http-body"; +import type { GatewayRequest } from "../src/translate/types"; +import { + handleRequest, + resetPipelineState, + setPostResponseStartObserverForTest, + setUpstreamInterceptor, +} from "../src/pipeline"; +import { loadConfig } from "../src/config"; class FakeRequest extends EventEmitter { method = "GET"; @@ -20,6 +28,8 @@ class FakeResponse extends EventEmitter { status = 0; chunks: Uint8Array[] = []; writeResults: boolean[] = []; + backpressureOnText?: string; + backpressureTriggered = false; autoCloseOnEnd = true; writeHead(status: number): this { @@ -30,6 +40,14 @@ class FakeResponse extends EventEmitter { write(chunk: Uint8Array): boolean { this.chunks.push(chunk); + if ( + !this.backpressureTriggered && + this.backpressureOnText && + Buffer.from(chunk).toString("utf8").includes(this.backpressureOnText) + ) { + this.backpressureTriggered = true; + return false; + } return this.writeResults.shift() ?? true; } @@ -71,6 +89,34 @@ function expectListenersCleaned(req: FakeRequest, res: FakeResponse): void { expect(req.socket.listenerCount("error")).toBe(0); } +function responsesEvent(type: string, data: Record): string { + return `event: ${type}\ndata: ${JSON.stringify({ type, ...data })}\n\n`; +} + +function terminalResponsesSSE(id: string): string { + return ( + responsesEvent("response.created", { + response: { id, model: "gpt-5.6-sol", status: "in_progress" }, + }) + + responsesEvent("response.completed", { + response: { + id, + model: "gpt-5.6-sol", + status: "completed", + output: [], + usage: { input_tokens: 1, output_tokens: 0 }, + }, + }) + ); +} + +function loadLocalConfig() { + const config = loadConfig(); + config.remoteGateway = false; + config.hostedMode = false; + return config; +} + describe("node:http ingress lifecycle branches", () => { test.each(["aborted", "incomplete-close", "request-error"] as const)( "%s aborts pending handler work and removes listeners", @@ -278,6 +324,81 @@ describe("node:http ingress lifecycle branches", () => { expectListenersCleaned(req, res); }); + test("deferred streaming work starts after end despite terminal backpressure", async () => { + const req = new FakeRequest(); + req.complete = true; + const res = new FakeResponse(); + res.backpressureOnText = "event: response.completed"; + let postResponseSawEnd: boolean | undefined; + setPostResponseStartObserverForTest(() => { + postResponseSawEnd = res.writableEnded; + }); + setUpstreamInterceptor( + async () => + new Response(terminalResponsesSSE("resp_node_backpressure"), { + headers: { "content-type": "text/event-stream" }, + }), + ); + const gatewayRequest: GatewayRequest = { + protocol: "openai-responses", + model: "gpt-5.6-sol", + system: "You are a coding agent.", + messages: [ + { role: "user", content: [{ type: "text", text: "continue" }] }, + ], + tools: [{ name: "read", description: "Read a file", inputSchema: {} }], + stream: true, + maxTokens: 1024, + metadata: {}, + rawHeaders: { + authorization: "Bearer test-key", + "x-lore-agent": "coder", + "x-lore-project": process.cwd(), + "x-lore-provider": "openai", + "x-lore-upstream-url": "https://api.openai.com/v1", + "x-lore-session-id": "node-backpressure-session", + }, + }; + + try { + const handling = handleNodeRequest( + asRequest(req), + asResponse(res), + () => handleRequest(gatewayRequest, loadLocalConfig()), + "127.0.0.1", + 3207, + ); + await vi.waitFor( + () => { + expect(res.backpressureTriggered).toBe(true); + expect(res.listenerCount("drain")).toBeGreaterThan(0); + }, + { timeout: 10_000 }, + ); + + expect(Buffer.concat(res.chunks).toString("utf8")).toContain( + "event: response.completed", + ); + expect(res.writableEnded).toBe(false); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + expect(postResponseSawEnd).toBeUndefined(); + + res.emit("drain"); + await handling; + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + expect(res.writableEnded).toBe(true); + expect(postResponseSawEnd).toBe(true); + expectListenersCleaned(req, res); + } finally { + setPostResponseStartObserverForTest(undefined); + setUpstreamInterceptor(undefined); + await resetPipelineState(); + } + }); + test("disconnect while waiting for drain cancels without reading more", async () => { const req = new FakeRequest(); req.complete = true; diff --git a/packages/gateway/test/server.test.ts b/packages/gateway/test/server.test.ts index 86045249..cfea2c32 100644 --- a/packages/gateway/test/server.test.ts +++ b/packages/gateway/test/server.test.ts @@ -23,6 +23,10 @@ import { startServer } from "../src/server"; import { loadConfig } from "../src/config"; import type { GatewayConfig } from "../src/config"; import { MAX_HTTP_REQUEST_DECOMPRESSED_BYTES } from "../src/http-body"; +import { + loopbackRequest, + type LoopbackRequestInit, +} from "./helpers/loopback-request"; type ServerHandle = Awaited>; @@ -114,11 +118,19 @@ function makeConfig(overrides?: Partial): GatewayConfig { } let server: ServerHandle; -let baseURL: string; + +function localRequest( + port: number, + path: string, + init: LoopbackRequestInit = {}, + hostname = "127.0.0.1", +): Promise { + const host = hostname.includes(":") ? `[${hostname}]` : hostname; + return loopbackRequest(`http://${host}:${port}${path}`, init); +} beforeAll(async () => { server = await startServer(makeConfig()); - baseURL = `http://127.0.0.1:${server.port}`; }); afterAll(async () => { @@ -127,7 +139,7 @@ afterAll(async () => { describe("server routing", () => { test("owner control identity is unavailable without a configured token", async () => { - const res = await fetch(`${baseURL}/_lore/control`, { + const res = await localRequest(server.port, "/_lore/control", { headers: { authorization: "Bearer attacker" }, }); expect(res.status).toBe(404); @@ -315,7 +327,9 @@ describe("server routing", () => { }); test("no-Origin OPTIONS remains a 204 without enabling browser CORS", async () => { - const res = await fetch(`${baseURL}/v1/messages`, { method: "OPTIONS" }); + const res = await localRequest(server.port, "/v1/messages", { + method: "OPTIONS", + }); expect(res.status).toBe(204); expect(res.headers.get("access-control-allow-origin")).toBeNull(); expect(res.headers.get("access-control-allow-methods")).toBeNull(); @@ -323,7 +337,7 @@ describe("server routing", () => { }); test("GET /health remains public without making it browser-readable cross-origin", async () => { - const res = await fetch(`${baseURL}/health`); + const res = await localRequest(server.port, "/health"); expect(res.status).toBe(200); const body = (await res.json()) as { status: string; version: string }; expect(body.status).toBe("ok"); @@ -332,7 +346,7 @@ describe("server routing", () => { }); test("unknown route returns a 404 error envelope", async () => { - const res = await fetch(`${baseURL}/definitely-not-a-route`); + const res = await localRequest(server.port, "/definitely-not-a-route"); expect(res.status).toBe(404); const body = (await res.json()) as { type: string; @@ -348,7 +362,7 @@ describe("server routing", () => { "/v1/responses", "/v1/codex/responses", ])("POST %s with invalid JSON returns 400", async (path) => { - const res = await fetch(`${baseURL}${path}`, { + const res = await localRequest(server.port, path, { method: "POST", headers: { "content-type": "application/json" }, body: "{ this is not valid json", @@ -372,7 +386,7 @@ describe("server routing", () => { "/v1/codex/responses", ])("POST %s with a zstd body of invalid JSON returns 400", async (path) => { const compressed = zstdCompressSync(Buffer.from("{ not json after all")); - const res = await fetch(`${baseURL}${path}`, { + const res = await localRequest(server.port, path, { method: "POST", headers: { "content-type": "application/json", @@ -389,35 +403,37 @@ describe("server routing", () => { }); test("GET /v1/models returns 502 when the upstream is unreachable", async () => { - const res = await fetch(`${baseURL}/v1/models`); + const res = await localRequest(server.port, "/v1/models"); expect(res.status).toBe(502); const body = (await res.json()) as { error: { type: string } }; expect(body.error.type).toBe("api_error"); }); - test("POST /v1/responses/compact rejects invalid JSON on the exact route", async () => { - const res = await fetch(`${baseURL}/v1/responses/compact`, { + test("POST /v1/responses/compact rejects invalid JSON before routing", async () => { + const res = await localRequest(server.port, "/v1/responses/compact", { method: "POST", - headers: { "content-type": "application/json" }, + headers: { + "content-type": "application/json", + "x-api-key": "test-key", + }, body: "{ not json", }); expect(res.status).toBe(400); - await expect(res.json()).resolves.toEqual({ + await expect(res.json()).resolves.toMatchObject({ error: "invalid_request", - message: "Invalid JSON body", }); }); test.each([ - ["/v1/messages", "gzip"], - ["/v1/chat/completions", "br"], - ["/v1/responses", "zstd"], - ["/v1beta/models/gemini-test:generateContent", "gzip"], - ["/v1/compact", "br"], - ["/v1/responses/compact", "zstd"], + ["/v1/messages", "gzip", 400], + ["/v1/chat/completions", "br", 400], + ["/v1/responses", "zstd", 400], + ["/v1beta/models/gemini-test:generateContent", "gzip", 400], + ["/v1/compact", "br", 404], + ["/v1/responses/compact", "zstd", 400], ] as const)( "POST %s rejects a %s decompression bomb", - async (path, encoding) => { + async (path, encoding, expectedStatus) => { const expanded = Buffer.alloc( MAX_HTTP_REQUEST_DECOMPRESSED_BYTES + 1, 0x61, @@ -428,7 +444,7 @@ describe("server routing", () => { : encoding === "br" ? brotliCompressSync(expanded) : zstdCompressSync(expanded); - const res = await fetch(`${baseURL}${path}`, { + const res = await localRequest(server.port, path, { method: "POST", headers: { "content-type": "application/json", @@ -437,7 +453,7 @@ describe("server routing", () => { }, body: compressed, }); - expect(res.status).toBe(400); + expect(res.status).toBe(expectedStatus); }, ); @@ -474,13 +490,11 @@ describe("server routing", () => { }); test("GET / redirects toward the dashboard (not a 500)", async () => { - const res = await fetch(`${baseURL}/`, { redirect: "manual" }); - // undici surfaces a manual redirect as an opaqueredirect (status 0); a - // real 3xx is also acceptable. Regression guard: Response.redirect()'s - // headers are immutable, so the old CORS wrapper used to throw and the - // root path 500'd instead of redirecting. + const res = await localRequest(server.port, "/"); + // Regression guard: Response.redirect()'s headers are immutable, so + // the old CORS wrapper used to throw and the root path returned 500. expect(res.status).not.toBe(500); - expect([0, 301, 302, 307, 308]).toContain(res.status); + expect([301, 302, 307, 308]).toContain(res.status); }); }); @@ -489,7 +503,7 @@ describe("startServer configuration", () => { const s = await startServer(makeConfig({ hosts: [] })); try { expect(s.hosts).toEqual(["127.0.0.1"]); - const res = await fetch(`http://127.0.0.1:${s.port}/health`); + const res = await localRequest(s.port, "/health"); expect(res.status).toBe(200); } finally { await s.stop(); @@ -499,7 +513,7 @@ describe("startServer configuration", () => { test("debug mode serves requests (covers debug logging branch)", async () => { const s = await startServer(makeConfig({ debug: true })); try { - const res = await fetch(`http://127.0.0.1:${s.port}/health`); + const res = await localRequest(s.port, "/health"); expect(res.status).toBe(200); } finally { await s.stop(); @@ -521,7 +535,7 @@ describe("startServer configuration", () => { try { // The unavailable host is dropped; only the bound host remains. expect(s.hosts).toEqual(["127.0.0.1"]); - const res = await fetch(`http://127.0.0.1:${s.port}/health`); + const res = await localRequest(s.port, "/health"); expect(res.status).toBe(200); } finally { await s.stop(); @@ -538,7 +552,7 @@ describe("startServer configuration", () => { try { expect(s.hosts).toEqual(["127.0.0.1"]); expect(s.port).toBeGreaterThan(0); - const res = await fetch(`http://127.0.0.1:${s.port}/health`); + const res = await localRequest(s.port, "/health"); expect(res.status).toBe(200); } finally { await s.stop(); @@ -575,7 +589,7 @@ describe("startServer configuration", () => { } try { expect(s.hosts).toContain("::1"); - const res = await fetch(`http://[::1]:${s.port}/health`); + const res = await localRequest(s.port, "/health", {}, "::1"); expect(res.status).toBe(200); const body = (await res.json()) as { status: string }; expect(body.status).toBe("ok"); diff --git a/packages/gateway/test/session-adoption.e2e.test.ts b/packages/gateway/test/session-adoption.e2e.test.ts index 064a484b..a31c323e 100644 --- a/packages/gateway/test/session-adoption.e2e.test.ts +++ b/packages/gateway/test/session-adoption.e2e.test.ts @@ -17,10 +17,11 @@ * the real HTTP server, simulate a restart via `harness.restartPipeline()` * (clears in-memory maps, keeps the DB), and assert on session_state rows. */ -import { describe, it, expect, afterEach } from "vitest"; +import { describe, it, expect, afterEach, vi } from "vitest"; import { DatabaseSync } from "node:sqlite"; import type { Harness } from "./helpers/harness"; -import { createHarness } from "./helpers/harness"; +import { createHarness, TEST_GATEWAY_AUTH_TOKEN } from "./helpers/harness"; +import { makeReplayInterceptor } from "./helpers/replay"; import { fingerprintMessages } from "../src/session"; import { deterministicID } from "../src/temporal-adapter"; import { @@ -29,10 +30,19 @@ import { DEFAULT_MODEL, DEFAULT_SYSTEM, } from "./helpers/fixtures"; +import { + getActiveSessions, + pendingPipelineSessionClaimCountForTest, + setProvisionalFinalizerPauseForTest, + setUpstreamInterceptor, +} from "../src/pipeline"; +import { authFingerprint, getLastSeenAuth, resolveAuth } from "../src/auth"; +import { enableHostedMode, _resetHostedModeForTest } from "@loreai/core"; const U0 = "alpha first task: please implement the parser module"; const U1 = "second follow-up: now add tests for the parser"; const U2 = "third instruction after the restart: refactor the helper"; +const U3 = "fourth instruction after another restart: verify migration"; function body(messages: Array<{ role: string; content: string }>) { return { @@ -55,11 +65,16 @@ function fixtures() { ]; } -type Row = { session_id: string; header_session_id: string | null }; +type Row = { + session_id: string; + header_session_id: string | null; + message_count: number; + project_path: string | null; +}; function loreSessionRows(h: Harness): Row[] { return h.queryDB( - "SELECT session_id, header_session_id FROM session_state WHERE header_name = 'x-lore-session-id'", + "SELECT session_id, header_session_id, message_count, project_path FROM session_state WHERE header_name = 'x-lore-session-id'", ); } @@ -88,6 +103,7 @@ describe("issue #796: restart-proof session adoption (Tier 3b)", () => { afterEach(async () => { if (harness) await harness.teardown(); + _resetHostedModeForTest(); }); it("adopts the prior session when a resumed conversation arrives under a new x-lore-session-id after restart", async () => { @@ -99,6 +115,8 @@ describe("issue #796: restart-proof session adoption (Tier 3b)", () => { }); expect(r.status).toBe(200); await r.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); // Turn 2 (same session V1 continues) — stores u1, updates message_count. r = await harness.chat( @@ -112,6 +130,8 @@ describe("issue #796: restart-proof session adoption (Tier 3b)", () => { ); expect(r.status).toBe(200); await r.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); // Exactly one conversation session so far, bound to V1. let rows = loreSessionRows(harness); @@ -136,6 +156,8 @@ describe("issue #796: restart-proof session adoption (Tier 3b)", () => { ); expect(r.status).toBe(200); await r.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); // Adopted: still ONE conversation session (no new row), same internal id, // now rebound to the new header value for the Tier-1 fast path. @@ -145,6 +167,504 @@ describe("issue #796: restart-proof session adoption (Tier 3b)", () => { expect(rows[0].header_session_id).toBe("V2"); }); + it("does not adopt a fingerprint candidate whose persisted credential owner differs", async () => { + harness = await createHarness({ fixtures: fixtures() }); + let response = await harness.chat( + body([{ role: "user", content: U0 }]), + "key-A", + { "x-lore-session-id": "owner-V1" }, + ); + await response.text(); + response = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + ]), + "key-A", + { "x-lore-session-id": "owner-V1" }, + ); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const original = loreSessionRows(harness)[0]; + const database = new DatabaseSync(harness.dbPath); + try { + database + .prepare( + "UPDATE session_state SET credential_fingerprint = ? WHERE session_id = ?", + ) + .run("different-owner", original.session_id); + } finally { + database.close(); + } + await harness.restartPipeline(); + + response = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + { role: "assistant", content: "A1 done." }, + { role: "user", content: U2 }, + ]), + "key-A", + { "x-lore-session-id": "owner-V2" }, + ); + expect(response.status).toBe(200); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const rows = loreSessionRows(harness); + expect(rows).toHaveLength(2); + expect( + rows.find((row) => row.header_session_id === "owner-V1")?.session_id, + ).toBe(original.session_id); + expect( + rows.find((row) => row.header_session_id === "owner-V2")?.session_id, + ).not.toBe(original.session_id); + }); + + it("fails closed for duplicate persisted header identities", async () => { + harness = await createHarness({ fixtures: fixtures() }); + const credentialFingerprint = authFingerprint({ + scheme: "api-key", + value: "duplicate-key", + }); + const database = new DatabaseSync(harness.dbPath); + try { + const insert = database.prepare( + `INSERT INTO session_state + (session_id, force_min_layer, updated_at, header_name, + header_session_id, credential_fingerprint, project_path, + project_path_provisional) + VALUES (?, 0, ?, 'x-lore-session-id', 'duplicate-header', ?, ?, 0)`, + ); + insert.run( + `duplicate-a-${crypto.randomUUID()}`, + Date.now(), + credentialFingerprint, + "/tmp/duplicate-project-a", + ); + insert.run( + `duplicate-b-${crypto.randomUUID()}`, + Date.now(), + credentialFingerprint, + "/tmp/duplicate-project-b", + ); + } finally { + database.close(); + } + await harness.restartPipeline(); + + const response = await harness.chat( + body([{ role: "user", content: U0 }]), + "duplicate-key", + { "x-lore-session-id": "duplicate-header" }, + ); + expect(response.status).not.toBe(200); + await response.text(); + expect(harness.upstreamBodies()).toHaveLength(0); + expect( + harness.queryDB( + "SELECT session_id FROM session_state WHERE header_session_id = 'duplicate-header'", + ), + ).toHaveLength(2); + }); + + it("does not persist fingerprint adoption after a failed resumed turn", async () => { + harness = await createHarness({ fixtures: fixtures() }); + let r = await harness.chat(body([{ role: "user", content: U0 }]), "key-A", { + "x-lore-session-id": "V1", + }); + await r.text(); + r = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + ]), + "key-A", + { "x-lore-session-id": "V1" }, + ); + await r.text(); + const original = loreSessionRows(harness)[0]; + await makeSessionLegacy(harness, original.session_id); + await harness.restartPipeline(); + const globalAuthBefore = getLastSeenAuth("anthropic")?.value; + + setUpstreamInterceptor(async () => + Promise.resolve( + new Response("provider failed", { + status: 500, + headers: { "content-type": "text/plain" }, + }), + ), + ); + r = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + { role: "assistant", content: "A1 done." }, + { role: "user", content: U2 }, + ]), + "key-A", + { "x-lore-session-id": "V2" }, + ); + expect(r.status).toBe(500); + await r.text(); + + const rows = loreSessionRows(harness); + expect(rows).toEqual([original]); + // A failed provisional turn must not repopulate worker credentials after + // restart; only a successfully committed turn may publish them. + expect(resolveAuth(original.session_id)).toBeNull(); + // Local direct-provider requests refresh the legacy process-global fallback + // at ingress even when their provisional session commit later fails. + expect(globalAuthBefore).toBeUndefined(); + expect(getLastSeenAuth("anthropic")?.value).toBe("key-A"); + + const retryReplay = makeReplayInterceptor([ + makeFixtureEntry({ + seq: 0, + requestMessages: [], + responseText: "Retry succeeded.", + }), + ]); + setUpstreamInterceptor(retryReplay); + r = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + { role: "assistant", content: "A1 done." }, + { role: "user", content: U2 }, + ]), + "key-A", + { "x-lore-session-id": "V2" }, + ); + expect(r.status).toBe(200); + await r.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + await harness.restartPipeline(); + r = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + { role: "assistant", content: "A1 done." }, + { role: "user", content: U2 }, + { role: "assistant", content: "Retry succeeded." }, + { role: "user", content: U3 }, + ]), + "key-A", + { "x-lore-session-id": "V3" }, + ); + expect(r.status).toBe(200); + await r.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const afterSecondRestart = loreSessionRows(harness); + expect(afterSecondRestart).toHaveLength(1); + expect(afterSecondRestart[0].session_id).toBe(original.session_id); + expect(afterSecondRestart[0].header_session_id).toBe("V3"); + }); + + it("rechecks legacy ownership inside the provisional commit savepoint", async () => { + harness = await createHarness({ fixtures: fixtures() }); + let response = await harness.chat( + body([{ role: "user", content: U0 }]), + "key-A", + { "x-lore-session-id": "savepoint-owner-old" }, + ); + await response.text(); + response = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + ]), + "key-A", + { "x-lore-session-id": "savepoint-owner-old" }, + ); + await response.text(); + const [original] = loreSessionRows(harness); + await makeSessionLegacy(harness, original.session_id); + const [legacy] = harness.queryDB<{ fingerprint: string }>( + "SELECT fingerprint FROM session_state WHERE session_id = ?", + [original.session_id], + ); + await harness.restartPipeline(); + + let releaseFinalizer!: () => void; + const finalizerPause = new Promise((resolve) => { + releaseFinalizer = resolve; + }); + let finalizerWaitingResolve!: () => void; + const finalizerWaiting = new Promise((resolve) => { + finalizerWaitingResolve = resolve; + }); + setProvisionalFinalizerPauseForTest( + finalizerPause, + finalizerWaitingResolve, + ); + + try { + response = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + { role: "assistant", content: "A1 done." }, + { role: "user", content: U2 }, + ]), + "key-A", + { "x-lore-session-id": "savepoint-owner-new" }, + ); + expect(response.status).toBe(200); + await response.text(); + await finalizerWaiting; + + const database = new DatabaseSync(harness.dbPath); + try { + database + .prepare( + "UPDATE session_state SET credential_fingerprint = ? WHERE session_id = ?", + ) + .run("external-owner", original.session_id); + } finally { + database.close(); + } + releaseFinalizer(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const [after] = harness.queryDB<{ + header_session_id: string; + credential_fingerprint: string; + fingerprint: string; + }>( + `SELECT header_session_id, credential_fingerprint, fingerprint + FROM session_state + WHERE session_id = ?`, + [original.session_id], + ); + expect(after.header_session_id).toBe("savepoint-owner-old"); + expect(after.credential_fingerprint).toBe("external-owner"); + expect(after.fingerprint).toBe(legacy.fingerprint); + expect( + harness.queryDB( + "SELECT id FROM temporal_messages WHERE session_id = ? AND content LIKE ?", + [original.session_id, `%${U2}%`], + ), + ).toHaveLength(0); + } finally { + releaseFinalizer(); + setProvisionalFinalizerPauseForTest(undefined); + } + }); + + it("persists the resumed turn when fingerprint adoption has no session header", async () => { + harness = await createHarness({ fixtures: fixtures() }); + let r = await harness.chat(body([{ role: "user", content: U0 }]), "key-A", { + "x-lore-session-id": "V1", + }); + await r.text(); + r = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + ]), + "key-A", + { "x-lore-session-id": "V1" }, + ); + await r.text(); + const original = loreSessionRows(harness)[0]; + await harness.restartPipeline(); + + r = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + { role: "assistant", content: "A1 done." }, + { role: "user", content: U2 }, + ]), + "key-A", + {}, + ); + expect(r.status).toBe(200); + await r.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const rows = loreSessionRows(harness); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + session_id: original.session_id, + header_session_id: "V1", + message_count: 5, + }); + expect( + harness.queryDB<{ count: number }>( + "SELECT COUNT(*) AS count FROM temporal_messages WHERE session_id = ? AND content LIKE ?", + [original.session_id, `%${U2}%`], + )[0]?.count, + ).toBeGreaterThan(0); + }); + + it("adopts a resumed conversation from a new clone path matched by git remote", async () => { + enableHostedMode(); + const originalPath = "/client/checkouts/adoption-original"; + const clonePath = "/client/checkouts/adoption-clone"; + const remote = `github.com/test/adoption-${crypto.randomUUID()}`; + harness = await createHarness({ + fixtures: fixtures(), + configOverrides: { + hostedMode: true, + gatewayAuthToken: TEST_GATEWAY_AUTH_TOKEN, + }, + }); + + let r = await harness.chat(body([{ role: "user", content: U0 }]), "key-A", { + "x-lore-session-id": "clone-V1", + "x-lore-project": originalPath, + "x-lore-git-remote": remote, + "x-lore-gateway-token": TEST_GATEWAY_AUTH_TOKEN, + }); + expect(r.status).toBe(200); + await r.text(); + r = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + ]), + "key-A", + { + "x-lore-session-id": "clone-V1", + "x-lore-project": originalPath, + "x-lore-git-remote": remote, + "x-lore-gateway-token": TEST_GATEWAY_AUTH_TOKEN, + }, + ); + expect(r.status).toBe(200); + await r.text(); + const original = loreSessionRows(harness)[0]; + await harness.restartPipeline(); + + // This path has never been registered as an alias. The persisted project's + // normalized remote is the only signal that scopes overlap to the original + // project rather than minting a second session for the clone. + r = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + { role: "assistant", content: "A1 done." }, + { role: "user", content: U2 }, + ]), + "key-A", + { + "x-lore-session-id": "clone-V2", + "x-lore-project": clonePath, + "x-lore-git-remote": remote, + "x-lore-gateway-token": TEST_GATEWAY_AUTH_TOKEN, + }, + ); + expect(r.status).toBe(200); + await r.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const rows = loreSessionRows(harness); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + session_id: original.session_id, + header_session_id: "clone-V2", + project_path: clonePath, + }); + }); + + it("does not trust a spoofed clone remote on a local gateway", async () => { + const originalPath = "/tmp/adoption-local-original"; + const unrelatedPath = "/tmp/adoption-local-unrelated"; + const spoofedRemote = `github.com/test/adoption-spoof-${crypto.randomUUID()}`; + harness = await createHarness({ fixtures: fixtures() }); + + let r = await harness.chat(body([{ role: "user", content: U0 }]), "key-A", { + "x-lore-session-id": "spoof-V1", + "x-lore-project": originalPath, + }); + await r.text(); + r = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + ]), + "key-A", + { + "x-lore-session-id": "spoof-V1", + "x-lore-project": originalPath, + }, + ); + await r.text(); + const original = loreSessionRows(harness)[0]; + const database = new DatabaseSync(harness.dbPath); + try { + database + .prepare("UPDATE projects SET git_remote = ? WHERE path = ?") + .run(spoofedRemote, originalPath); + } finally { + database.close(); + } + await harness.restartPipeline(); + + // unrelatedPath is not a git repository on this local gateway. A forged + // header must not select originalPath's project and authorize adoption. + r = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + { role: "assistant", content: "A1 done." }, + { role: "user", content: U2 }, + ]), + "key-A", + { + "x-lore-session-id": "spoof-V2", + "x-lore-project": unrelatedPath, + "x-lore-git-remote": spoofedRemote, + }, + ); + expect(r.status).toBe(200); + await r.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const rows = loreSessionRows(harness); + expect(rows).toHaveLength(2); + expect( + rows.find((row) => row.session_id === original.session_id), + ).toMatchObject({ + header_session_id: "spoof-V1", + project_path: originalPath, + }); + expect( + rows.find((row) => row.header_session_id === "spoof-V2"), + ).toMatchObject({ + project_path: unrelatedPath, + }); + }); + it("does NOT adopt when the resumed conversation has a different fingerprint (different first message)", async () => { harness = await createHarness({ fixtures: fixtures() }); @@ -228,7 +748,16 @@ describe("issue #796: restart-proof session adoption (Tier 3b)", () => { }); it("adopts a pre-credential session only after same-project transcript overlap", async () => { - harness = await createHarness({ fixtures: fixtures() }); + harness = await createHarness({ + fixtures: [ + ...fixtures(), + makeFixtureEntry({ + seq: 3, + requestMessages: [], + responseText: "A3 done.", + }), + ], + }); const legacyHeader = "legacy-exact-header"; let r = await harness.chat(body([{ role: "user", content: U0 }]), "key-A", { @@ -267,6 +796,8 @@ describe("issue #796: restart-proof session adoption (Tier 3b)", () => { ); expect(r.status).toBe(200); await r.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); const after = harness.queryDB( `SELECT session_id, header_session_id, credential_fingerprint @@ -277,6 +808,312 @@ describe("issue #796: restart-proof session adoption (Tier 3b)", () => { expect(after[0].session_id).toBe(before[0].session_id); expect(after[0].header_session_id).toBe(migratedHeader); expect(after[0].credential_fingerprint).toMatch(/^[0-9a-f]{16}$/); + + const staleAlias = await harness.chat( + body([{ role: "user", content: "/lore:amnesia:on" }]), + "", + { "x-lore-session-id": legacyHeader }, + ); + expect(staleAlias.status).toBe(200); + expect(await staleAlias.text()).toMatch(/no active session/i); + expect( + harness.queryDB<{ amnesia: number }>( + "SELECT amnesia FROM session_state WHERE session_id = ?", + [before[0].session_id], + )[0]?.amnesia, + ).toBe(0); + + await harness.restartPipeline(); + const secondMigratedHeader = "legacy-after-second-restart"; + r = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + { role: "assistant", content: "A1 done." }, + { role: "user", content: U2 }, + { role: "assistant", content: "A2 done." }, + { role: "user", content: U3 }, + ]), + "key-A", + { "x-lore-session-id": secondMigratedHeader }, + ); + expect(r.status).toBe(200); + await r.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const afterSecondRestart = loreSessionRows(harness); + expect(afterSecondRestart).toHaveLength(1); + expect(afterSecondRestart[0].session_id).toBe(before[0].session_id); + expect(afterSecondRestart[0].header_session_id).toBe(secondMigratedHeader); + }); + + it("does not adopt an ownerless legacy session without a presented credential", async () => { + harness = await createHarness({ fixtures: fixtures() }); + let response = await harness.chat( + body([{ role: "user", content: U0 }]), + "key-A", + { "x-lore-session-id": "legacy-auth-required-old" }, + ); + await response.text(); + response = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + ]), + "key-A", + { "x-lore-session-id": "legacy-auth-required-old" }, + ); + await response.text(); + const [original] = loreSessionRows(harness); + await makeSessionLegacy(harness, original.session_id); + await harness.restartPipeline(); + + response = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + { role: "assistant", content: "A1 done." }, + { role: "user", content: U2 }, + ]), + "", + { "x-lore-session-id": "legacy-auth-required-new" }, + ); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const rows = loreSessionRows(harness); + expect( + rows.find((row) => row.header_session_id === "legacy-auth-required-old") + ?.session_id, + ).toBe(original.session_id); + const independent = rows.find( + (row) => row.header_session_id === "legacy-auth-required-new", + ); + expect(independent).toBeDefined(); + expect(independent?.session_id).not.toBe(original.session_id); + }); + + it("does not fingerprint-match credentialless headerless sessions", async () => { + harness = await createHarness({ fixtures: fixtures() }); + let response = await harness.chat( + body([{ role: "user", content: U0 }]), + "", + { "x-lore-project": "/tmp/credentialless-project-a" }, + ); + expect(response.status).toBe(200); + await response.text(); + response = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + ]), + "", + { "x-lore-project": "/tmp/credentialless-project-b" }, + ); + expect(response.status).toBe(200); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const rows = harness.queryDB<{ session_id: string; project_path: string }>( + `SELECT session_id, project_path + FROM session_state + WHERE project_path IN (?, ?)`, + ["/tmp/credentialless-project-a", "/tmp/credentialless-project-b"], + ); + expect(rows).toHaveLength(2); + expect(new Set(rows.map((row) => row.session_id)).size).toBe(2); + }); + + it("requires the live fingerprint candidate to have the same credential owner", async () => { + harness = await createHarness({ fixtures: fixtures() }); + let response = await harness.chat( + body([{ role: "user", content: U0 }]), + "key-A", + { "x-lore-project": "/tmp/live-owner-project" }, + ); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const [state] = getActiveSessions().values(); + state.fingerprint = await fingerprintMessages( + [{ role: "user", content: U0 }], + { + authSuffix: authFingerprint({ + scheme: "api-key", + value: "key-B", + }), + }, + ); + + response = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + ]), + "key-B", + { "x-lore-project": "/tmp/live-owner-project" }, + ); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const rows = harness.queryDB<{ session_id: string }>( + "SELECT session_id FROM session_state WHERE project_path = ?", + ["/tmp/live-owner-project"], + ); + expect(new Set(rows.map((row) => row.session_id)).size).toBe(2); + }); + + it("does not live-match an authenticated fingerprint across confident projects", async () => { + harness = await createHarness({ fixtures: fixtures() }); + let response = await harness.chat( + body([{ role: "user", content: U0 }]), + "key-A", + { "x-lore-project": "/tmp/live-project-a" }, + ); + await response.text(); + response = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + ]), + "key-A", + { "x-lore-project": "/tmp/live-project-b" }, + ); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const rows = harness.queryDB<{ session_id: string; project_path: string }>( + `SELECT session_id, project_path + FROM session_state + WHERE project_path IN (?, ?)`, + ["/tmp/live-project-a", "/tmp/live-project-b"], + ); + expect(rows).toHaveLength(2); + expect(new Set(rows.map((row) => row.session_id)).size).toBe(2); + }); + + it("fails closed for equally close live fingerprint candidates", async () => { + harness = await createHarness({ fixtures: fixtures() }); + let response = await harness.chat( + body([{ role: "user", content: U0 }]), + "key-A", + { "x-lore-project": "/tmp/live-ambiguous-project" }, + ); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + const active = getActiveSessions(); + const [state] = active.values(); + const duplicateSessionID = `live-duplicate-${crypto.randomUUID()}`; + (active as Map).set(duplicateSessionID, { + ...state, + sessionID: duplicateSessionID, + }); + + response = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + ]), + "key-A", + { "x-lore-project": "/tmp/live-ambiguous-project" }, + ); + await response.text(); + expect(getActiveSessions().size).toBe(3); + }); + + it("allows only the first concurrent credential to claim a legacy session", async () => { + harness = await createHarness({ fixtures: fixtures() }); + let response = await harness.chat( + body([{ role: "user", content: U0 }]), + "seed-key", + { "x-lore-session-id": "legacy-race-seed" }, + ); + await response.text(); + response = await harness.chat( + body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + ]), + "seed-key", + { "x-lore-session-id": "legacy-race-seed" }, + ); + await response.text(); + const [original] = loreSessionRows(harness); + await makeSessionLegacy(harness, original.session_id); + await harness.restartPipeline(); + + let releaseFirst!: () => void; + const firstPause = new Promise((resolve) => { + releaseFirst = resolve; + }); + let firstWaitingResolve!: () => void; + const firstWaiting = new Promise((resolve) => { + firstWaitingResolve = resolve; + }); + const replay = makeReplayInterceptor([ + makeFixtureEntry({ + seq: 0, + requestMessages: [], + responseText: "First claimant succeeded.", + }), + ]); + let upstreamCalls = 0; + setUpstreamInterceptor(async (...args) => { + upstreamCalls++; + firstWaitingResolve(); + await firstPause; + return replay(...args); + }); + + const resumed = body([ + { role: "user", content: U0 }, + { role: "assistant", content: "A0 done." }, + { role: "user", content: U1 }, + { role: "assistant", content: "A1 done." }, + { role: "user", content: U2 }, + ]); + try { + const first = harness.chat(resumed, "key-A", { + "x-lore-session-id": "legacy-race-a", + }); + await firstWaiting; + const second = harness.chat(resumed, "key-B", { + "x-lore-session-id": "legacy-race-b", + }); + await vi.waitFor(() => + expect(pendingPipelineSessionClaimCountForTest()).toBe(1), + ); + + releaseFirst(); + response = await first; + expect(response.status).toBe(200); + await response.text(); + const rejected = await second; + expect(rejected.status).toBe(404); + expect(await rejected.text()).toMatch(/authenticated session/i); + expect(upstreamCalls).toBe(1); + + const rows = loreSessionRows(harness); + expect(rows).toHaveLength(1); + expect(rows[0].session_id).toBe(original.session_id); + expect(rows[0].header_session_id).toBe("legacy-race-a"); + } finally { + releaseFirst(); + } }); it("does NOT let another transcript claim an exact legacy header", async () => { diff --git a/packages/gateway/test/session-credential-learning.test.ts b/packages/gateway/test/session-credential-learning.test.ts index ac689f3f..d3d1f8c0 100644 --- a/packages/gateway/test/session-credential-learning.test.ts +++ b/packages/gateway/test/session-credential-learning.test.ts @@ -156,6 +156,8 @@ describe("Tier 2 credential exclusion", () => { ], headersA, ); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); const learned = harness.queryDB<{ header_name: string; diff --git a/packages/gateway/test/session-rotation-merge.test.ts b/packages/gateway/test/session-rotation-merge.test.ts index c262337b..c0eaacc9 100644 --- a/packages/gateway/test/session-rotation-merge.test.ts +++ b/packages/gateway/test/session-rotation-merge.test.ts @@ -20,6 +20,7 @@ * tests in session.test.ts. */ import { describe, it, expect, afterEach } from "vitest"; +import { DatabaseSync } from "node:sqlite"; import type { Harness } from "./helpers/harness"; import { createHarness } from "./helpers/harness"; import { @@ -28,6 +29,7 @@ import { DEFAULT_MODEL, DEFAULT_SYSTEM, } from "./helpers/fixtures"; +import { enableHostedMode, _resetHostedModeForTest } from "@loreai/core"; // A faithful Claude Code coding turn carries the anchored OAuth billing header // at system[0] (Claude Code emits it whenever `_CLAUDE_CODE_ASSUME_FIRST_PARTY_ @@ -67,6 +69,7 @@ describe("Tier 1b session-merge regression (x-claude-code-session-id)", () => { afterEach(async () => { if (harness) await harness.teardown(); + _resetHostedModeForTest(); }); it("does NOT merge two distinct Claude Code conversations into one session", async () => { @@ -229,7 +232,7 @@ describe("Tier 1b session-merge regression (x-claude-code-session-id)", () => { }); }); -describe("Tier 1b rotation: x-session-affinity (OpenCode nanoid) still rotates safely", () => { +describe("x-session-affinity restart adoption", () => { let harness: Harness; afterEach(async () => { @@ -278,31 +281,64 @@ describe("Tier 1b rotation: x-session-affinity (OpenCode nanoid) still rotates s ); }); - it("allows rotation when the project is unchanged (genuine restart)", async () => { - // Same project, new nanoid → legitimate OpenCode restart → resume the - // SAME session (the original purpose of Tier 1b). + it("resumes a genuine restart only with project-scoped multi-message overlap", async () => { + // A new nanoid is not continuity evidence by itself. A genuine OpenCode + // restart resumes only after fingerprint adoption confirms at least two + // leading user messages in the same project. harness = await createHarness({ - fixtures: [ - ...makeConversationFixtures([ - { userMessage: "restart one", assistantText: "S1." }, - ]), - ...makeConversationFixtures([ - { userMessage: "restart two", assistantText: "S2." }, - ]), - ], + fixtures: makeConversationFixtures([ + { userMessage: "restart one", assistantText: "S1." }, + { userMessage: "restart two", assistantText: "S2." }, + { userMessage: "restart three", assistantText: "S3." }, + ]), }); - const r1 = await harness.chat(body("restart one"), "key-A", { + let r1 = await harness.chat(body("restart one"), "key-A", { "x-session-affinity": "nanoid-before-restart", "x-lore-project": "/proj/oc-same", }); expect(r1.status).toBe(200); await r1.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); - const r2 = await harness.chat(body("restart two"), "key-A", { - "x-session-affinity": "nanoid-after-restart", - "x-lore-project": "/proj/oc-same", - }); + r1 = await harness.chat( + { + ...body("restart two"), + messages: [ + { role: "user", content: "restart one" }, + { role: "assistant", content: "S1." }, + { role: "user", content: "restart two" }, + ], + }, + "key-A", + { + "x-session-affinity": "nanoid-before-restart", + "x-lore-project": "/proj/oc-same", + }, + ); + expect(r1.status).toBe(200); + await r1.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const r2 = await harness.chat( + { + ...body("restart three"), + messages: [ + { role: "user", content: "restart one" }, + { role: "assistant", content: "S1." }, + { role: "user", content: "restart two" }, + { role: "assistant", content: "S2." }, + { role: "user", content: "restart three" }, + ], + }, + "key-A", + { + "x-session-affinity": "nanoid-after-restart", + "x-lore-project": "/proj/oc-same", + }, + ); expect(r2.status).toBe(200); await r2.text(); @@ -315,9 +351,9 @@ describe("Tier 1b rotation: x-session-affinity (OpenCode nanoid) still rotates s expect(sessions[0].project_path).toBe("/proj/oc-same"); }); - it("allows rotation when no X-Lore-Project header is sent (Fix 2 is a no-op)", async () => { - // When the incoming request has no confident project header, Fix 2 cannot - // compare projects → rotation proceeds as before (benign restart). + it("does not let an arbitrary pathless fresh affinity take over", async () => { + // The same credential and header name are not continuity proof. With no + // project-scoped content overlap, a pathless fresh affinity stays isolated. harness = await createHarness({ fixtures: [ ...makeConversationFixtures([ @@ -350,8 +386,124 @@ describe("Tier 1b rotation: x-session-affinity (OpenCode nanoid) still rotates s const sessions = harness.queryDB( "SELECT session_id, project_path FROM session_state WHERE header_name = 'x-session-affinity'", ); - // ONE session — rotation proceeded (Fix 2 was a no-op, no incoming project). - expect(sessions.length).toBe(1); + expect(sessions.length).toBe(2); + expect(new Set(sessions.map((session) => session.session_id)).size).toBe(2); + }); + + it("does not bypass project-scoped overlap through a conflicting remote", async () => { + enableHostedMode(); + const projectPath = "/client/projects/rotation-backfill"; + const conflictingPath = "/client/projects/rotation-backfill-clone"; + const remote = `github.com/test/rotation-${crypto.randomUUID()}`; + harness = await createHarness({ + fixtures: makeConversationFixtures([ + { userMessage: "rotation merge first", assistantText: "First." }, + { userMessage: "rotation merge second", assistantText: "Second." }, + { userMessage: "rotation merge third", assistantText: "Third." }, + ]), + }); + + let response = await harness.chat(body("rotation merge first"), "key-A", { + "x-session-affinity": "rotation-merge-before", + "x-lore-project": projectPath, + }); + expect(response.status).toBe(200); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + response = await harness.chat( + { + ...body("rotation merge second"), + messages: [ + { role: "user", content: "rotation merge first" }, + { role: "assistant", content: "First." }, + { role: "user", content: "rotation merge second" }, + ], + }, + "key-A", + { + "x-session-affinity": "rotation-merge-before", + "x-lore-project": projectPath, + }, + ); + expect(response.status).toBe(200); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + // Adversarial order: the path-only project already exists, then a clone + // carrying its remote arrives before the provisional rotation commits. + // Backfilling projectPath now has to merge this conflicting remote row. + const conflictingId = crypto.randomUUID(); + const database = new DatabaseSync(harness.dbPath); + try { + database + .prepare( + "INSERT INTO projects (id, path, name, git_remote, created_at) VALUES (?, ?, ?, ?, ?)", + ) + .run( + conflictingId, + conflictingPath, + "rotation-backfill-clone", + remote, + Date.now(), + ); + } finally { + database.close(); + } + + response = await harness.chat( + { + ...body("rotation merge third"), + messages: [ + { role: "user", content: "rotation merge first" }, + { role: "assistant", content: "First." }, + { role: "user", content: "rotation merge second" }, + { role: "assistant", content: "Second." }, + { role: "user", content: "rotation merge third" }, + ], + }, + "key-A", + { + "x-session-affinity": "rotation-merge-after", + "x-lore-project": projectPath, + "x-lore-git-remote": remote, + }, + ); + expect(response.status).toBe(200); + await response.text(); + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + const sessions = harness.queryDB( + `SELECT session_id, header_session_id, header_name, project_path, + project_path_provisional, credential_fingerprint, message_count + FROM session_state + WHERE header_name = 'x-session-affinity'`, + ); + expect(sessions).toHaveLength(2); + const rotated = sessions.find( + (session) => session.header_session_id === "rotation-merge-after", + ); + expect(rotated).toMatchObject({ + header_session_id: "rotation-merge-after", + project_path: projectPath, + project_path_provisional: 0, + message_count: 5, + }); + expect( + harness.queryDB<{ count: number }>( + "SELECT COUNT(*) AS count FROM temporal_messages WHERE session_id = ? AND content LIKE ?", + [rotated?.session_id, "%rotation merge third%"], + )[0]?.count, + ).toBeGreaterThan(0); + expect( + harness.queryDB<{ count: number }>( + "SELECT COUNT(*) AS count FROM projects WHERE id = ?", + [conflictingId], + )[0]?.count, + ).toBe(0); }); }); diff --git a/packages/gateway/test/setup-transaction.test.ts b/packages/gateway/test/setup-transaction.test.ts index 8cd81062..5ed8e8cf 100644 --- a/packages/gateway/test/setup-transaction.test.ts +++ b/packages/gateway/test/setup-transaction.test.ts @@ -305,6 +305,7 @@ function runSigkillChild(operation: "setup" | "undo", commit: number): void { const child = spawnSync( process.execPath, [ + "--conditions=development", "--import", "tsx", join(import.meta.dirname, "setup-sigkill-child.ts"), @@ -338,6 +339,7 @@ function runExternalEffectSigkillChild( const child = spawnSync( process.execPath, [ + "--conditions=development", "--import", "tsx", join(import.meta.dirname, "setup-external-sigkill-child.ts"), diff --git a/packages/gateway/test/single-flight-stable-ltm.test.ts b/packages/gateway/test/single-flight-stable-ltm.test.ts index 5fce48da..fc76be79 100644 --- a/packages/gateway/test/single-flight-stable-ltm.test.ts +++ b/packages/gateway/test/single-flight-stable-ltm.test.ts @@ -117,4 +117,73 @@ describe("singleFlightStableLtm", () => { expect(r).toEqual({ formatted: "final", tokenCount: 99 }); } }); + + test("session eviction aborts the owner before a late compute can publish", async () => { + vi.resetModules(); + const { evictStableLtmSessionForTest, singleFlightStableLtm } = + await import("../src/pipeline"); + + let release!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + let ownerSignal: AbortSignal | undefined; + const compute = vi.fn(async (signal: AbortSignal) => { + ownerSignal = signal; + await blocked; + return { formatted: "stale", tokenCount: 5 }; + }); + + const pending = singleFlightStableLtm("session-evicted", compute); + await vi.waitFor(() => expect(ownerSignal).toBeDefined()); + evictStableLtmSessionForTest("session-evicted"); + + await expect(pending).rejects.toMatchObject({ name: "AbortError" }); + expect(ownerSignal?.aborted).toBe(true); + release(); + + const fresh = await singleFlightStableLtm("session-evicted", async () => ({ + formatted: "fresh", + tokenCount: 6, + })); + expect(fresh).toEqual({ formatted: "fresh", tokenCount: 6 }); + expect(compute).toHaveBeenCalledTimes(1); + }); + + test("one caller cancelling does not abort a shared session-owned compute", async () => { + vi.resetModules(); + const { singleFlightStableLtm } = await import("../src/pipeline"); + const owner = new AbortController(); + let release!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + let computeSignal: AbortSignal | undefined; + const compute = vi.fn(async (signal: AbortSignal) => { + computeSignal = signal; + await blocked; + return { formatted: "shared", tokenCount: 7 }; + }); + + const cancelledCaller = singleFlightStableLtm( + "session-shared-owner", + compute, + owner.signal, + ); + await vi.waitFor(() => expect(computeSignal).toBeDefined()); + const healthyCaller = singleFlightStableLtm( + "session-shared-owner", + compute, + ); + owner.abort(new DOMException("caller disconnected", "AbortError")); + + await expect(cancelledCaller).rejects.toMatchObject({ name: "AbortError" }); + expect(computeSignal?.aborted).toBe(false); + release(); + await expect(healthyCaller).resolves.toEqual({ + formatted: "shared", + tokenCount: 7, + }); + expect(compute).toHaveBeenCalledTimes(1); + }); }); diff --git a/packages/gateway/test/store-turn-temporal.test.ts b/packages/gateway/test/store-turn-temporal.test.ts index 6a5711e6..536a1678 100644 --- a/packages/gateway/test/store-turn-temporal.test.ts +++ b/packages/gateway/test/store-turn-temporal.test.ts @@ -287,7 +287,7 @@ describe("storeTurnTemporal (#1084)", () => { // Reproduce the real v81 layouts, then restart through migrate(). This // keeps IDs/FTS rows/tool data intact while v82 backfills source_id=id and - // rebuilds the owned tool-call primary key. + // rebuilds the owned tool-call primary key; later migrations also run. db().exec(` DROP INDEX idx_temporal_source_identity; ALTER TABLE temporal_messages DROP COLUMN source_id; @@ -324,7 +324,7 @@ describe("storeTurnTemporal (#1084)", () => { `); close(); expect(db().query("SELECT version FROM schema_version").get()).toEqual({ - version: 82, + version: 84, }); const noStoreLore = gatewayMessagesToLore(conversation, sessionID); diff --git a/packages/gateway/test/sync-domain-join.integration.test.ts b/packages/gateway/test/sync-domain-join.integration.test.ts index 58156c1c..e78295c1 100644 --- a/packages/gateway/test/sync-domain-join.integration.test.ts +++ b/packages/gateway/test/sync-domain-join.integration.test.ts @@ -9,6 +9,7 @@ */ import { execFileSync } from "node:child_process"; import { createClient, type SupabaseClient } from "@supabase/supabase-js"; +import { loopbackRequest } from "./helpers/loopback-request"; import { afterAll, beforeAll, @@ -71,7 +72,7 @@ function clientFor(uid: string): SupabaseClient { : input instanceof URL ? input.toString() : input.url; - return fetch(url.replace("/rest/v1", ""), init); + return loopbackRequest(url.replace("/rest/v1", ""), init); }; return createClient(h.restUrl as string, jwt, { auth: { persistSession: false, autoRefreshToken: false }, diff --git a/packages/gateway/test/sync-engine.integration.test.ts b/packages/gateway/test/sync-engine.integration.test.ts index ff70742e..f43c5b71 100644 --- a/packages/gateway/test/sync-engine.integration.test.ts +++ b/packages/gateway/test/sync-engine.integration.test.ts @@ -13,6 +13,7 @@ */ import { execFileSync } from "node:child_process"; import { createClient, type SupabaseClient } from "@supabase/supabase-js"; +import { loopbackRequest } from "./helpers/loopback-request"; import { crypto, db, @@ -89,7 +90,7 @@ function clientFor(uid: string): SupabaseClient { : input instanceof URL ? input.toString() : input.url; - return fetch(url.replace("/rest/v1", ""), init); + return loopbackRequest(url.replace("/rest/v1", ""), init); }; return createClient(h.restUrl as string, jwt, { auth: { persistSession: false, autoRefreshToken: false }, diff --git a/packages/gateway/test/sync-team-invite.integration.test.ts b/packages/gateway/test/sync-team-invite.integration.test.ts index f19fdccd..11901685 100644 --- a/packages/gateway/test/sync-team-invite.integration.test.ts +++ b/packages/gateway/test/sync-team-invite.integration.test.ts @@ -9,6 +9,7 @@ */ import { execFileSync } from "node:child_process"; import { createClient, type SupabaseClient } from "@supabase/supabase-js"; +import { loopbackRequest } from "./helpers/loopback-request"; import { db, keystore, setKV, syncData } from "@loreai/core"; import { afterAll, @@ -67,7 +68,7 @@ function clientFor(uid: string): SupabaseClient { : input instanceof URL ? input.toString() : input.url; - return fetch(url.replace("/rest/v1", ""), init); + return loopbackRequest(url.replace("/rest/v1", ""), init); }; return createClient(h.restUrl as string, jwt, { auth: { persistSession: false, autoRefreshToken: false }, diff --git a/packages/gateway/test/team-cmd.test.ts b/packages/gateway/test/team-cmd.test.ts index 6cfdc728..5578daa0 100644 --- a/packages/gateway/test/team-cmd.test.ts +++ b/packages/gateway/test/team-cmd.test.ts @@ -917,12 +917,14 @@ describe("discover --invite (E-5-d-2)", () => { await commandTeam(["discover"], {}); // The 3rd arg is `repos` — undefined when no explicit positional AND no git remote in cwd; // some test CWDs DO have a remote, in which case it's a one-element array. Accept either. - expect(vi.mocked(team.discoverGitHubContributors)).toHaveBeenCalledWith( - FAKE_CLIENT, - "gho_x", - expect.anything(), - undefined, - ); + const call = vi.mocked(team.discoverGitHubContributors).mock.lastCall; + expect(call?.[0]).toBe(FAKE_CLIENT); + expect(call?.[1]).toBe("gho_x"); + expect( + call?.[2] === undefined || + (Array.isArray(call[2]) && call[2].length === 1), + ).toBe(true); + expect(call?.[3]).toBeUndefined(); expect(logs.join("\n")).toMatch(/already on Lore/); }); diff --git a/packages/gateway/test/team.integration.test.ts b/packages/gateway/test/team.integration.test.ts index d50ccb38..767ac4bd 100644 --- a/packages/gateway/test/team.integration.test.ts +++ b/packages/gateway/test/team.integration.test.ts @@ -9,6 +9,7 @@ */ import { execFileSync } from "node:child_process"; import { createClient, type SupabaseClient } from "@supabase/supabase-js"; +import { loopbackRequest } from "./helpers/loopback-request"; import { db, keystore, setKV, syncData } from "@loreai/core"; import { afterAll, @@ -67,7 +68,7 @@ function clientFor(uid: string): SupabaseClient { : input instanceof URL ? input.toString() : input.url; - return fetch(url.replace("/rest/v1", ""), init); + return loopbackRequest(url.replace("/rest/v1", ""), init); }; return createClient(h.restUrl as string, jwt, { auth: { persistSession: false, autoRefreshToken: false }, diff --git a/packages/gateway/test/upstream-extra-header-routing.test.ts b/packages/gateway/test/upstream-extra-header-routing.test.ts index ee377969..b0836a5b 100644 --- a/packages/gateway/test/upstream-extra-header-routing.test.ts +++ b/packages/gateway/test/upstream-extra-header-routing.test.ts @@ -238,6 +238,7 @@ describe("foreground upstream extra-header base-path binding", () => { const first = await handleRequest(request({}), config); expect(first.status).toBe(200); + await first.text(); const state = [...getActiveSessions().values()][0]; expect(state.cacheAnalytics.lastRequestBody).not.toBeNull(); @@ -247,6 +248,7 @@ describe("foreground upstream extra-header base-path binding", () => { config, ); expect(failed.status).toBe(502); + await failed.text(); expect(state.lastUpstream?.url).toBe("https://attacker.example"); expect(state.cacheAnalytics.lastRequestBody).toBeNull(); }); diff --git a/packages/gateway/test/vertex-routing.test.ts b/packages/gateway/test/vertex-routing.test.ts index 7fe11e79..9effebb7 100644 --- a/packages/gateway/test/vertex-routing.test.ts +++ b/packages/gateway/test/vertex-routing.test.ts @@ -16,6 +16,7 @@ */ import { describe, test, expect, afterEach } from "vitest"; import { existsSync, unlinkSync } from "node:fs"; +import { loopbackRequest } from "./helpers/loopback-request"; /** A non-streaming Anthropic message response (Vertex returns native shape). */ function vertexJSONResponse(): Response { @@ -96,7 +97,7 @@ describe("X-Lore-Provider: vertex routing (Vertex AI Claude)", () => { } }; - const resp = await fetch(`${baseURL}/v1/messages`, { + const resp = await loopbackRequest(`${baseURL}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", diff --git a/packages/gateway/test/worker-model.test.ts b/packages/gateway/test/worker-model.test.ts index 9e6d6430..62909975 100644 --- a/packages/gateway/test/worker-model.test.ts +++ b/packages/gateway/test/worker-model.test.ts @@ -196,6 +196,91 @@ describe("fetchModelData", () => { expect(first).toBe(second); // Same reference — cached }); + test("a stale fetch cannot overwrite a cache seeded after it started", async () => { + let resolveFetch!: (response: Response) => void; + globalThis.fetch = vi.fn( + () => + new Promise((resolve) => { + resolveFetch = resolve; + }), + ) as unknown as typeof fetch; + + const stale = fetchModelData(); + await vi.waitFor(() => expect(resolveFetch).toBeDefined()); + _setModelDataForTest({ + seeded: { + id: "seeded", + cost: { input: 9, output: 9, cache_read: 9 }, + limit: { context: 9, output: 9 }, + }, + }); + resolveFetch( + new Response( + JSON.stringify( + buildModelsDevResponse({ + stale: { + cost: { input: 1, output: 1, cache_read: 1 }, + limit: { context: 1, output: 1 }, + }, + }), + ), + { status: 200 }, + ), + ); + + await stale; + expect(getModelEntrySync("seeded").cost?.input).toBe(9); + expect(getModelEntrySync("stale").cost?.input).not.toBe(1); + }); + + test("an older fetch cannot publish over or clear a newer in-flight owner", async () => { + const resolves: Array<(response: Response) => void> = []; + globalThis.fetch = vi.fn( + () => + new Promise((resolve) => { + resolves.push(resolve); + }), + ) as unknown as typeof fetch; + + const older = fetchModelData(); + await vi.waitFor(() => expect(resolves).toHaveLength(1)); + clearModelDataCache(); + const newer = fetchModelData(); + await vi.waitFor(() => expect(resolves).toHaveLength(2)); + resolves[0]( + new Response( + JSON.stringify( + buildModelsDevResponse({ + older: { + cost: { input: 1, output: 1, cache_read: 1 }, + limit: { context: 1, output: 1 }, + }, + }), + ), + { status: 200 }, + ), + ); + await older; + + expect(fetchModelData()).toBe(newer); + resolves[1]( + new Response( + JSON.stringify( + buildModelsDevResponse({ + newer: { + cost: { input: 2, output: 2, cache_read: 2 }, + limit: { context: 2, output: 2 }, + }, + }), + ), + { status: 200 }, + ), + ); + await newer; + expect(getModelEntrySync("newer").cost?.input).toBe(2); + expect(getModelEntrySync("older").cost?.input).not.toBe(1); + }); + test("returns empty map on API error with no cache", async () => { globalThis.fetch = vi.fn(() => Promise.resolve(new Response("Server Error", { status: 500 })),