diff --git a/apps/website/content/docs/headless/getting-started.mdx b/apps/website/content/docs/headless/getting-started.mdx index f57c333b4..1a0562206 100644 --- a/apps/website/content/docs/headless/getting-started.mdx +++ b/apps/website/content/docs/headless/getting-started.mdx @@ -6,7 +6,7 @@ nav: Headless engine A headless renderer starts with `createLocalRowModel`. Add `createGrid` only when your renderer needs UI state. The grid below is exactly that: 75 services rendered from a plain ``, with `createLocalRowModel` driving sort and filter and `createGrid` driving row selection. -`setQuery` — triggered here by sorting a column or typing into the filter — does not settle synchronously. The model rebuilds cooperatively, yielding between slices so a large query cannot block the frame, and publishes progress and failures through `status`. Two things follow, and the example does both: **select** what you subscribe to, or you re-render on every slice, and **read `status`**, or a rebuild that fails leaves stale rows on screen with nothing to say so. +`setQuery` — triggered here by typing into the filter — does not settle synchronously. The model rebuilds cooperatively, yielding between slices so a large query cannot block the frame, and publishes progress and failures through `status`. (Sorting a column is the exception: a sort-only change on ungrouped data re-orders rows the model has already indexed, so it settles synchronously.) Two things follow, and the example does both: **select** what you subscribe to, or you re-render on every slice, and **read `status`**, or a rebuild that fails leaves stale rows on screen with nothing to say so. diff --git a/apps/website/content/docs/headless/state-model.mdx b/apps/website/content/docs/headless/state-model.mdx index 43104922f..29d39fba3 100644 --- a/apps/website/content/docs/headless/state-model.mdx +++ b/apps/website/content/docs/headless/state-model.mdx @@ -6,7 +6,7 @@ nav: Headless engine The row model and UI grid are independent observable stores. Subscribe only to the state your renderer uses. -`setQuery` does not settle synchronously — the model rebuilds cooperatively, yielding between slices so a large query cannot block the frame. On a small dataset that's over before a human (or React) can see it happen, which is why the button below sorts 150,000 rows instead of 75: watch `status` cycle through `rebuilding` with a live percentage, then settle back to `ready`. +A `setQuery` that changes the filter does not settle synchronously — the model rebuilds cooperatively, yielding between slices so a large query cannot block the frame. A sort-only change on ungrouped data is the one exception: it re-orders rows the model has already indexed, so it settles synchronously and never publishes a `rebuilding` phase — a plain sort needs no progress UI. On a small dataset even the cooperative rebuild is over before a human (or React) can see it happen, which is why the button below filters 150,000 rows instead of 75: watch `status` cycle through `rebuilding` with a live percentage, then settle back to `ready`. @@ -43,7 +43,8 @@ committed, and mutations keep committing into it meanwhile: `setRows`, `applyTransaction` and both expansion paths publish a new snapshot while a rebuild runs. A renderer that stops re-reading the snapshot during a rebuild drops those, which is exactly the streaming-plus-filter case. The table in the -example above stays on the last sort throughout, exactly like this. +example above stays on the last committed filter result throughout, exactly +like this. `completedRows` and `totalRows` count the rebuild's **work units, not rows**. Grouping and concurrent mutations add units as the transition runs, so diff --git a/apps/website/content/examples/headless-rebuild-progress/RebuildProgress.tsx b/apps/website/content/examples/headless-rebuild-progress/RebuildProgress.tsx index a19103455..86dc6836a 100644 --- a/apps/website/content/examples/headless-rebuild-progress/RebuildProgress.tsx +++ b/apps/website/content/examples/headless-rebuild-progress/RebuildProgress.tsx @@ -41,7 +41,7 @@ export function RebuildProgress< const label = progressText.startsWith("rebuilding:") ? `Rebuilding… ${progressText.slice("rebuilding:".length)}%` : progressText === "ready" - ? "Sorted." + ? "Ready." : progressText; return ( diff --git a/apps/website/content/examples/headless-rebuild-progress/RebuildProgressDemo.tsx b/apps/website/content/examples/headless-rebuild-progress/RebuildProgressDemo.tsx index 6c03eea1a..78585d7d1 100644 --- a/apps/website/content/examples/headless-rebuild-progress/RebuildProgressDemo.tsx +++ b/apps/website/content/examples/headless-rebuild-progress/RebuildProgressDemo.tsx @@ -19,7 +19,7 @@ export function RebuildProgressDemo() { // Selecting `snapshot` (not the whole state) means this component bails // out on identity between rebuild slices — it only renders once, when the - // sort actually lands. `RebuildProgress` above is the one re-rendering on + // filter actually lands. `RebuildProgress` above is the one re-rendering on // every slice in the meantime. const readSnapshot = useCallback( () => rowModel.getState().snapshot, @@ -31,22 +31,28 @@ export function RebuildProgressDemo() { readSnapshot, ); - const [descending, setDescending] = useState(true); + const [filtered, setFiltered] = useState(false); - const resort = () => { - const next = !descending; - setDescending(next); + // A FILTER change, not a sort: a sort-only change on ungrouped data + // settles synchronously and never publishes a `rebuilding` phase, so it + // could not demonstrate the progress readout at all. + const toggleFilter = () => { + const next = !filtered; + setFiltered(next); rowModel.setQuery({ ...snapshot.query, - sort: [{ columnId: "amount", direction: next ? "desc" : "asc" }], + filters: next + ? [{ columnId: "region", operator: "equals", value: "west" }] + : [], }); }; return (
-

diff --git a/apps/website/content/examples/headless-rebuild-progress/__tests__/RebuildProgressDemo.test.tsx b/apps/website/content/examples/headless-rebuild-progress/__tests__/RebuildProgressDemo.test.tsx index 0b921868c..ad46a4707 100644 --- a/apps/website/content/examples/headless-rebuild-progress/__tests__/RebuildProgressDemo.test.tsx +++ b/apps/website/content/examples/headless-rebuild-progress/__tests__/RebuildProgressDemo.test.tsx @@ -43,12 +43,15 @@ describe("RebuildProgressDemo", () => { }); fireEvent.click( - screen.getByRole("button", { name: /sort 150,000 orders/i }), + screen.getByRole("button", { name: /filter 150,000 orders/i }), ); await waitFor( () => { - expect(status).toHaveTextContent("Sorted."); + expect(status).toHaveTextContent("Ready."); + // The filter landed: only the 30,000 west-region orders survive, + // and every preview row is one of them. + expect(screen.getByText(/30,000 rows indexed/)).toBeInTheDocument(); }, { timeout: REBUILD_TIMEOUT }, ); @@ -57,7 +60,64 @@ describe("RebuildProgressDemo", () => { // Proves the rebuild actually published at least one intermediate // `rebuilding` slice before landing on `ready` — the whole reason this // example exists. On the small 75-row custom-renderer example this - // would be a coin flip; at 150,000 rows it is not. + // would be a coin flip; at 150,000 rows it is not. A sort-only change + // could never pass this: on ungrouped data it settles synchronously + // with no `rebuilding` phase at all. + expect(sawRebuilding).toBe(true); + + const previewRows = screen.getAllByRole("row").slice(1); + expect(previewRows.length).toBeGreaterThan(0); + for (const row of previewRows) { + expect(row).toHaveTextContent("west"); + } + }, + REBUILD_TIMEOUT + 5_000, + ); + + it( + "clears the filter cooperatively on the second click", + async () => { + render(); + await waitFor(() => screen.getByText(/150,000 rows indexed/), { + timeout: REBUILD_TIMEOUT, + }); + + fireEvent.click( + screen.getByRole("button", { name: /filter 150,000 orders/i }), + ); + await waitFor(() => screen.getByText(/30,000 rows indexed/), { + timeout: REBUILD_TIMEOUT, + }); + + let sawRebuilding = false; + const status = screen.getByRole("status"); + const observer = new MutationObserver(() => { + if (/Rebuilding…/.test(status.textContent ?? "")) { + sawRebuilding = true; + } + }); + observer.observe(status, { + childList: true, + characterData: true, + subtree: true, + }); + + fireEvent.click( + screen.getByRole("button", { name: /show all 150,000 orders/i }), + ); + + await waitFor( + () => { + expect(status).toHaveTextContent("Ready."); + expect(screen.getByText(/150,000 rows indexed/)).toBeInTheDocument(); + }, + { timeout: REBUILD_TIMEOUT }, + ); + + observer.disconnect(); + // Removing a filter re-runs the same cooperative path over all + // 150,000 source rows, so the toggle demonstrates progress in both + // directions. expect(sawRebuilding).toBe(true); }, REBUILD_TIMEOUT + 5_000, diff --git a/apps/website/content/examples/headless-rebuild-progress/data.ts b/apps/website/content/examples/headless-rebuild-progress/data.ts index 0165bb781..a173c2562 100644 --- a/apps/website/content/examples/headless-rebuild-progress/data.ts +++ b/apps/website/content/examples/headless-rebuild-progress/data.ts @@ -7,10 +7,10 @@ export interface Order { const REGIONS = ["north", "south", "east", "west", "central"]; -// Deliberately large and deterministic (no Math.random): big enough that -// `setQuery` cannot settle inside one animation frame, so the rebuild really -// does publish multiple `rebuilding` slices instead of jumping straight to -// `ready` — see the note on the smaller custom-renderer example. +// Deliberately large and deterministic (no Math.random): big enough that a +// filter change cannot settle inside one animation frame, so the rebuild +// really does publish multiple `rebuilding` slices instead of jumping +// straight to `ready` — see the note on the smaller custom-renderer example. export const ORDER_COUNT = 150_000; export const orders: Order[] = Array.from({ length: ORDER_COUNT }, (_, i) => ({ diff --git a/apps/website/content/examples/headless-rebuild-progress/example.ts b/apps/website/content/examples/headless-rebuild-progress/example.ts index 33c49fc18..d4558f711 100644 --- a/apps/website/content/examples/headless-rebuild-progress/example.ts +++ b/apps/website/content/examples/headless-rebuild-progress/example.ts @@ -3,7 +3,7 @@ import { defineExample } from "../../../lib/docs/examples/define"; export default defineExample({ title: "Watching a rebuild", description: - "Re-sorting 150,000 rows cannot settle inside one animation frame, so status.kind cycles through rebuilding with a live completedRows/totalRows progress readout before returning to ready.", + "Filtering 150,000 rows cannot settle inside one animation frame, so status.kind cycles through rebuilding with a live completedRows/totalRows progress readout before returning to ready.", files: [ "RebuildProgressDemo.tsx", "RebuildProgress.tsx", diff --git a/docs/superpowers/plans/2026-08-17-sort-fast-path.md b/docs/superpowers/plans/2026-08-17-sort-fast-path.md new file mode 100644 index 000000000..31fde14b6 --- /dev/null +++ b/docs/superpowers/plans/2026-08-17-sort-fast-path.md @@ -0,0 +1,1371 @@ +# Sort Fast Path Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make a sort-only `setQuery` on an ungrouped model complete synchronously by carrying prior evaluation results forward and bulk-building the indexes, closing #457's 515ms-at-50k gap to TanStack parity. + +**Architecture:** A query-delta classifier in `compiled-query.ts` (static methods on `CompiledQueryPlan`, so private facets are comparable) decides when a `setQuery` changed nothing but the applied sort. When it fires and the query is ungrouped, `create-local-row-model.ts` skips the cooperative transition entirely: a new module `sort-rebuild.ts` rebuilds each record's metadata around carried values (`resortRecordMetadata`), `Array.sort`s the filter-passing records, and bulk-builds the visible tree in O(n) via a new sorted-input constructor in `order-statistic-tree.ts`. Everything else keeps the cooperative path. + +**Tech Stack:** TypeScript, vitest (`pnpm --filter @pretable-internal/row-model test`), pnpm workspace, Playwright bench (`apps/bench`). + +**Spec:** `docs/superpowers/specs/2026-08-17-sort-fast-path-design.md` — read it first. + +**Conventions that bind every task:** +- `packages/*` code is vanilla — no new dependencies. +- TDD: write the failing test, watch it fail, implement, watch it pass, commit. +- Comments state constraints the code can't show — never narrate the change. +- Run tests from the package dir or with `--filter @pretable-internal/row-model`; the suite currently passes 327+ tests. + +--- + +### Task 1: Query-delta classifier + +**Files:** +- Modify: `packages/row-model/src/compiled-query.ts` +- Test: `packages/row-model/src/__tests__/query-delta.test.ts` (create) + +The classifier compares two compiled plans facet-by-facet. It lives on `CompiledQueryPlan` as static methods because facet state (`#runtimeQuery`, `#runtimeColumns`, `#filterAuthority`, `#sortAuthority`) is private, and statics can read privates of instances. Exported free functions wrap the statics. Nothing is added to the package's public index — `create-local-row-model.ts` imports from `"./compiled-query"` directly. + +**Conservatism invariant (the thing the tests pin):** any input the classifier cannot positively identify — either plan not created by `compileQuery`, any facet not provably equal — classifies as *changed*. A wrong `false` from `isSortOnlyChange` costs a slow transition; a wrong `true` corrupts results. + +- [ ] **Step 1: Write the failing tests** + +Create `packages/row-model/src/__tests__/query-delta.test.ts`: + +```ts +import { describe, expect, test } from "vitest"; + +import { createColumnHelper } from "../index"; +import { + compileQuery, + isSortOnlyChange, + type CompiledQuery, +} from "../compiled-query"; + +interface Row { + id: number; + team: string; + score: number; + note: string; +} + +const helper = createColumnHelper(); +const columns = [ + helper.accessor("team", { type: "text" }), + helper.accessor("score", { type: "number", aggregate: "sum" }), + helper.accessor("note", { type: "text" }), +] as const; + +type Cols = typeof columns; + +const plan = ( + query: Parameters>[0]["query"], + overrides?: Partial>[0]>, +): CompiledQuery => + compileQuery({ + derivations: columns, + query, + operation: "set-query", + ...overrides, + }); + +const baseQuery = { + filters: [{ columnId: "team", operator: "equals", value: "a" }], + sort: [{ columnId: "score", direction: "desc" }], + rowGroups: [], +} as const; + +describe("isSortOnlyChange", () => { + test("true when only the sort differs", () => { + const previous = plan(baseQuery); + const next = plan({ ...baseQuery, sort: [{ columnId: "team" }] }); + expect(isSortOnlyChange(previous, next)).toBe(true); + }); + + test("true for direction flip, added column, removal to unsorted", () => { + const previous = plan(baseQuery); + for (const sort of [ + [{ columnId: "score", direction: "asc" }], + [ + { columnId: "score", direction: "desc" }, + { columnId: "note", direction: "asc" }, + ], + [], + ] as const) { + expect(isSortOnlyChange(previous, plan({ ...baseQuery, sort }))).toBe( + true, + ); + } + }); + + test("false when the sort is identical (nothing changed)", () => { + const previous = plan(baseQuery); + // compileQuery dedupes identical plans; build without `previous` so we + // get a distinct object with equal facets. + const next = plan(baseQuery); + expect(isSortOnlyChange(previous, next)).toBe(false); + }); + + test("false when filters also changed", () => { + const previous = plan(baseQuery); + const next = plan({ + ...baseQuery, + filters: [{ columnId: "team", operator: "equals", value: "b" }], + sort: [{ columnId: "team" }], + }); + expect(isSortOnlyChange(previous, next)).toBe(false); + }); + + test("false when rowGroups also changed", () => { + const previous = plan(baseQuery); + const next = plan({ + ...baseQuery, + rowGroups: [{ columnId: "team" }], + sort: [{ columnId: "team" }], + }); + expect(isSortOnlyChange(previous, next)).toBe(false); + }); + + test("false when derivations changed for an active column", () => { + const replaced = [ + helper.accessor("team", { type: "text" }), + { ...columns[1], accessor: (row: Row) => row.score * 2 }, + helper.accessor("note", { type: "text" }), + ] as const; + const previous = plan(baseQuery); + const next = compileQuery({ + derivations: replaced as never, + query: { ...baseQuery, sort: [{ columnId: "team" }] } as never, + operation: "set-query", + }); + expect(isSortOnlyChange(previous, next as never)).toBe(false); + }); + + test("false when filter authority differs", () => { + const previous = plan(baseQuery); + const next = plan( + { ...baseQuery, sort: [{ columnId: "team" }] }, + { filterAuthority: "external" }, + ); + expect(isSortOnlyChange(previous, next)).toBe(false); + }); + + test("false when sort authority differs", () => { + const previous = plan(baseQuery); + const next = plan( + { ...baseQuery, sort: [{ columnId: "team" }] }, + { sortAuthority: "external" }, + ); + expect(isSortOnlyChange(previous, next)).toBe(false); + }); + + test("false under external sort authority in both plans (runtime sort is empty twice)", () => { + const previous = plan(baseQuery, { sortAuthority: "external" }); + const next = plan( + { ...baseQuery, sort: [{ columnId: "team" }] }, + { sortAuthority: "external" }, + ); + expect(isSortOnlyChange(previous, next)).toBe(false); + }); + + test("false for foreign plan objects", () => { + const previous = plan(baseQuery); + const fake = { + query: previous.query, + derivations: previous.derivations, + } as unknown as CompiledQuery; + expect(isSortOnlyChange(fake, previous)).toBe(false); + expect(isSortOnlyChange(previous, fake)).toBe(false); + }); +}); +``` + +Note: check `createColumnHelper` / filter operator names against an existing test (e.g. `transitions.test.ts`) and adjust the fixture syntax to match real usage before running — the shapes above are illustrative of intent, the assertions are the contract. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pnpm --filter @pretable-internal/row-model test -- query-delta` +Expected: FAIL — `isSortOnlyChange` is not exported. + +- [ ] **Step 3: Implement the classifier** + +In `packages/row-model/src/compiled-query.ts`: + +1. Hoist `orderingEqual` out of `queryEqual` (line ~844) to module scope unchanged; have `queryEqual` call it. +2. Add to `CompiledQueryPlan`: + +```ts +/** + * Facet delta between two plans this module compiled. `undefined` when either + * plan is foreign — the caller must treat that as "everything changed". + * Compares RUNTIME facets: under external sort authority the runtime sort is + * `[]`, so a public sort change classifies as no applied change. + */ +static classifyDelta( + previous: unknown, + next: unknown, +): + | { + readonly derivationsChanged: boolean; + readonly filtersChanged: boolean; + readonly groupsChanged: boolean; + readonly sortChanged: boolean; + readonly authorityChanged: boolean; + } + | undefined { + if ( + !(previous instanceof CompiledQueryPlan) || + !(next instanceof CompiledQueryPlan) + ) { + return undefined; + } + return Object.freeze({ + derivationsChanged: !( + derivationsEqualForPlan( + previous.#runtimeColumns, + next.#runtimeColumns, + previous.#runtimeQuery, + ) && + derivationsEqualForPlan( + previous.#runtimeColumns, + next.#runtimeColumns, + next.#runtimeQuery, + ) + ), + filtersChanged: !filtersEqual( + previous.#runtimeQuery.filters, + next.#runtimeQuery.filters, + ), + groupsChanged: !orderingEqual( + previous.#runtimeQuery.rowGroups, + next.#runtimeQuery.rowGroups, + ), + sortChanged: !orderingEqual( + previous.#runtimeQuery.sort, + next.#runtimeQuery.sort, + ), + authorityChanged: + previous.#filterAuthority !== next.#filterAuthority || + previous.#sortAuthority !== next.#sortAuthority, + }); +} +``` + +3. Export free functions after the class: + +```ts +export type CompiledQueryDelta = NonNullable< + ReturnType +>; + +export function classifyQueryDelta( + previous: CompiledQuery, + next: CompiledQuery, +): CompiledQueryDelta | undefined { + return CompiledQueryPlan.classifyDelta(previous, next); +} + +/** True only when the applied sort is the sole difference between the plans. */ +export function isSortOnlyChange( + previous: CompiledQuery, + next: CompiledQuery, +): boolean { + const delta = CompiledQueryPlan.classifyDelta(previous, next); + return ( + delta !== undefined && + delta.sortChanged && + !delta.derivationsChanged && + !delta.filtersChanged && + !delta.groupsChanged && + !delta.authorityChanged + ); +} +``` + +Do NOT add these to `packages/row-model/src/index.ts`. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pnpm --filter @pretable-internal/row-model test -- query-delta` +Expected: PASS, all cases. + +- [ ] **Step 5: Run the full package suite (no regressions)** + +Run: `pnpm --filter @pretable-internal/row-model test` +Expected: PASS (327+ tests). + +- [ ] **Step 6: Commit** + +```bash +git add packages/row-model/src/compiled-query.ts packages/row-model/src/__tests__/query-delta.test.ts +git commit -m "feat(row-model): classify query deltas between compiled plans" +``` + +--- + +### Task 2: O(n) sorted-input tree constructor + +**Files:** +- Modify: `packages/row-model/src/persistent/order-statistic-tree.ts` +- Test: `packages/row-model/src/__tests__/order-statistic-tree.test.ts` (add to the existing tree test file; if tree tests live elsewhere, follow that location) + +A balanced tree built bottom-up from an already-sorted array, mirroring the existing internal `createDeferredMeasureTransientOrderStatisticTree` pattern (`order-statistic-tree.ts:918`): exported from the module, deliberately omitted from the package index. + +**Correctness constraint:** the tree's total order is `context.compare` then `compareIds` on tie (`compareEntries`, line ~186). Input must be strictly increasing under that composite order — verified in an O(n) pass that throws on violation (this also rejects duplicate ids). A tree built from misordered input would corrupt every later `rankOf`/`insertOrReplace`, so the check is unconditional, not dev-only. + +- [ ] **Step 1: Write the failing tests** + +Add to the tree test file: + +```ts +import { + compareOrderStatisticTreeIds, + createOrderStatisticTree, + createOrderStatisticTreeFromSortedEntries, +} from "../persistent/order-statistic-tree"; + +interface Entry { + readonly id: number; + readonly rank: number; +} + +const context = { + getId: (entry: Entry) => entry.id, + compare: (left: Entry, right: Entry) => left.rank - right.rank, + measure: { + empty: 0, + fromEntry: () => 1, + combine: (left: number, right: number) => left + right, + }, +}; + +describe("createOrderStatisticTreeFromSortedEntries", () => { + const entries = Array.from({ length: 1000 }, (_, index) => ({ + id: index, + rank: index * 2, + })); + + test("matches incremental construction observably", () => { + const like = createOrderStatisticTree(context); + const bulk = createOrderStatisticTreeFromSortedEntries(like, entries); + let incremental = like; + for (const entry of entries) incremental = incremental.insertOrReplace(entry); + expect(bulk.size).toBe(incremental.size); + for (let index = 0; index < entries.length; index += 25) { + expect(bulk.entryAt(index)).toEqual(incremental.entryAt(index)); + expect(bulk.rankOf(entries[index]!.id)).toBe( + incremental.rankOf(entries[index]!.id), + ); + } + }); + + test("produces a balanced tree", () => { + const like = createOrderStatisticTree(context); + const bulk = createOrderStatisticTreeFromSortedEntries(like, entries); + // Use the module's diagnostics accessor for balance; adjust the call to + // however OrderStatisticTreeDiagnostics is obtained in existing tests. + expect(diagnosticsOf(bulk).balanced).toBe(true); + expect(diagnosticsOf(bulk).count).toBe(entries.length); + }); + + test("supports later incremental mutation", () => { + const like = createOrderStatisticTree(context); + let tree = createOrderStatisticTreeFromSortedEntries(like, entries); + tree = tree.insertOrReplace({ id: 5000, rank: 3 }); + expect(tree.size).toBe(entries.length + 1); + expect(tree.rankOf(5000)).toBe(2); + tree = tree.remove(0); + expect(tree.rankOf(5000)).toBe(1); + }); + + test("throws on misordered input", () => { + const like = createOrderStatisticTree(context); + expect(() => + createOrderStatisticTreeFromSortedEntries(like, [ + { id: 1, rank: 10 }, + { id: 2, rank: 5 }, + ]), + ).toThrow(TypeError); + }); + + test("throws on equal-rank entries whose ids are misordered", () => { + const like = createOrderStatisticTree(context); + const misordered = [ + { id: 2, rank: 7 }, + { id: 1, rank: 7 }, + ]; + expect( + compareOrderStatisticTreeIds(2, 1), + ).toBeGreaterThan(0); // control: proves the fixture is actually misordered + expect(() => + createOrderStatisticTreeFromSortedEntries(like, misordered), + ).toThrow(TypeError); + }); + + test("throws on duplicate ids", () => { + const like = createOrderStatisticTree(context); + expect(() => + createOrderStatisticTreeFromSortedEntries(like, [ + { id: 1, rank: 1 }, + { id: 1, rank: 2 }, + ]), + ).toThrow(TypeError); + }); + + test("empty input yields an empty tree", () => { + const like = createOrderStatisticTree(context); + const bulk = createOrderStatisticTreeFromSortedEntries(like, []); + expect(bulk.size).toBe(0); + }); + + test("throws for a foreign tree object", () => { + expect(() => + createOrderStatisticTreeFromSortedEntries( + { size: 0 } as never, + [], + ), + ).toThrow(TypeError); + }); +}); +``` + +Duplicate-id check note: duplicates compare equal on rank and equal on id, so `compareEntries` returns 0, which the strict `< 0` requirement rejects — the duplicate test passes through the same throw path by design. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pnpm --filter @pretable-internal/row-model test -- order-statistic` +Expected: FAIL — `createOrderStatisticTreeFromSortedEntries` is not exported. + +- [ ] **Step 3: Implement** + +In `order-statistic-tree.ts`, inside the module (it needs `TreeNode`, `createNode`, `compareEntries`, `nodeHeight`, and the `PersistentOrderStatisticTree` constructor): + +```ts +/** Internal bulk-build primitive; deliberately omitted from the package index. */ +export function compareOrderStatisticTreeIds( + left: OrderStatisticTreeId, + right: OrderStatisticTreeId, +): number { + return compareIds(left, right); +} + +/** + * Internal bulk-build primitive; deliberately omitted from the package index. + * Builds a balanced persistent tree in O(n) from entries already sorted by + * the tree's total order (compare, then id on ties). The order is verified + * unconditionally: a misordered build would silently corrupt every later + * rank and lookup, which is strictly worse than the O(n) check. + */ +export function createOrderStatisticTreeFromSortedEntries< + TId extends OrderStatisticTreeId, + TEntry, + TMeasure, +>( + like: OrderStatisticTree, + sorted: readonly TEntry[], +): OrderStatisticTree { + if (!(like instanceof PersistentOrderStatisticTree)) { + throw new TypeError("Bulk builds require a tree created by this module."); + } + return like[buildFromSortedEntries](sorted); +} +``` + +Add a module-scoped symbol next to `createDeferredMeasureDraft` and a method on `PersistentOrderStatisticTree` (mirror how `createDeferredMeasureDraft` is declared and access the instance's context the same way that method does): + +```ts +[buildFromSortedEntries]( + sorted: readonly TEntry[], +): PersistentOrderStatisticTree { + const context = this.#context; // match the field name used by the class + for (let index = 1; index < sorted.length; index += 1) { + const previous = sorted[index - 1]!; + const current = sorted[index]!; + if ( + compareEntries( + previous, + context.getId(previous), + current, + context.getId(current), + context, + ) >= 0 + ) { + throw new TypeError( + "Bulk build input must be strictly sorted by the tree's total order.", + ); + } + } + const byId = createPersistentMap().asTransient(); + for (const entry of sorted) byId.set(context.getId(entry), entry); + const build = ( + low: number, + high: number, + ): TreeNode | null => { + if (low >= high) return null; + const middle = (low + high) >> 1; + const entry = sorted[middle]!; + const node = createNode(entry, context.getId(entry), context, null); + node.left = build(low, middle); + node.right = build(middle + 1, high); + node.count = 1 + (node.left?.count ?? 0) + (node.right?.count ?? 0); + node.height = 1 + Math.max(nodeHeight(node.left), nodeHeight(node.right)); + // Recompute the subtree measure the same way the incremental path does — + // reuse the existing measure-recompute helper rather than reimplementing + // the combine order (left, own, right). + recomputeNodeMeasure(node, context); + return node; + }; + return new PersistentOrderStatisticTree( + build(0, sorted.length), + byId.freeze(), + context, + ); +} +``` + +Before writing this, read how the incremental path recomputes `node.measure` (search for where `measure` is assigned after rotations) and call that exact helper; if it is inlined, extract it to a shared function rather than duplicating the combine order. Also confirm the constructor's parameter order by reading it — do not guess. + +Midpoint bias note: `(low + high) >> 1` yields height ⌈log2(n+1)⌉, within the AVL balance bound for every n; the balance test in Step 1 is the proof. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pnpm --filter @pretable-internal/row-model test -- order-statistic` +Expected: PASS. + +- [ ] **Step 5: Full package suite** + +Run: `pnpm --filter @pretable-internal/row-model test` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add packages/row-model/src/persistent/order-statistic-tree.ts packages/row-model/src/__tests__/ +git commit -m "feat(row-model): O(n) balanced tree construction from sorted entries" +``` + +--- + +### Task 3: Metadata carryover (`resortRecordMetadata`) + +**Files:** +- Modify: `packages/row-model/src/compiled-query.ts` +- Test: `packages/row-model/src/__tests__/sort-fast-path.test.ts` (create) + +Rebuilds one row's `CompiledRowMetadata` under a new plan without re-running accessors whose values the old metadata already retains. Old metadata retains values in three places: `sortKeys` (old sort columns), `groupPath` (group columns — empty when ungrouped), and `aggregateLeaves[].allLeaf.value` (aggregate columns). Filter columns retain only the boolean verdict — a column that was filter-only in the old plan and is newly sorted must run its accessor. + +**Precondition (documented, enforced by the caller):** only valid when `isSortOnlyChange(previousPlan, nextPlan)` — that guarantees accessor identity for every next-active column, filter set equality (so `filterPasses` carries), and group set equality (so `groupPath` carries). + +- [ ] **Step 1: Write the failing tests** + +Create `packages/row-model/src/__tests__/sort-fast-path.test.ts` (this file grows across Tasks 3–5): + +```ts +import { describe, expect, test, vi } from "vitest"; + +import { createColumnHelper } from "../index"; +import { + compileQuery, + resortRecordMetadata, + type CompiledQuery, +} from "../compiled-query"; +import { PretableRowModelError } from "../errors"; + +interface Row { + id: number; + team: string; + score: number; + note: string; +} + +describe("resortRecordMetadata", () => { + const makeColumns = (spies: { note?: (row: Row) => string }) => { + const helper = createColumnHelper(); + return [ + helper.accessor("team", { type: "text" }), + helper.accessor("score", { type: "number", aggregate: "sum" }), + helper.accessor((row) => (spies.note ?? ((r: Row) => r.note))(row), { + id: "note", + type: "text", + }), + ] as const; + }; + + const row: Row = { id: 1, team: "a", score: 10, note: "n" }; + + test("carries sort, filter, and aggregate values without re-running accessors", () => { + const noteSpy = vi.fn((r: Row) => r.note); + const columns = makeColumns({ note: noteSpy }); + const previousPlan = compileQuery({ + derivations: columns, + query: { + filters: [{ columnId: "team", operator: "equals", value: "a" }], + sort: [{ columnId: "score", direction: "desc" }], + rowGroups: [], + }, + operation: "set-query", + }); + const previous = previousPlan.evaluate({ rowId: 1, row, sourceOrder: 0 }); + const nextPlan = compileQuery({ + derivations: columns, + query: { + filters: [{ columnId: "team", operator: "equals", value: "a" }], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [], + }, + previous: previousPlan, + operation: "set-query", + }); + const rebuilt = resortRecordMetadata(nextPlan, previous); + expect(rebuilt.filterPasses).toBe(previous.filterPasses); + expect(rebuilt.groupPath).toBe(previous.groupPath); + expect(rebuilt.sortKeys).toEqual([{ columnId: "score", value: 10 }]); + expect(rebuilt.aggregateLeaves[0]!.allLeaf.value).toBe(10); + // The sort column's value came from previous.sortKeys; the aggregate value + // from previous.aggregateLeaves. The un-referenced `note` accessor never ran. + expect(noteSpy).not.toHaveBeenCalled(); + }); + + test("runs the accessor for a newly-active sort column only", () => { + const noteSpy = vi.fn((r: Row) => r.note); + const columns = makeColumns({ note: noteSpy }); + const previousPlan = compileQuery({ + derivations: columns, + query: { + filters: [], + sort: [{ columnId: "score", direction: "desc" }], + rowGroups: [], + }, + operation: "set-query", + }); + const previous = previousPlan.evaluate({ rowId: 1, row, sourceOrder: 0 }); + expect(noteSpy).not.toHaveBeenCalled(); + const nextPlan = compileQuery({ + derivations: columns, + query: { + filters: [], + sort: [{ columnId: "note", direction: "asc" }], + rowGroups: [], + }, + previous: previousPlan, + operation: "set-query", + }); + const rebuilt = resortRecordMetadata(nextPlan, previous); + expect(rebuilt.sortKeys).toEqual([{ columnId: "note", value: "n" }]); + expect(noteSpy).toHaveBeenCalledTimes(1); + }); + + test("aggregate leaves embed the NEW dependency", () => { + const columns = makeColumns({}); + const previousPlan = compileQuery({ + derivations: columns, + query: { + filters: [], + sort: [{ columnId: "score", direction: "desc" }], + rowGroups: [], + }, + operation: "set-query", + }); + const previous = previousPlan.evaluate({ rowId: 1, row, sourceOrder: 3 }); + const nextPlan = compileQuery({ + derivations: columns, + query: { + filters: [], + sort: [{ columnId: "team", direction: "asc" }], + rowGroups: [], + }, + previous: previousPlan, + operation: "set-query", + }); + const rebuilt = resortRecordMetadata(nextPlan, previous); + expect(rebuilt.aggregateLeaves[0]!.allLeaf.dependency.sortKeys).toBe( + rebuilt.sortKeys, + ); + expect(rebuilt.aggregateLeaves[0]!.allLeaf.dependency.sourceOrder).toBe(3); + expect(rebuilt.aggregateLeaves[0]!.filteredLeaf).toBe( + rebuilt.aggregateLeaves[0]!.allLeaf, + ); + }); + + test("second call for the same row returns the cached metadata", () => { + const columns = makeColumns({}); + const previousPlan = compileQuery({ + derivations: columns, + query: { filters: [], sort: [{ columnId: "score" }], rowGroups: [] }, + operation: "set-query", + }); + const previous = previousPlan.evaluate({ rowId: 1, row, sourceOrder: 0 }); + const nextPlan = compileQuery({ + derivations: columns, + query: { filters: [], sort: [{ columnId: "team" }], rowGroups: [] }, + previous: previousPlan, + operation: "set-query", + }); + const first = resortRecordMetadata(nextPlan, previous); + expect(resortRecordMetadata(nextPlan, previous)).toBe(first); + // And evaluate() on the same plan sees the seeded cache: + expect(nextPlan.evaluate({ rowId: 1, row, sourceOrder: 0 })).toBe(first); + }); + + test("accessor failure surfaces the slow path's error shape", () => { + const columns = makeColumns({ + note: () => { + throw new Error("boom"); + }, + }); + const previousPlan = compileQuery({ + derivations: columns, + query: { filters: [], sort: [{ columnId: "score" }], rowGroups: [] }, + operation: "set-query", + }); + const previous = previousPlan.evaluate({ rowId: 1, row, sourceOrder: 0 }); + const nextPlan = compileQuery({ + derivations: columns, + query: { filters: [], sort: [{ columnId: "note" }], rowGroups: [] }, + previous: previousPlan, + operation: "set-query", + }); + try { + resortRecordMetadata(nextPlan, previous); + expect.unreachable(); + } catch (error) { + expect(error).toBeInstanceOf(PretableRowModelError); + expect((error as PretableRowModelError).code).toBe("accessor-failed"); + expect((error as PretableRowModelError).details).toMatchObject({ + rowId: 1, + columnId: "note", + }); + } + }); + + test("throws for a foreign plan", () => { + const columns = makeColumns({}); + const previousPlan = compileQuery({ + derivations: columns, + query: { filters: [], sort: [{ columnId: "score" }], rowGroups: [] }, + operation: "set-query", + }); + const previous = previousPlan.evaluate({ rowId: 1, row, sourceOrder: 0 }); + expect(() => + resortRecordMetadata({} as CompiledQuery, previous as never), + ).toThrow(TypeError); + }); +}); +``` + +Before running, verify against the real API: the exact `PretableRowModelError` detail field names (read `errors.ts`), the exact `evaluate` input/metadata types, and whether `helper.accessor` supports the function-accessor form used for `note` (read `column-types.ts` or an existing test; if the helper differs, express the spy through whatever form the helper supports — the point is an accessor whose invocation count we can observe). + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pnpm --filter @pretable-internal/row-model test -- sort-fast-path` +Expected: FAIL — `resortRecordMetadata` is not exported. + +- [ ] **Step 3: Implement** + +In `compiled-query.ts`, add a static on `CompiledQueryPlan` plus a free-function export. Reuse the exact metadata construction from `evaluate` (lines ~1422–1464) — same freeze pattern, same cache entry shape: + +```ts +static resortMetadata( + plan: unknown, + previous: CompiledRowMetadata, +): CompiledRowMetadata { + if (!(plan instanceof CompiledQueryPlan)) { + throw new TypeError("Metadata carryover requires a compiled query plan."); + } + const cached = plan.#evaluationCache.get(previous.row as object); + if ( + cached && + Object.is(cached.rowId, previous.rowId) && + cached.sourceOrder === previous.sourceOrder + ) { + return cached.metadata as never; + } + const carried = (columnId: string): { found: boolean; value: unknown } => { + for (const key of previous.sortKeys) { + if (key.columnId === columnId) return { found: true, value: key.value }; + } + for (const key of previous.groupPath) { + if (key.columnId === columnId) return { found: true, value: key.value }; + } + for (const leaf of previous.aggregateLeaves) { + if (leaf.columnId === columnId) { + return { found: true, value: leaf.allLeaf.value }; + } + } + return { found: false, value: undefined }; + }; + const valueOf = (columnId: string): unknown => { + const prior = carried(columnId); + if (prior.found) return prior.value; + const column = plan.#byId.get(columnId)!; + try { + return column.accessor(previous.row as never); + } catch (cause) { + throw new PretableRowModelError( + "accessor-failed", + `Column ${columnId} accessor failed.`, + { + operation: plan.#operation, + rowId: previous.rowId, + columnId, + cause, + }, + ); + } + }; + const sortKeys = Object.freeze( + plan.#runtimeQuery.sort.map((entry) => + Object.freeze({ + columnId: entry.columnId, + value: valueOf(entry.columnId), + }), + ), + ); + const dependency = Object.freeze({ + sourceOrder: previous.sourceOrder, + sortKeys, + }); + const aggregateLeaves = Object.freeze( + plan.#aggregateColumns.map((column, index) => { + const prior = previous.aggregateLeaves[index]; + const value = + prior !== undefined && prior.columnId === column.id + ? prior.allLeaf.value + : valueOf(column.id); + const allLeaf = Object.freeze({ + id: previous.rowId, + row: previous.row, + value, + dependency, + }); + return Object.freeze({ + columnId: column.id, + aggregate: column.aggregate, + allLeaf, + filteredLeaf: previous.filterPasses ? allLeaf : undefined, + }); + }), + ); + const metadata = Object.freeze({ + rowId: previous.rowId, + row: previous.row, + sourceOrder: previous.sourceOrder, + filterPasses: previous.filterPasses, + groupPath: previous.groupPath, + sortKeys, + aggregateLeaves, + }); + plan.#evaluationCache.set(previous.row as object, { + rowId: previous.rowId, + sourceOrder: previous.sourceOrder, + metadata: metadata as never, + }); + return metadata as never; +} +``` + +Free function (typed against the real generics — mirror `evaluate`'s signature style): + +```ts +/** + * Rebuilds one row's metadata under `nextPlan` from a prior evaluation, + * re-running accessors only for columns whose values the prior metadata does + * not retain. Valid ONLY when `isSortOnlyChange(previousPlan, nextPlan)` — + * the caller owns that check; this function trusts filter verdicts and group + * paths it is handed. + */ +export function resortRecordMetadata( + nextPlan: CompiledQuery, + previous: CompiledRowMetadata, TRowId, TColumns>, +): CompiledRowMetadata, TRowId, TColumns> { + return CompiledQueryPlan.resortMetadata( + nextPlan, + previous as never, + ) as never; +} +``` + +Adjust the `never` casts to whatever the file's existing internal casting idiom is (`as unknown as` chains appear throughout) — match it. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pnpm --filter @pretable-internal/row-model test -- sort-fast-path` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add packages/row-model/src/compiled-query.ts packages/row-model/src/__tests__/sort-fast-path.test.ts +git commit -m "feat(row-model): rebuild row metadata by carryover for sort-only plan changes" +``` + +--- + +### Task 4: Synchronous root rebuild (`sort-rebuild.ts`) + +**Files:** +- Create: `packages/row-model/src/sort-rebuild.ts` +- Modify: `packages/row-model/src/diagnostics.ts` +- Test: `packages/row-model/src/__tests__/sort-fast-path.test.ts` (extend) + +Composes Tasks 1–3 into a whole-root rebuild. Also adds the instrumentation counters (`synchronousRebuilds`, `synchronousRebuildMs`) — they land here because this module is what reports them. + +- [ ] **Step 1: Add the instrumentation fields** + +In `diagnostics.ts`, add to the work interface (near `transitionRows`, line ~24): + +```ts +/** Sort-only rebuilds taken synchronously, bypassing the cooperative path. */ +readonly synchronousRebuilds: number; +/** Total wall time inside synchronous sort-only rebuilds. */ +readonly synchronousRebuildMs: number; +``` + +Initialize both to `0` in `createInstrumentation` (line ~84) and add them to any reset/snapshot key list the file maintains (there is a key array near line 61 — read it and follow its rule; `schedulerSliceDurations` is listed there, so scalar fields may or may not need registration — mirror how `transitionRows` is handled). + +- [ ] **Step 2: Write the failing tests** + +Extend `sort-fast-path.test.ts`: + +```ts +import { rebuildRootForSortOnlyChange } from "../sort-rebuild"; +import { createInstrumentation } from "../diagnostics"; // match real export name + +describe("rebuildRootForSortOnlyChange", () => { + // Build a real root via createLocalRowModel (flat query, some filtered-out + // rows, an aggregate column), read it with the existing internal accessor + // used by transitions.test.ts (getLocalRowModelRevisionCauseForTesting's + // sibling — find how tests obtain the root; if none exists for the root + // itself, drive the comparison entirely through the public snapshot of a + // model wired in Task 5 and keep this describe focused on the pure + // function via a hand-built root). + // + // Pure-function assertions, given captured root + sort-only nextPlan: + + test("the rebuilt root's visible order equals a cold build under nextPlan", () => { + // Oracle: createFlatVisibleIndex(records.map(re-evaluated under a FRESH + // equivalent plan), freshPlan.compareRows) — the existing pure builder in + // visible-index.ts. Compare rowId sequences at every rank. + }); + + test("filtered-out rows stay out of visible but keep updated records in rows", () => {}); + + test("revision and parentRevision are the requested values; cause kind is set-query", () => {}); + + test("sourceOrder and expansion are carried by reference from the captured root", () => {}); + + test("publicRow and integrity are carried by reference per record", () => {}); + + test("instrumentation counts one rebuild and nonzero duration", () => { + // pass createInstrumentation() and a controllable now(): first call 0, + // second call 7 -> synchronousRebuildMs === 7, synchronousRebuilds === 1. + }); +}); +``` + +Write these as real tests, not stubs — the comments above define each test's contract; the fixture is shared. Fixture requirements (memory: choose data that can disprove): at least 6 rows; the old sort order, the new sort order, and source order must be three pairwise-distinct permutations (assert this inside the test); at least one row filtered out; a tie on the new sort key so the rowId tiebreak is exercised. + +- [ ] **Step 3: Run tests to verify they fail** + +Run: `pnpm --filter @pretable-internal/row-model test -- sort-fast-path` +Expected: FAIL — module `../sort-rebuild` does not exist. + +- [ ] **Step 4: Implement `sort-rebuild.ts`** + +```ts +import { + isSortOnlyChange, + resortRecordMetadata, + type CompiledQuery, +} from "./compiled-query"; +import type { LocalRowModelInstrumentation } from "./diagnostics"; +import type { RevisionRoot, RowRecord } from "./internal-types"; +import type { PretableRowId } from "./column-types"; +import { + compareOrderStatisticTreeIds, + createOrderStatisticTreeFromSortedEntries, + instrumentOrderStatisticTree, +} from "./persistent/order-statistic-tree"; +import { createFlatVisibleTree } from "./visible-index"; + +/** + * Synchronous whole-root rebuild for a sort-only plan change on an ungrouped + * query. Runs to completion on the caller's stack — the deliberate trade + * measured in #457: scheduler hops cost frames in the browser, and the + * carryover makes the total work small enough to spend inline. + */ +export function rebuildRootForSortOnlyChange< + TRow extends object, + TRowId extends PretableRowId, + TColumns, +>(options: { + readonly captured: RevisionRoot; + readonly nextPlan: CompiledQuery; + readonly revision: number; + readonly now: () => number; + readonly instrumentation?: LocalRowModelInstrumentation; +}): RevisionRoot { + const { captured, nextPlan, revision, now, instrumentation } = options; + if (!isSortOnlyChange(captured.queryPlan, nextPlan)) { + throw new TypeError( + "Synchronous rebuild requires a sort-only plan change.", + ); + } + if (nextPlan.query.rowGroups.length > 0) { + throw new TypeError("Synchronous rebuild requires an ungrouped query."); + } + const startedAt = now(); + const rowsDraft = captured.rows.asTransient(); + const visible: RowRecord[] = []; + for (const entry of captured.sourceOrder.entries()) { + const previous = captured.rows.get(entry.rowId); + if (previous === undefined) continue; + const metadata = resortRecordMetadata( + nextPlan, + previous.metadata as never, + ) as unknown as RowRecord["metadata"]; + const record = Object.freeze({ ...previous, metadata }); + rowsDraft.set(record.rowId, record); + if (metadata.filterPasses) visible.push(record); + } + const compareRows = nextPlan.compareRows as unknown as ( + left: RowRecord["metadata"], + right: RowRecord["metadata"], + ) => number; + // The composite (compareRows, then id) mirrors the tree's own total order; + // the bulk constructor verifies it and would throw on divergence. + visible.sort( + (left, right) => + compareRows(left.metadata, right.metadata) || + compareOrderStatisticTreeIds(left.rowId, right.rowId), + ); + const tree = createOrderStatisticTreeFromSortedEntries( + instrumentOrderStatisticTree( + createFlatVisibleTree(compareRows), + instrumentation, + ), + visible, + ); + const root = Object.freeze({ + revision, + parentRevision: revision - 1, + rows: rowsDraft.freeze(), + sourceOrder: captured.sourceOrder, + visible: Object.freeze({ rows: tree }), + queryPlan: nextPlan, + expansion: captured.expansion, + cause: Object.freeze({ kind: "set-query" as const }), + }); + if (instrumentation !== undefined) { + instrumentation.work.synchronousRebuilds += 1; + instrumentation.work.synchronousRebuildMs += Math.max( + 0, + now() - startedAt, + ); + } + return root; +} +``` + +Check against reality before finishing: the exact `RevisionRoot` field set (read `internal-types.ts` — if it has fields beyond the eight above, carry them from `captured`), whether `sourceOrder.entries()` is the iteration API the cooperative path uses (it is — `cooperative-transition.ts:436`), whether `instrumentPersistentMap` should wrap the rows draft path (mirror what the cooperative candidate does at line 356), and whether the `cause` shape matches what `runTransitionSlice` publishes. Do not add this module to `index.ts`. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `pnpm --filter @pretable-internal/row-model test -- sort-fast-path` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add packages/row-model/src/sort-rebuild.ts packages/row-model/src/diagnostics.ts packages/row-model/src/__tests__/sort-fast-path.test.ts +git commit -m "feat(row-model): synchronous whole-root rebuild for sort-only changes" +``` + +--- + +### Task 5: Wire the fast path into `setQuery` + +**Files:** +- Modify: `packages/row-model/src/create-local-row-model.ts:1099-1138` (the `setQuery` method) +- Test: `packages/row-model/src/__tests__/sort-fast-path.test.ts` (extend) + +The branch goes after the `nextPlan === queryPlan` short-circuit and before `startTransition`. On error it must reproduce `failTransition`'s observable semantics (error status carrying this transition id, rejected `finished`, root unchanged) without a transition object. + +- [ ] **Step 1: Write the failing tests** + +Extend `sort-fast-path.test.ts`, using the `ManualScheduler` class from `transitions.test.ts` (import it if exported; otherwise copy the minimal shape — schedule pushes, flush drains): + +```ts +describe("setQuery sort-only fast path", () => { + // Shared fixture: flat model, filter active, aggregate column, 8 rows, + // ManualScheduler injected via transitionScheduler, instrumentation + // attached the way work.test.ts attaches it. + + test("resolves synchronously without any scheduler task", async () => { + // setQuery(sort-only change); assert scheduler.entries stays empty, + // status.kind === "ready" immediately, snapshot shows the new order, + // await transition.finished resolves to root.revision (old + 1), + // instrumentation.work.synchronousRebuilds === 1. + }); + + test("mutation twin: a filter change takes the cooperative path", () => { + // Same model; setQuery changing a filter value: scheduler.entries is + // non-empty OR status.kind === "rebuilding", and synchronousRebuilds + // stays 0. Proves the fast-path predicate can fail. + }); + + test("sorting still sorts (old behavior survives)", () => { + // Assert the actual row order of the snapshot after the fast path against + // the hand-computed expected permutation — not merely that the path ran. + // Include a tie on the sort key: tied rows order by rowId. + }); + + test("supersedes an in-flight cooperative transition", async () => { + // Start a filter-change transition (do NOT flush the scheduler), then a + // sort-only setQuery. The first transition's finished rejects with + // PretableTransitionCancelledError(reason "superseded"); the second + // resolves; final snapshot reflects OLD filters + NEW sort (the fast path + // rebuilt from the last committed root, not the abandoned candidate). + }); + + test("notifies subscribers exactly once", () => {}); + + test("onQueryChange / snapshot.query reports the new sort", () => {}); + + test("setRows immediately after a fast setQuery applies incrementally", () => { + // After the fast path, a setRows update to one row must land in the + // correct sorted position via the normal incremental path (the rebuilt + // tree accepts later inserts — Task 2 proved the primitive; this proves + // the wiring). + }); + + test("equivalence with a cold model", () => { + // Model A: created with query Q1, setQuery(Q2 sort-only), flush nothing. + // Model B: created directly with Q2 (cold build, same rows). + // Assert identical visibleRowCount and identical rowAt(i) row identity + // across the full range, and identical query snapshots. + }); + + test("accessor failure: error status carries the transition id, root unchanged", async () => { + // Newly-active sort column whose accessor throws for one row. setQuery + // must not throw synchronously; state.status matches the slow path's + // shape: { kind: "error", transitionId: id, error: PretableRowModelError }. + // finished rejects with the same error; snapshot still shows the OLD + // order; a subsequent valid setQuery recovers. + // FIRST write this test against the SLOW path (filter+sort change) to pin + // the expected observable shape, then point it at the fast path. + }); +}); +``` + +Write all of these as real tests with the shared fixture. Fixture discipline (memories: choose data that can disprove, assert the old behavior survives): old order, new order, and source order pairwise distinct — assert the controls inside the fixture setup; the mutation twin is mandatory, not optional. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pnpm --filter @pretable-internal/row-model test -- sort-fast-path` +Expected: FAIL — fast-path assertions fail (scheduler receives tasks; `synchronousRebuilds` stays 0). + +- [ ] **Step 3: Implement the branch** + +In `setQuery` (`create-local-row-model.ts`), after the `nextPlan === queryPlan` block (line ~1124), insert: + +```ts +if ( + isSortOnlyChange(queryPlan, nextPlan) && + nextPlan.query.rowGroups.length === 0 +) { + cancelActiveTransition("superseded"); + const previousRevision = root.revision; + const revision = previousRevision + 1; + let committedRoot: RevisionRoot; + try { + committedRoot = rebuildRootForSortOnlyChange({ + captured: root, + nextPlan, + revision, + now: transitionRuntime.now, + instrumentation, + }); + } catch (error) { + const typed = transitionError(error, "set-query"); + state = Object.freeze({ + snapshot, + status: Object.freeze({ + kind: "error" as const, + transitionId: id, + error: typed, + }), + }); + const finished = Promise.reject(typed); + void finished.catch(() => undefined); + return { + transition: Object.freeze({ + id, + requestedQuery: nextPlan.query, + finished, + cancel: () => cancelTransitionHandle(id, "set-query"), + }), + notify: true, + }; + } + queryPlan = committedRoot.queryPlan; + query = committedRoot.queryPlan.query; + derivations = committedRoot.queryPlan.derivations; + commit(committedRoot, READY); + distinctValues.publishTransitionRoot(committedRoot); + changeJournal.appendBarrier(previousRevision, revision); + return { + transition: Object.freeze({ + id, + requestedQuery: nextPlan.query, + finished: Promise.resolve(revision), + cancel: () => cancelTransitionHandle(id, "set-query"), + }), + notify: true, + }; +} +``` + +Imports: `isSortOnlyChange` from `"./compiled-query"`, `rebuildRootForSortOnlyChange` from `"./sort-rebuild"`. + +Verify against the surrounding code while implementing: that `transitionError` (line ~694) is the right error mapper here (it is what `failTransition` uses); that reentrancy guards (`guarded("set-query", ...)` already wraps this) hold — the rebuild runs user accessors, and a reentrant `setRows` from inside an accessor must hit the same reentrancy error the slow path produces (add a test if `guarded` covers it; read how `guarded` detects reentrancy first); and that no `queryPlan.query.rowGroups` check is needed for the previous plan (`isSortOnlyChange` already proved groups unchanged, so checking `nextPlan` alone suffices). + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pnpm --filter @pretable-internal/row-model test -- sort-fast-path` +Expected: PASS, including the error-path and supersede tests. + +- [ ] **Step 5: Full package suite** + +Run: `pnpm --filter @pretable-internal/row-model test` +Expected: PASS. Pay attention to `transitions.test.ts` and `work.test.ts` — any test that asserted a sort-only `setQuery` schedules cooperative work will now fail; each such failure is a deliberate behavior change. Update those tests to either use a non-sort-only change (when the test's subject is the cooperative machinery) or assert the new synchronous behavior (when the test's subject is sorting). Record every such edit in the commit message. + +- [ ] **Step 6: Commit** + +```bash +git add packages/row-model/src/create-local-row-model.ts packages/row-model/src/__tests__/ +git commit -m "feat(row-model): sort-only setQuery completes synchronously on flat queries" +``` + +--- + +### Task 6: Repo-wide verification and gate accounting + +**Files:** +- Possibly modify: `apps/bench` gate config (comment only), package api reports + +- [ ] **Step 1: Find the slice-bound gate and annotate it** + +Run: `grep -rn "rebuild_slice_max_ms" apps/bench --include="*.ts" -l` + +At the gate's definition site, add a comment (not a threshold change): + +```ts +// The flat sort fast path (#457) is synchronous BY DESIGN and reports under +// work.synchronousRebuildMs, never as a scheduler slice — it is exempt from +// this bound. Grouped and non-sort-only transitions remain governed by it. +``` + +Confirm by reading the gate's data source that it consumes `schedulerSliceDurations` (which the fast path never touches), so the exemption is structural, not aspirational. + +- [ ] **Step 2: Repo-wide checks** + +Run, in order (build BEFORE api — stale `dist/` silently strips exports): + +```bash +pnpm build +pnpm typecheck +pnpm lint +pnpm test +pnpm api +``` + +Expected: all green; `pnpm api` produces no report diff (`git status` clean on `*.api.md`) because nothing new is exported from any package index. If a report changed, something leaked into a public surface — fix the export, don't commit the report. + +Note (memory: local test flakes): the react vitest suite times out 1–2 random tests per full run locally; re-run a failure once before investigating. + +- [ ] **Step 3: Commit (gate comment only, if any)** + +```bash +git add apps/bench +git commit -m "docs(bench): note the sort fast path's designed exemption from the slice bound" +``` + +--- + +### Task 7: Performance verification (the actual success gate) + +**Files:** none committed except the scratch script's results pasted into the PR body. + +The spec's bar: browser-measured, S2 sort at target scale `completed ×3` with `interaction_latency_ms` within ~2x of TanStack's in the same run; no regression at 3k; Node decomposition rerun as work accounting. + +- [ ] **Step 1: Node decomposition (work accounting, informational)** + +Write a scratch script (in the session scratchpad, NOT committed) that mirrors #457's methodology: `createLocalRowModel` + the S2 dataset generator from `apps/bench` (find it: `grep -rn "S2" apps/bench/src --include="*.ts" -l`), 50,000 rows, apply the S2 sort interaction, await `finished`, report wall time. Run against this branch's build. + +Expected: wall time drops from ~515ms to double-digit milliseconds. Record before/after numbers. + +- [ ] **Step 2: Browser bench — protocol** + +Memories that bind this step: **bench A/B — change ONE thing** (the two sides must be this branch vs its merge-base, each with `packages/react` dist rebuilt before measuring); **bench port collision** (ensure no parallel session holds the bench port — check `lsof -i :4173` first and isolate if held); **check load** (quiet machine; run the control twice and check spread before trusting any comparison). + +```bash +# side A: merge-base +git worktree list # confirm you are in the task worktree +pnpm build +pnpm bench:matrix --adapters=pretable,tanstack --scenarios=S2 --scripts=sort --scale=target --repeats=3 +# record, then side B: this branch, rebuild, rerun identically +``` + +Check the actual flag names against `apps/bench` docs/scripts before running (`--scale=target` vs however target scale is spelled; #457 used the words "target" and "hypothesis"). + +Also run the 3k guard: same command with `--scale=hypothesis`, plus `--scripts=filter-metadata` once to confirm no collateral change on a non-sort interaction. + +- [ ] **Step 3: Evaluate against the spec bar** + +- 50k: pretable `completed ×3`, latency ≤ ~2x TanStack's same-run latency. +- 3k: at or below the 50–59ms band from #457's A/B table. +- If the bar is missed: STOP, record the numbers, and report — the spec names the likely next lever (the residual is then in tree/map constants or renderer, and that is a finding, not a tweak-until-green situation). Do not chase the number by changing scheduling parameters; that lever is refuted. + +- [ ] **Step 4: Grouped gate unaffected** + +Run the bench script that exercises the grouped gate (the group script is comparative post-#477): confirm `rebuild_slice_max_ms` still passes and grouped sort latency is unchanged (it must be — grouped queries never enter the fast path; this run proves it). + +--- + +### Task 8: PR and merge on green + +- [ ] **Step 1: Re-check origin/main** (memory: parallel sessions) + +```bash +git fetch origin main && git log --oneline HEAD..origin/main +``` + +If new commits touch `packages/row-model`, read them before rebasing; rebase and rerun the full package suite. + +- [ ] **Step 2: Push and open the PR** + +```bash +git push -u origin blove/spec-457-cbb90d +gh pr create --title "feat(row-model): sort-only setQuery completes synchronously at TanStack-parity cost" --body " + +🤖 Generated with [Claude Code](https://claude.com/claude-code)" +``` + +- [ ] **Step 3: Merge on green** + +Watch checks (`gh pr checks --watch`). Vercel quota memory applies: the preview smoke test is required; if the daily quota is exhausted, wait — do not bypass. On green, squash-merge. Then verify the merge actually happened (memory: never record an unverified merge state): + +```bash +gh pr view --json state,mergedAt +git fetch origin main && git log --oneline -1 origin/main +``` + +--- + +## Self-review notes (already applied) + +- Spec coverage: classifier → Task 1; carryover → Task 3; bulk build → Task 2; synchronous wiring + error semantics → Task 5; instrumentation/gate → Tasks 4/6; equivalence + mutation + old-behavior tests → Tasks 4/5; browser-measured success bar → Task 7. Deferred items (lever 4, filter path, grouped carryover) have no tasks by design. +- The one intentional deviation from the spec's literal text: the classifier compares runtime facets including both authorities (post-#467), amended in the spec. +- Type-consistency: `isSortOnlyChange`, `resortRecordMetadata`, `rebuildRootForSortOnlyChange`, `createOrderStatisticTreeFromSortedEntries`, `compareOrderStatisticTreeIds`, `work.synchronousRebuilds`, `work.synchronousRebuildMs` are the only new names; used consistently above. +- Known intentional looseness: fixture syntax in test code must be aligned with the package's real helper/API shapes at implementation time (called out inline in Tasks 1, 3, 4); the assertions are the contract. diff --git a/docs/superpowers/plans/2026-08-18-decorated-entries-and-bulk-mount.md b/docs/superpowers/plans/2026-08-18-decorated-entries-and-bulk-mount.md new file mode 100644 index 000000000..2bce0cd1a --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-decorated-entries-and-bulk-mount.md @@ -0,0 +1,90 @@ +# Decorated Entries + Bulk Mount Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close #457's last two failing verdicts — the grouped-gate regression (per-comparison WeakMap resolution) and the sort-during-mount race (450ms blank cooperative mount) — via decorated tree entries, a synchronous bulk mount path, and reorder composition into active replacements. + +**Architecture:** C1 (row-model): tree entries across the flat visible tree, grouped leaf trees, and aggregate trees carry `{record|leaf, keys}`; comparators become `compareWithSortKeys` property reads. C2a (layout-core + renderer-dom): when the base `RowHeightIndex` holds no retained state, a replacement builds synchronously in O(n); the controller runs it to completion in one pass. C2b (renderer-dom): `captureActiveTarget` accepts a reorder reset as a final retarget; `finishReplacement` composes it via `candidate.reorder()`. + +**Spec:** `docs/superpowers/specs/2026-08-18-decorated-entries-and-bulk-mount-design.md`. Grounding documents (read the ones your task cites): the lifecycle map (in the C-cycle coordination context), `/grouped-gate-regression-findings.md`, `/grouped-rebuild-timing.mjs`, `/cycle-2-results.md`. + +**Conventions:** as cycles 1–2 (TDD, mutation-hardening, constraints-only comments, prettier/lint/typecheck per commit, never touch ~/repos/pretable, verify HEAD before amends). + +--- + +### Task C1: Decorated tree entries (grouped + flat) + +**Files:** +- Modify: `packages/row-model/src/visible-index.ts`, `group-index.ts`, `cooperative-transition.ts`, `transaction-draft.ts`, `sort-rebuild.ts`, `create-local-row-model.ts` (recompile-path reseed), possibly `persistent/aggregate-tree.ts` (leaf shape) and `internal-types.ts` +- Test: existing suites are the oracle; timing recorded via `grouped-rebuild-timing.mjs` + +The change: everywhere a persistent tree currently stores a bare record (flat visible tree, grouped leaf trees) or a bare aggregate leaf, store the entry WITH its resolved keys, and compare via `compareWithSortKeys(plan, l.record, l.keys, r.record, r.keys)`. One shape convention across all three trees. Key resolution happens exactly once per insert (`sortKeysOf`, or keys the caller already holds — sort-rebuild's decorated pairs flow through). + +Method: +- [ ] **Step 1:** Map every tree construction + insert + entry-read site (grep `createFlatVisibleTree`, `insertOrReplace`, `entryAt`, `range(`, leaf-tree builders in group-index, aggregate-tree leaf construction). Write the inventory into your working notes; the A2 audit lists the comparator sites. +- [ ] **Step 2:** Introduce the decorated entry type (e.g. `OrderedRowEntry<...> = { record, keys }`) and change `createFlatVisibleTree` to store it: `getId: (entry) => entry.record.rowId`, `compare: (l, r) => compareWithSortKeys(plan, l.record, l.keys, r.record, r.keys)`. Adapt every consumer read (`entry.rowId` → `entry.record.rowId` etc.) and every insert site to construct `{record, keys: sortKeysOf(plan, record)}` (or pass held keys). Run the package suite after EACH file's adaptation. +- [ ] **Step 3:** Same for grouped leaf trees and `compareAggregateLeaves`' tree (group-index.ts; the aggregate-tree leaf gains keys per the minimal-ripple encoding you choose — document the choice). +- [ ] **Step 4:** `sort-rebuild.ts`: its `{record, keys}` pairs now feed `createOrderStatisticTreeFromSortedEntries` DIRECTLY (the tree's entry type matches) — delete the map-back-to-records step. The bulk constructor's verification comparator now reads entry keys (no store gets). +- [ ] **Step 5:** Full package suite green (baseline 412; equivalence oracles: the A2 grouped pin test, A3 grouped/aggregate tests, cycle-1 fast-path tests). Root build + typecheck (layout/renderer untouched by C1 but the barrel types ripple — verify). +- [ ] **Step 6:** Timing: `pnpm build`, then run `grouped-rebuild-timing.mjs` per its header (3 passes, pre-change HEAD vs your working tree, interleaved, quiet machine). Target: slice-work total recovered to ~pre-A2 (~1870ms band in that harness). Record numbers in `/c1-grouped-timing.md`. If NOT recovered, STOP and report with the numbers. +- [ ] **Step 7:** Mutation-harden: (a) make one insert site store stale/wrong keys → an equivalence test must fail (if none does, the fixtures can't disprove decoration — fix the fixture); (b) revert one comparator to compareRecordRows → timing regresses (recorded, not unit-asserted). Prettier/lint. Commit: +```bash +git add -A packages/row-model && git commit -m "perf(row-model): tree entries carry their sort keys" +``` + +--- + +### Task C2a: Synchronous bulk replacement when nothing is retained + +**Files:** +- Modify: `packages/layout-core/src/row-height-index.ts`, `packages/renderer-dom/src/row-layout-controller.ts` +- Test: both packages' suites + +Layout-core: +- [ ] **Step 1 (TDD):** tests first — `hasRetainedState` predicate (false on a fresh/empty index; true with ≥1 measurement, ≥1 tombstone, or retained entries — read what state categories exist: `#measurements`, `#tombstones`, `#tombstoneOrder`); bulk path equivalence oracle: for a no-retained-state base, the new synchronous path over `{rowCount, entryAt}` equals the cooperative builder's result at EVERY rank (offsets, heights, total) for sizes {0, 1, 32, 1000}; post-bulk mutations (measure, replace, reorder) behave identically to a cooperatively-built twin; the predicate DISABLES the path (a base with one measurement uses the cooperative builder — assert via whatever distinguishes them: diagnostics counters or builder-phase observables). +- [ ] **Step 2:** Implement. Recommended shape (adapt to the class): inside `beginReplacement`, when `!this.hasRetainedState`, return a builder whose first `advance()` completes everything via `buildBalancedSequence` + bulk identity-map construction (values = measured ?? source estimatedHeight ?? defaultHeight — read RHI:1906-1926's exact height rule and reproduce it; at true mount all three collapse to defaultHeight but the rule must match for the equivalence oracle). This keeps the controller's builder contract untouched. Constraint comment: why synchronous (spec C2a; ~20ms at 50k measured in B3). +- [ ] **Step 3:** Controller: in `startReplacement`, extend the eager gate: `if (state.observedRevision === null || !state.rowHeights.hasRetainedState)` → `runReplacementSlice(replacement, /*ignoreDeadline*/ true)` (read the existing eager gate at ~:1412-1419 first — the bulk builder makes ignoreDeadline complete in ONE unit, so this is cheap; keep `eagerInitialRowLimit` working as-is for its documented purpose). Verify: a 50k mount publishes during `activateController`'s synchronize pass — no scheduler entries (controller test, TDD). +- [ ] **Step 4:** Mutations: (a) predicate always-true → the retained-state test fails (a measured base must NOT take the bulk path); (b) bulk path skips the height rule (always defaultHeight) → equivalence oracle with source estimates fails. Suites green (layout-core 109+, renderer-dom 137+), repo typecheck. Commit: +```bash +git add packages/layout-core packages/renderer-dom && git commit -m "perf(layout-core,renderer-dom): replacements over unmeasured state build synchronously" +``` + +--- + +### Task C2b: Compose a reorder into an active replacement + +**Files:** +- Modify: `packages/renderer-dom/src/row-layout-controller.ts` +- Test: `packages/renderer-dom/src/__tests__/indexed-renderer.test.ts` (B4's harness) + +- [ ] **Step 1 (TDD, red first):** + 1. Reorder reset arriving mid-replacement (drive a real cooperative replacement — a base with retained measurements so C2a doesn't bulk it — then a real sort-only setQuery): NO restart (`replacementStartCount` unchanged), `reorderComposeCount` +1, final published order equals a from-scratch oracle, staged measurements taken during the replacement survive into the final index. + 2. Reorder then CHANGES before finish → fail-closed restart (`reorderComposeFallbackCount` +1 or restart counter — pick the observable and be consistent). + 3. Reorder then newer reorder → last wins (final order = second sort's). + 4. Compose-time `reorder()` throw (lie about the row set) → restart fallback, no error publish. + 5. Anchor: the composed finish restores the anchor against the FINAL order with replacement semantics (mirror B4's anchor test through the compose path). +- [ ] **Step 2:** Implement per spec C2b: `captureActiveTarget` accepts `{kind:"reset", reason:"reorder", toRevision === targetRevision}` as a retarget — set `replacement.pendingReorder = target` (the snapshot), advance `capturedRevision = toRevision`, swap `latestTarget`, republish rebuilding status (mirror the changes-retarget branch); if `pendingReorder` is ALREADY set and the wake is anything but a newer aligned reorder → return false (restart). Read the finish gate (RLC:994-1002) — `pendingReorder` must NOT block the gate; in `finishReplacement`, after staged replay, `if pendingReorder: candidate = candidate.reorder(replacementSourceOf(latestTarget))` inside the existing try so throws hit the restart fallback. Counters on the B4 seam. + - CAREFUL: after accepting a reorder, `appliedRevision`/catch-up contiguity — the pending queue's changesets end at the pre-reorder revision; the finish gate compares `appliedRevision === capturedRevision`, which the reorder advanced. Reconcile deliberately (e.g. `pendingReorder` records `{target, fromApplied: previousCapturedRevision}` and the gate treats `appliedRevision === fromApplied && pendingReorder` as satisfied) — read the gate and pick the minimal coherent rule; document it in a constraint comment. +- [ ] **Step 3:** Mutations: (i) skip the compose (publish without reorder) → order oracle fails; (ii) accept changes after pendingReorder → test 2 fails; (iii) drop the throw fallback → test 4 fails. Suites green; repo typecheck; prettier. Commit: +```bash +git add packages/renderer-dom && git commit -m "feat(renderer-dom): reorders compose into active replacements at finish" +``` + +--- + +### Task C3: End-to-end re-verification (the four bars) + +Same protocol as cycle-2 B5 (see `cycle-2-results.md` for method), no commits: +- [ ] Grouped gate ×3 quiet runs: ≤8ms, near 7.7. +- [ ] Browser bench: 50k + 3k sort, pretable+tanstack, repeats 3; filter-metadata collateral; mount metrics if the bench reports time-to-first-window (check `bench-runtime.ts` for a mount/first-paint metric; else use one PLAYWRIGHT_PERF_TRACE mount trace to read first-publish timing). +- [ ] One 50k sort trace: interaction window attribution; `reorderFallbackCount === 0` and compose/reorder counters engaged (temporary instrumentation acceptable, reverted after, as B5 did). +- [ ] Full repo gates: typecheck, lint, test, api (build first). +- [ ] Write `/cycle-3-results.md` with all four verdicts; STOP — the merge decision goes to Brian with the numbers. + +--- + +## Self-review notes (applied) + +- Spec coverage: C1 → Task C1; C2a → Task C2a; C2b → Task C2b; bars → C3. Out-of-scope items have no tasks. +- The one deliberate implementation freedom: C1's aggregate-leaf keys encoding and C2a's builder-vs-method shape are implementer choices bounded by the spec's contracts; both must be reported. +- Sequencing enforced: C1 unblocks the gate before C2 touches the renderer; C3 evaluates everything together. diff --git a/docs/superpowers/plans/2026-08-18-sort-reuse-and-reorder-signal.md b/docs/superpowers/plans/2026-08-18-sort-reuse-and-reorder-signal.md new file mode 100644 index 000000000..0ff98f43d --- /dev/null +++ b/docs/superpowers/plans/2026-08-18-sort-reuse-and-reorder-signal.md @@ -0,0 +1,283 @@ +# Sort-Key Ownership + Reorder Signal Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Take the 50k S2 sort from ~400ms to the end-to-end ~2x-of-TanStack bar by (A) moving sort keys out of per-row records so a sort-only change carries records/rows-map by identity, and (B) publishing an order-only journal signal so the renderer's height index permutes instead of re-ingesting 50k rows. + +**Architecture:** Phase A restructures `packages/row-model`: `CompiledRowMetadata` loses `sortKeys`, each `CompiledQueryPlan` owns a WeakMap sort-key store, `compareRows` becomes record-based, four internal consumers reroute, `sort-rebuild.ts` v2 reuses records and the rows HAMT, `resortRecordMetadata` is deleted. Phase B adds reset reason `"reorder"` to the change journal (public union — old consumers fail closed to full replacement), a synchronous `RowHeightIndex.reorder()` in layout-core, and a permutation path in renderer-dom's row-layout controller. + +**Tech Stack:** TypeScript, vitest per package, pnpm workspace, api-extractor gate (`pnpm build` then `pnpm api`), Playwright bench. + +**Spec:** `docs/superpowers/specs/2026-08-18-sort-reuse-and-reorder-signal-design.md` — read it first. Cycle-1 spec for background: `docs/superpowers/specs/2026-08-17-sort-fast-path-design.md`. + +**Facts established during spec work (verify only if something contradicts):** +- All `metadata.sortKeys` / `dependency` consumers are internal to row-model: `transaction-draft.ts:348` (`sameFlatOrder` → `sameKeyValues`), `group-index.ts:897-921` (`compareAggregateLeaves`, synthesizes fake metadata from `dependency.sortKeys`), `group-index.ts:855`, `persistent/aggregate-tree.ts:480` (`Object.is(left.dependency, right.dependency)`). +- `compareRows` call sites: `visible-index.ts` (38/64/89/100), `cooperative-transition.ts:406`, `transaction-draft.ts:1385`, `sort-rebuild.ts:55-71`, `group-index.ts:855,905`. +- Journal: `JournalEntry` barrier already carries `reason: ResetReason`; public reset reasons are `"unknown-revision" | "journal-evicted" | "bulk-replace"` (`types.ts:294`); `PretableChangeSequence`/`changesSince` are in `core.api.md` (259/1088). +- `changesSince` consumers: renderer-dom `row-layout-controller.ts` (`validateChanges:1400` — checks `sequence.kind !== "changes"` → replacement; already fail-closed for any reset reason) and `grid-core/create-grid-ui-core.ts` (audit in B2). +- Browser attribution: 192ms sync setQuery + ~215ms height-index re-ingest; React commit 5.3ms (`scratchpad/sort-browser-attribution.md`). Node attribution: `scratchpad/sort-residual-profile.md`. + +**Conventions binding every task:** packages/* vanilla TS, no new deps. TDD: failing test first, red run shown, then green. Comments = constraints only. Mutation-harden every new load-bearing assertion (prove it can fail). Never `git checkout` in the main repo (`~/repos/pretable`); work only in this worktree. Amend-on-HEAD only after verifying HEAD. + +--- + +## Phase A — sort keys move to the plan + +### Task A1: Plan-owned sort-key store + record-based `compareRows` (dual-source transition state) + +**Files:** +- Modify: `packages/row-model/src/compiled-query.ts` +- Test: `packages/row-model/src/__tests__/sort-key-store.test.ts` (create) + +End state of this task: the plan owns a store and a record-based comparator; `metadata.sortKeys` STILL EXISTS (removed in A3) so the package stays green. The new comparator resolves through the store only. + +- [ ] **Step 1: Write failing tests** (`sort-key-store.test.ts`) + +Contracts (adapt fixture syntax from `query-delta.test.ts`): +1. `evaluate` populates the plan's store: after `plan.evaluate({rowId, row, sourceOrder})`, the new internal accessor (exported for tests as `getSortKeysForTesting(plan, row)` or via the comparator's observable behavior — prefer behavior: see test 2) resolves without running accessors again (spy-count proof, same style as cycle 1's carryover tests). +2. New `compareRecordRows(plan, left, right)` (working name; see Step 3) orders two evaluated inputs `{rowId, row, sourceOrder}` identically to the current metadata-based `compareRows` over the same fixture — table of pairs including: number column asc/desc, text collation, nulls first/last, custom comparator, sort-key tie resolving by `sourceOrder`. +3. Fail-loud: comparing a row the plan never evaluated (and whose keys were never swap-filled) throws (message naming the defect, e.g. "row has no sort keys under this plan"). +4. Store swap-fill: `fillSortKeysFromPrevious(nextPlan, previousPlan, row)` (working name) carries values for overlapping sort columns from the previous plan's store and runs accessors only for newly-active sort columns (spy proof — this relocates cycle 1's carry rule; the carry source is the PREVIOUS PLAN'S STORE, not metadata). +5. Accessor failure during swap-fill surfaces the same `accessor-failed` `PretableRowModelError` shape `evaluate` uses. + +- [ ] **Step 2: Red run** + +`pnpm --filter @pretable-internal/row-model test -- sort-key-store` → FAIL (exports missing). + +- [ ] **Step 3: Implement in compiled-query.ts** + +- Private field on `CompiledQueryPlan`: `#sortKeys = new WeakMap[]>()`. +- `evaluate`: where `sortKeys` is built today (inside `#finalizeMetadata`), also `this.#sortKeys.set(input.row, sortKeys)`. (Metadata keeps carrying them until A3.) +- Static + free-function pattern (as in cycle 1) for: + - `compareRecordRows(plan, left: {rowId; row; sourceOrder}, right)` — resolves both sides' keys from `plan.#sortKeys` (throw with a defect-naming message on miss), then runs the existing per-ordering comparison loop (`compareValues`) ending in the `sourceOrder` tiebreak. Refactor the existing `compareRows` body so both comparators share ONE loop implementation (parameterized on a key-lookup function) — do not fork the comparison semantics. + - `fillSortKeysFromPrevious(nextPlan, previousPlan, input: {rowId; row; sourceOrder})` — for each entry of nextPlan's runtime sort: carry from `previousPlan.#sortKeys.get(row)` by columnId where present, else run the accessor (wrapped in the accessor-failed error shape); freeze and store into `nextPlan.#sortKeys`; return the keys. Statics can read both instances' privates. +- Do NOT change `compareRows`, metadata, or any consumer yet. No index.ts edits. + +- [ ] **Step 4: Green + full suite** + +Package suite green (376 + new). Commit: +```bash +git add packages/row-model/src/compiled-query.ts packages/row-model/src/__tests__/sort-key-store.test.ts +git commit -m "feat(row-model): plan-owned sort-key store and record-based comparator" +``` + +--- + +### Task A2: Reroute the four consumers to the store + +**Files:** +- Modify: `packages/row-model/src/visible-index.ts`, `cooperative-transition.ts`, `transaction-draft.ts`, `group-index.ts`, `sort-rebuild.ts` +- Test: existing suites (behavior-neutral task) + targeted additions in `sort-key-store.test.ts` + +Reroute every comparator/keys read to the store while `metadata.sortKeys` still exists. Behavior must be UNCHANGED — the whole existing suite is the test. This task makes A3 (shape removal) mechanical. + +- [ ] **Step 1: Reroute, one call site at a time, running the package suite after each** + +1. `visible-index.ts`: `createFlatVisibleTree` takes the PLAN (or a record comparator) instead of a metadata comparator; its tree `compare` becomes `compareRecordRows(plan, left, right)` over records (records carry rowId/row/sourceOrder directly — metadata no longer consulted). `createFlatVisibleIndex`/`createVisibleIndex` signatures adapt. Update callers (cooperative-transition.ts:404-412, sort-rebuild.ts). +2. `transaction-draft.ts:1385` (tree comparator) — same replacement. `sameFlatOrder` (~:346): replace `sameKeyValues(previous.metadata.sortKeys, next.metadata.sortKeys)` with a store-based comparison: previous keys from the PREVIOUS plan's store, next keys from the CURRENT plan's store (read the surrounding code to learn which plans are in scope there; if both records were evaluated under the same plan, both resolve from it). Preserve the exact `Object.is` value-equality semantics of `sameKeyValues`. +3. `group-index.ts:855` — record-based comparator. +4. `group-index.ts:897-921` `compareAggregateLeaves` — replace the synthesized-metadata hack with `compareRecordRows(queryPlan, {rowId: left.id, row: left.row, sourceOrder: leftDependency.sourceOrder}, ...)`. The store resolves keys by `row` object — the leaf carries `row`. (Leaves' rows were evaluated under this plan, so the store has them; if a test proves otherwise, that is a real finding — stop and report, do not lazily fill here.) +5. `sort-rebuild.ts` — comparator adaptation only (v2 rewrite is A3). + +- [ ] **Step 2: Targeted additions** + +In `sort-key-store.test.ts`: one test per rerouted site is NOT needed (the suite covers behavior); add exactly one integration test: build a model with groups + aggregates + sort, run a full setQuery cycle, assert output equals a pre-reroute oracle (hardcode the expected output from a green pre-change run — this pins that the reroute changed nothing). + +- [ ] **Step 3: Full suite green, commit** + +```bash +git add -A packages/row-model +git commit -m "refactor(row-model): resolve sort keys through the plan store everywhere" +``` + +--- + +### Task A3: Remove `sortKeys` from metadata and `dependency`; sort-rebuild v2; delete `resortRecordMetadata` + +**Files:** +- Modify: `packages/row-model/src/compiled-query.ts`, `internal-types.ts` (if metadata type lives there — find `CompiledRowMetadata`'s definition), `sort-rebuild.ts`, `group-index.ts` (dependency construction), `transaction-draft.ts` (dependency comparisons ~:401-413, :704) +- Delete: `resortRecordMetadata` + its statics/tests +- Test: `sort-fast-path.test.ts` (rewrite affected describes), `sort-key-store.test.ts` + +- [ ] **Step 1: Write the new-invariant tests FIRST** (they fail against A2 state) + +In `sort-fast-path.test.ts`, the model-level fast-path describe gains/changes to: +1. **Identity carries:** after a sort-only `setQuery`, the new root's rows map is the SAME object (`toBe`) as before; every record and every `publicRow` is `toBe`-identical; `visibleRowCount` unchanged; order changed per the fixture's expected permutation. (Get the root via the same fixture path Task-4-cycle-1 used, or assert identity through the public snapshot: `rowAt(i)` returns identity-stable rows across the change.) +2. **Stale-hazard (the crown):** sort-only fast path, THEN `setRows` updating one row's sort-key value → the row re-ranks correctly; THEN `setRows` updating a non-key field → the row does NOT move (pins `sameFlatOrder` through the store). Both assertions against hand-computed expected orders with fixture controls. +3. **Aggregate-reuse:** with an aggregate column, sort-only change leaves every `aggregateLeaves[i].dependency` `toBe`-identical (per record identity this is implied — assert it anyway) AND aggregate VALUES still correct (positive twin). +4. **Work counters:** `synchronousRebuilds === 1`; new counters `sortKeyCarries + sortKeyEvaluations === rowCount` with carries dominating when sort columns overlap, evaluations > 0 when a new column enters the sort (two scenarios). +5. Existing equivalence/error/supersede tests from cycle 1 stay, with metadata-content assertions replaced by identity assertions where metadata no longer changes. + +- [ ] **Step 2: Red run, then implement** + +- `CompiledRowMetadata`: remove `sortKeys`; `dependency` becomes `Object.freeze({sourceOrder})` (in `#finalizeMetadata`); remove the now-dead `valueOf` plumbing for sort keys from the shared helper (keys now flow to the store, not metadata — evaluate still computes them, in the same place, storing to the WeakMap only). +- `transaction-draft.ts:401-413`: dependency comparison drops the sortKeys leg (dependency is now `{sourceOrder}`; read what the comparison protects — aggregate-leaf reuse — and keep the `sourceOrder` check; sort-key changes no longer dirty leaves BY DESIGN — the aggregate-reuse test pins correctness). `:704` spread adapts. +- `group-index.ts` dependency construction sites adapt. +- Delete `resortRecordMetadata`, `CompiledQueryPlan.resortMetadata`, and their describe block. +- `sort-rebuild.ts` v2: + +```ts +// After the guards (unchanged): +const startedAt = now(); +const visible: RowRecord[] = []; +for (const entry of captured.sourceOrder.entries()) { + const previous = captured.rows.get(entry.rowId); + if (previous === undefined) continue; + fillSortKeysFromPrevious(nextPlan, captured.queryPlan, previous); + if (previous.metadata.filterPasses) visible.push(previous); +} +visible.sort( + (left, right) => + compareRecordRows(nextPlan, left, right) || + compareOrderStatisticTreeIds(left.rowId, right.rowId), +); +// tree build: unchanged except the comparator plumbing from A2 +const root = Object.freeze({ + revision, + parentRevision: revision - 1, + rows: captured.rows, // identity — the entire point + sourceOrder: captured.sourceOrder, + visible: Object.freeze({ rows: tree }), + queryPlan: nextPlan, + expansion: captured.expansion, + cause: Object.freeze({ kind: "set-query" as const }), +}); +``` + +- Diagnostics: add `work.sortKeyCarries` / `work.sortKeyEvaluations` (mirror existing counter wiring exactly); `fillSortKeysFromPrevious` bumps them when instrumentation is present (thread it or count in the rebuild loop — pick what fits; counts must distinguish carry vs accessor). + +- [ ] **Step 3: Green: fast-path tests, then FULL package suite** + +Expect fallout in tests that asserted metadata sortKeys content — each edit individually justified in the report (same discipline as cycle-1 Task 5). + +- [ ] **Step 4: Mutation-harden** + +(a) Make `fillSortKeysFromPrevious` skip the store write → fail-loud comparator test must fail. (b) Make sort-rebuild rebuild records (`{...previous}` copies) → identity test must fail. (c) Break the carry (always run accessor) → counter test must fail. Report each. + +- [ ] **Step 5: Commit** + +```bash +git add -A packages/row-model +git commit -m "feat(row-model): sort-only changes carry records and the rows map by identity" +``` + +--- + +### Task A4: Phase-A measurement gate (Node) + +No repo changes. Re-run the scratchpad `sort-decomposition.mjs` (it imports from this worktree's src via tsx) at 50k and 3k, 5 repeats. Record in `/phase-a-node-results.md`. Expectation from the profile: 50k drops from ~250ms toward ~75-110ms (sort ~35ms + tree ~39ms + key fill + iteration; GC share should collapse with allocations). If 50k is NOT under ~140ms, STOP — report before Phase B (the ceiling analysis was wrong somewhere, and B's payoff math changes). + +--- + +## Phase B — reorder signal and permutation layout + +### Task B1: Journal reset reason `"reorder"` + +**Files:** +- Modify: `packages/row-model/src/types.ts` (~:294 reason union), `change-journal.ts`, `create-local-row-model.ts` (fast-path publish), `sort-rebuild` publish call site if the reason flows through `publishCommittedRoot` +- Test: the journal's existing test file (find it: grep `appendBarrier` in `__tests__`) + `sort-fast-path.test.ts` + +- [ ] **Step 1: Failing tests** + +1. Journal-level: `appendBarrier(prev, rev, "reorder")` then `changesSince(prev, rev)` → `{kind: "reset", reason: "reorder", ...}` (whatever shape reset carries today — read it first). +2. **Mixed-range conservatism:** reorder barrier + a changes entry in range → reason is NOT "reorder" (falls back to the standard barrier/reset behavior — read what a barrier-in-range returns today, likely `"bulk-replace"`, and pin that). Two reorder barriers in range → still "reorder". Reorder barrier + non-reorder barrier → NOT "reorder". +3. Model-level: sort-only fast `setQuery` → `model.changesSince(prevRevision)` reports reset reason "reorder"; a filter `setQuery` (cooperative) still reports the pre-existing reason; `setRows` after a fast sort → mixed → NOT "reorder". + +- [ ] **Step 2: Implement** + +- `types.ts`: reason union gains `"reorder"`. +- `change-journal.ts`: barrier entries already carry a reason — `changesSince` must aggregate: when the range contains barriers, the returned reset reason is `"reorder"` iff EVERY entry in range is a barrier with reason "reorder"; otherwise today's behavior. Read `changesSince`'s current barrier handling (~:292) before writing. +- Fast path publish: `publishCommittedRoot` gains a reason parameter? NO — keep the shared recipe intact: the fast path is the only reorder producer; pass the reason through `publishCommittedRoot(committedRoot, previousRevision, revision, reason = "bulk-replace")` defaulting to today's value so `runTransitionSlice` is unchanged... READ what reason `appendBarrier` currently defaults to and preserve it for every existing caller; only the sort-only fast path passes "reorder". + +- [ ] **Step 3: Green + full row-model suite + commit** + +```bash +git add -A packages/row-model +git commit -m "feat(row-model): sort-only commits publish a reorder reset reason" +``` + +--- + +### Task B2: Public-surface + consumer audit + +**Files:** possibly `packages/core/*.api.md` (regenerated, committed), `packages/grid-core/src/create-grid-ui-core.ts` (audit; change only if it would MISPARSE), apps/website docs guards. + +- [ ] **Step 1:** `pnpm build && pnpm api` (order matters). `core.api.md` will change (reason union). Commit the regenerated report(s). If anything BEYOND the reason union changed, stop and investigate (leak). +- [ ] **Step 2:** Audit `grid-core/create-grid-ui-core.ts`'s `changesSince` handling: confirm it treats ANY `kind: "reset"` as full-resync regardless of reason (fail-closed). Add a test pinning that a "reorder" reset is handled as reset (not dropped/misparsed) IF no such test exists. Do not add reorder-awareness to grid-core in this project. +- [ ] **Step 3:** Run the website test suite (docs guards pin api.md-derived union tables — memory: guards fail closed). If a docs table lists the reset reasons union, update the table per the guard's instructions (the guard's failure message documents the registration flow). Run `pnpm test` repo-wide. +- [ ] **Step 4:** Commit. + +```bash +git add -A +git commit -m "chore(core,docs): reorder reset reason through the public surface and guards" +``` + +--- + +### Task B3: `RowHeightIndex.reorder()` + +**Files:** +- Modify: `packages/layout-core/src/row-height-index.ts` +- Test: layout-core's height-index test file (find it) + +- [ ] **Step 1: Failing tests** + +1. **Equivalence oracle:** build an index with N measured rows (mixed measured/estimated entries — read how tests build one today), permute the order, call `reorder({rowCount, entryAt})`; assert the rank→offset table (every rank's offset + total height) equals a full `replace()` over the same order. Include: reversal, single swap, identity permutation. +2. **Reuse counters:** entries reused === N; entries re-measured === 0 (add counters mirroring layout-core's existing instrumentation pattern — find how `beginReplacement` counts and mirror). +3. **Missing key throws:** a key in the new order absent from existing entries → throws (typed error per the file's error conventions). +4. **Row-count mismatch throws** (fewer/more rows than entries — the reorder contract says the SET is unchanged). +5. Post-reorder mutations work: a subsequent measurement update / replacement behaves normally. + +- [ ] **Step 2: Implement** + +Synchronous method on the index (same class/immutability discipline as `replace` — read whether `replace` returns a new index; `reorder` must match): walk the new order, look up each existing entry by key, rebuild the ordered structure + prefix sums in one pass (reuse the bulk machinery `replace` uses under the hood where it fits — the deferred-measure tree pattern exists in this codebase family; read what `row-height-index` uses internally before choosing). Constraint comment: why synchronous (order-only, heights known, ~10-20ms at 50k measured need). + +- [ ] **Step 3: Mutation-harden** (skip a prefix-sum recompute → oracle fails; fabricate a missing-key entry instead of throwing → test 3 fails). Green, commit. + +```bash +git add packages/layout-core +git commit -m "feat(layout-core): synchronous reorder over existing height entries" +``` + +--- + +### Task B4: Controller permutation path + +**Files:** +- Modify: `packages/renderer-dom/src/row-layout-controller.ts` +- Test: renderer-dom's controller test file (find how it mocks model + journal today; follow those harnesses) + +- [ ] **Step 1: Failing tests** + +1. A model publishing a reset with reason "reorder" (mock `changesSince`) → controller does NOT `startReplacement` (assert via its observable: replacementStartCount or the published state sequence) and publishes a ready root whose rank→offset table matches a full replacement oracle. +2. Anchor semantics: same scroll-anchor observable as `startReplacement` produces for the same scenario (read the existing anchor tests and mirror the strongest one). +3. Fallbacks, each one a test: reason ≠ reorder → replacement; `reorder()` throws (inject via a key mismatch) → replacement, no error published to the consumer beyond what replacement produces; revision mismatch → replacement. +4. Counters: reorder path taken count; fallback count (wire into the controller's existing diagnostics pattern if one exists; else layout-core's counters suffice — decide from the code and say which in the report). + +- [ ] **Step 2: Implement** + +In `synchronize` (~:1517), before the `validateChanges` branch: if the sequence is `{kind:"reset", reason:"reorder"}` AND revisions line up (`fromRevision === state.observedRevision`, `toRevision === target.revision` — read reset's actual field names), take the permutation path: capture anchor → `state.rowHeights.reorder(...)` with the same `{rowCount, entryAt}` source shape `startReplacement` builds (share that source-construction code — extract it if needed) → restore anchor semantics identically to the replacement path → `publishReady`. Wrap in try/catch → `startReplacement(target, true)` on ANY throw. + +- [ ] **Step 3: Green (renderer-dom suite), mutation-harden the fallback (break the revision check → fallback test fails), commit.** + +```bash +git add packages/renderer-dom +git commit -m "feat(renderer-dom): sort-only commits permute row heights instead of re-ingesting" +``` + +--- + +### Task B5: End-to-end verification + +1. Full repo: `pnpm build && pnpm typecheck && pnpm lint && pnpm test && pnpm api` (react-suite flake rule applies: re-run once). +2. Browser A/B per the one-variable protocol (same method as the cycle-1 Task 7 run, recorded in `/sort-fast-path-perf-results.md`): merge-base vs branch, both scales, pretable+tanstack, repeats 3; plus filter-metadata collateral check; plus the grouped gate. +3. Work-based assertions from the bench trace: one PLAYWRIGHT_PERF_TRACE run at 50k; confirm via `analyze-cdp.mjs --window=interaction` that the post-publish re-ingest slices are GONE (no layout-core ingest frames in the window) and the interaction window shrinks accordingly. +4. Write `/cycle-2-results.md` with all four success-criteria verdicts. STOP after recording — the merge decision returns to the user with these numbers (the PR is being held; do not open or merge it inside this task). + +--- + +## Self-review notes (applied) + +- Spec coverage: store+comparator → A1; reroutes → A2; shape removal + v2 + deletion + counters → A3; Node gate → A4; journal → B1; public surface + fail-closed audits → B2; layout reorder → B3; controller → B4; final gate → B5. Reserve lever and out-of-scope items have no tasks by design. +- Names used consistently: `compareRecordRows`, `fillSortKeysFromPrevious`, `work.sortKeyCarries`, `work.sortKeyEvaluations`, reset reason `"reorder"`, `RowHeightIndex.reorder`. (A1's "working name" labels mean the implementer may improve a name ONLY by reporting it; all later tasks then follow the reported name.) +- Known looseness, deliberate: exact fixture syntax and internal helper shapes are verified against reality at implementation time (this repo's pattern; assertions are the contract). Two decision points are delegated with explicit stop conditions: A4's ceiling check and B2's leak check. diff --git a/docs/superpowers/specs/2026-08-17-sort-fast-path-design.md b/docs/superpowers/specs/2026-08-17-sort-fast-path-design.md new file mode 100644 index 000000000..4a02f3866 --- /dev/null +++ b/docs/superpowers/specs/2026-08-17-sort-fast-path-design.md @@ -0,0 +1,191 @@ +# Sort fast path: query-delta classification + synchronous rebuild + +**Issue:** [#457](https://github.com/cacheplane/pretable/issues/457) — S2 sort at target scale (50k rows) never settles: 515ms of transition work against a 400ms window. Related: [#452](https://github.com/cacheplane/pretable/issues/452) (~2x interaction gap at 3k). + +**Date:** 2026-08-17 + +## Problem + +A sort change runs the full cooperative transition: every row gets a fresh +`queryPlan.evaluate` (all active accessors, all filters, cold per-plan cache), +a record freeze, a HAMT insert, and an AVL insert — ~10µs/row, ~515ms at 50k +(`cooperative-transition.ts:556`, `compiled-query.ts:1373`). TanStack sorts the +same rows in ~33ms with `slice().sort()` over cached values. + +Constraints established by prior measurement (issue #457 comment, full A/B): + +- **Lever 1 (slice budget) is refuted in-browser.** The browser inverts the + Node curve: larger budgets improved Node wall time ~2x and worsened + `interaction_latency_ms` by 1–2 frames. No scheduling change ships. +- **The Node harness decomposes work; it does not predict browser latency.** + Latency claims are made only from the browser bench. +- The fix must come from **work reduction**. + +## Decisions (made during brainstorming) + +- **Lever 4 (progress publishes) is deferred.** If work reduction lands, the + frozen-surface window shrinks below the threshold where it matters. Revisit + only if the measured result still breaches the settle window. +- **General delta classification, sort implemented first.** The classifier is + built to describe any query change; only the sort-change fast path ships in + this project. Filter-change reuse is a designed-for follow-up. +- **The fast path is fully synchronous.** It completes atomically inside + `setQuery`: no candidate, no slices, no mid-transition delta replay. + Accepted trade: no jank bound on pathological datasets (huge row counts or + expensive custom comparators block the main thread for the duration). +- **Approach A — prior-metadata carryover.** Reuse is a one-shot transfer from + the old root's records into the new ones. No long-lived cross-plan value + cache, no invalidation surface. + +## Scope + +In scope: + +1. A **query-delta classifier** in `packages/row-model`. +2. A **synchronous sort-change rebuild path** in `create-local-row-model.ts`, + for **ungrouped** queries only. +3. Instrumentation and bench-gate accounting for the new path. + +Out of scope (explicitly): + +- Progress publishes / partial sorts (deferred, see above). +- Filter-change fast path (classifier supports describing it; implementation + is a follow-up). +- Grouped queries (stay on the cooperative path; eligible for carryover in a + later project). +- Any scheduling/budget change (refuted). + +## Success criteria + +Measured in the **browser** bench (`bench:matrix`), like-for-like, one +variable, dist rebuilt between variants: + +1. S2 sort at target scale (50k): status `completed ×3` (currently + `partial ×3`), and `interaction_latency_ms` within **~2x of TanStack's** + measured in the same run. +2. No regression at hypothesis scale (3k): `interaction_latency_ms` at or + below the current 50–59ms band. +3. Grouped rebuild gate (`rebuild_slice_max_ms`) untouched and green. +4. Node decomposition rerun for the work accounting (informational, not the + latency claim). + +## Design + +### Unit 1: query-delta classifier + +A pure function over the old and new compiled plans: + +``` +classifyQueryDelta(oldPlan, newPlan) -> { + derivationsChanged: boolean + filtersChanged: boolean + groupsChanged: boolean + sortChanged: boolean + filterAuthorityChanged: boolean +} +``` + +plus a derived predicate `isSortOnlyChange(delta)` requiring: derivations +identical (reusing `derivationsEqualForPlan`), filters identical, rowGroups +identical, filter authority identical, sort authority identical (#467, which +landed after this spec was drafted, added `CompiledSortAuthority`; under +external authority the runtime sort is `[]`, so the classifier compares +**runtime** facets and a sort change under external authority classifies as +no runtime sort change — it stays on today's path), and sort different. The +caller additionally requires operation `set-query`. + +**Conservatism rule:** any comparison the classifier cannot decide structurally +classifies as _changed_. The slow path is always correct; the classifier can +only cause missed optimizations, never wrong results. This is the property the +tests pin. + +### Unit 2: synchronous rebuild path + +In `setQuery`, when `isSortOnlyChange` holds **and** the query is ungrouped +(`rowGroups.length === 0`, both plans): + +1. **Carryover, per old record:** + - `filterPasses` and `groupPath` carried verbatim (filters and groups are + unchanged by precondition; `groupPath` is `[]` for ungrouped). + - Sort-key **values** sourced from the old metadata where the column was + already active under the old plan; accessors run only for newly-active + sort columns. Accessor failures surface the same + `PretableRowModelError("accessor-failed", …)` as the slow path. + - New `sortKeys`, `dependency`, and `aggregateLeaves` are constructed + around the carried values (they embed the dependency object, which + changes with the sort, so the objects are rebuilt; the _values_ are not + recomputed). + - New record frozen with the new metadata. +2. **Sort:** `Array.sort` over the filter-passing records using the new plan's + `compareRows`. Comparator errors (custom comparator returning NaN/non-number) + surface exactly as on the slow path. +3. **Bulk build:** visible tree via the existing deferred-measure transient + (`createDeferredMeasureTransientOrderStatisticTree`, + `order-statistic-tree.ts:918`); rows map via its transient + (`asTransient()`/`freeze()`). +4. **Publish:** one new revision root, one emission, synchronously inside + `setQuery`. Cause kind `set-query`, same shape as a cooperative finish. + +**In-flight transitions:** an active cooperative transition is superseded +exactly as an ordinary overlapping `setQuery` supersedes it today (release the +candidate, capture the current published root). The fast path adds no new +lifecycle states. + +**Streaming:** because the path completes atomically, `setRows` arriving after +it applies to the already-published root through the normal incremental path. +There is no window in which deltas need replaying. + +### Instrumentation and gates + +- The fast path reports its wall time under a new instrumentation field (e.g. + `work.synchronousRebuildMs`), **not** as a scheduler slice. + `rebuild_slice_max_ms` keeps its meaning (cooperative slices only). +- The bench gate configuration gets an explicit note: the flat sort fast path + is exempt from the slice bound **by design** (see the synchronous-burst + decision above). +- The existing candidate diagnostics are untouched; the fast path never + constructs a candidate. + +## Error handling + +- Accessor failure on a newly-active sort column: `PretableRowModelError` + with the same code, operation, rowId, and columnId as the slow path would + produce. The model's state is unchanged on throw (the new root is built + fully before publish). +- Custom comparator misbehavior: same `TypeError` as `compareValues` throws + today, thrown during the synchronous sort, state unchanged on throw. +- Classifier uncertainty is not an error — it routes to the slow path. + +## Testing + +TDD throughout (test first, watch it fail, implement). + +1. **Classifier:** table-driven cases per facet (sort added/removed/reordered/ + direction-flipped; each other facet changed alone → not sort-only; + authority flip → not sort-only). Conservative-default cases pinned. +2. **Equivalence:** for the same inputs, the fast-path root is observably + identical to the slow-path root — visible order, per-row metadata contents + (sortKeys, filterPasses, aggregate leaf values), revision/cause shape. + Cover: sort → different sort, sort → unsorted (`[]`, source order), + unsorted → sort, newly-active sort column, multi-column sort, custom + comparator, active filters present (unchanged), aggregates present. +3. **Old behavior survives:** delete-the-feature mutation — with the fast path + forced off, the suite still passes; with it on, sorting still _sorts_ + (assert row order, not merely that the path ran). Fixture data chosen so a + wrong order is distinguishable from the right one. +4. **Errors:** accessor-failure and comparator-failure tests on the fast path, + asserting state is unchanged after the throw. +5. **Streaming/supersede:** `setQuery` (fast) during an active cooperative + transition; `setRows` immediately after a fast `setQuery`. +6. **Verification protocol:** Node decomposition rerun (work accounting); + browser A/B per the one-variable protocol — rebuild react `dist` between + variants, same machine window, n≥5, pretable and TanStack from the same + run. The latency claim is made only from this measurement. + +## Follow-ups (filed, not built) + +- Filter-change fast path on top of the classifier (values survive, verdicts + re-run). +- Grouped-query carryover (the grouped bulk builder exists; reuse would cut + its evaluate share). +- Progress publishes, only if target-scale results still breach the window. diff --git a/docs/superpowers/specs/2026-08-18-decorated-entries-and-bulk-mount-design.md b/docs/superpowers/specs/2026-08-18-decorated-entries-and-bulk-mount-design.md new file mode 100644 index 000000000..c166fef2e --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-decorated-entries-and-bulk-mount-design.md @@ -0,0 +1,153 @@ +# Decorated entries + bulk mount: closing the last two #457 verdicts + +**Issue:** [#457](https://github.com/cacheplane/pretable/issues/457) — third cycle. +**Builds on:** cycles 1–2 on this branch (fast path, identity carry, reorder +signal, permutation layout path). Cycle-2 verdicts (`cycle-2-results.md`): +flat sort transformed (8.4ms when the reorder path engages) but bar 1 still +fails because the benched sort races the still-running mount ingest, and +bar 3 (grouped gate) regressed 7.7 → 8.2–9.2ms from per-comparison WeakMap +resolution on the grouped path. + +**Date:** 2026-08-18 + +## Problem, precisely + +1. **Grouped gate (C1):** A2 rerouted grouped comparators through + `compareRecordRows`, which resolves both sides' sort keys from the plan's + WeakMap **per comparison**. The grouped cooperative rebuild is incremental + (`insert` per row per unit; no bulk sort to decorate), ~4M comparisons × + 2 gets ≈ the measured +11% slice-work delta + (`grouped-gate-regression-findings.md`). The flat visible tree pays the + same tax on incremental inserts (ungated; filter-metadata at 58.3ms = + top of band). The zero-allocation memoized-comparator treatment was + implemented and measured **neutral** — the gets themselves are the cost. +2. **Sort-during-mount (C2):** at 50k the initial replacement ingests + cooperatively for ~450ms during which **nothing paints** (`window: []`; + the eager gate covers ≤32 rows), and a sort arriving mid-ingest + fail-closes into a full re-ingest (the benched 342–350ms). The + lifecycle map (exploration 2026-08-18) established: at initial mount + there are no retained measurements, so the builder's 450ms produces an + index whose every entry is `defaultHeight` — trivially computable; B3 + measured an O(n) balanced bulk build of 50k entries at 15–20ms. The + builder cannot produce partial indexes (no root until its final phase), + so partial/refinement publishing was rejected as the wrong tool + (decision: rescope approved 2026-08-18). + +## Success criteria + +1. Grouped gate: `rebuild_slice_max_ms ≤ 8` across 3 quiet-machine runs, + at or near the cycle-1 7.7ms. +2. 50k S2 sort (bench, same protocol as cycle 2): `completed ×3` AND + `interaction_latency_ms` within ~2x same-run TanStack — now genuinely + reachable: mount builds in ~20ms, so the benched sort hits the reorder + path (measured 8.4ms when engaged). +3. Mount improvement (new, work-based): time-to-first-published-window at + 50k mount drops from ~450ms to <50ms (assert via bench mount metrics or + the controller diagnostics; exact observable chosen at implementation). +4. No regressions: 3k band, filter-metadata band, full repo suites, api + reports unchanged beyond intended. + +## Workstream C1 — decorated entries (grouped + flat) + +Tree entries carry their resolved sort keys; comparisons become property +reads. Zero WeakMap gets in any O(n log n) or per-insert comparison path. + +- **Entry shape:** the flat visible tree, grouped leaf trees, and aggregate + trees store `{ record, keys }` (aggregate leaves: `{ leaf, keys }` or keys + alongside the existing leaf shape — implementer picks the minimal-ripple + encoding per tree, one convention across all three). +- **Key source at insert:** the inserting code resolves once via + `sortKeysOf(plan, record)` (one get per insert) or passes keys it already + holds (sort-rebuild's decorated pairs feed the tree directly — unifying + with the cycle-2 decorated sort). +- **Comparators:** `compareWithSortKeys(plan, l.record, l.keys, r.record, +r.keys)` everywhere a tree comparator runs. `compareRecordRows` remains + for one-shot comparisons only. +- **Invariants preserved:** a tree is bound to one plan (the A2 + rebuild-or-reseed invariant); entry keys are valid for the tree's + lifetime. Entry replacement on row update replaces the keys with it (the + update path already re-evaluates the row under the current plan). +- **Ripple:** consumers that read tree entries (visible-index snapshot, + group-index, cooperative-transition, transaction-draft, sort-rebuild) + adapt from `entry` to `entry.record`. All internal to row-model. + +## Workstream C2 — bulk mount + compose-at-finish + +### C2a: synchronous bulk replacement when nothing is retained + +- `RowHeightIndex` (layout-core) gains a cheap predicate (e.g. + `hasRetainedState`: any measurements, tombstones, or retained entries) and + a synchronous bulk path: when the base index has NO retained state, a + replacement over `{rowCount, entryAt}` builds the balanced sequence + + identity map in one O(n) pass (reuse B3's `buildBalancedSequence` + + bulk-map machinery). Exposed as either a fast path inside + `beginReplacement` (builder completes on first `advance`) or a distinct + synchronous method — implementer picks what fits the class; the + controller-visible contract is "initial mounts complete in one slice". +- Controller: `startReplacement` keeps its current shape; the eager gate + (`eagerInitialRowLimit`) is superseded for the no-retained-state case — + when the predicate holds, run the replacement to completion synchronously + (same accepted synchronous-burst trade as cycles 1–2; the bench gate + note already covers the philosophy). Retained-state replacements keep + cooperative slicing unchanged. +- The blank-mount fix follows: first `publishReady` lands ~20ms after + activation at 50k. + +### C2b: compose a reorder into an active replacement at finish + +- `captureActiveTarget`: a reset with reason `"reorder"` whose + `toRevision` matches the incoming target revision is ACCEPTED as a + retarget (today any reset → restart): swap `latestTarget`, advance + `capturedRevision`, set a `pendingReorder` flag on the replacement. No + builder restart. +- **Composition rule (conservative):** a reorder retarget is accepted only + as the FINAL retarget — if any further wake arrives after + `pendingReorder` is set (changes or another reset of any reason except a + newer reorder, which simply replaces the flag's target), fail closed to + restart. This avoids reasoning about index-based pending operations + applied across a permutation. (A newer reorder replacing an older one is + safe: reorders are wholesale.) +- `finishReplacement`: after catch-up drains and staged measurements + replay, if `pendingReorder` is set, `candidate = +candidate.reorder(replacementSourceOf(latestTarget))` before publish. + Any throw → the existing fallback (restart on latestTarget). +- Counters: extend the B4 diagnostics (`reorderComposeCount`, + `reorderComposeFallbackCount`) on the existing seam. + +## Out of scope + +- Partial/refinement publishing (rejected for this problem; a future issue + may revisit for filter-rebuild latency). +- Any scheduling/budget change (still refuted). +- The flat O(n) order container (reserve lever, still unpulled). + +## Testing + +House standard: TDD, mutation-hardened, positive/negative twins. + +C1: grouped gate re-measured (3 quiet runs, the success bar); grouped + +flat equivalence suites stay green with entries decorated (the A2 pin test +and A3 grouped tests are the oracles); a Node grouped-rebuild timing +comparison (existing `grouped-rebuild-timing.mjs`) showing the slice-work +delta recovered to ~pre-A2 levels — recorded, not unit-asserted. + +C2a: layout-core — bulk path equivalence oracle vs cooperative build over +identical sources (every rank), predicate tests (any retained measurement/ +tombstone disables it), post-bulk mutations behave; controller — 50k-scale +mount publishes on the first synchronize pass (no scheduler entries), +existing replacement tests untouched for the retained-state path. + +C2b: controller — reorder mid-replacement composes (no restart, counters), +final order equals a from-scratch oracle; reorder followed by changes → +fail-closed restart; reorder replacing reorder → last wins; staged +measurements survive composition; anchor semantics preserved. + +End-to-end: the cycle-2 B5 protocol re-run (browser A/B, all four bars, +trace attribution confirming the measured sort takes the reorder or +compose path — `reorderFallbackCount === 0` in the benched runs). + +## Sequencing + +C1 first (unblocks the failing gate; independently verifiable in Node), +then C2a, then C2b, then the end-to-end re-verification. The PR stays held +until all bars are evaluated together. diff --git a/docs/superpowers/specs/2026-08-18-sort-reuse-and-reorder-signal-design.md b/docs/superpowers/specs/2026-08-18-sort-reuse-and-reorder-signal-design.md new file mode 100644 index 000000000..68b5ffa48 --- /dev/null +++ b/docs/superpowers/specs/2026-08-18-sort-reuse-and-reorder-signal-design.md @@ -0,0 +1,226 @@ +# Sort-key ownership + reorder signal: end-to-end sort latency + +**Issue:** [#457](https://github.com/cacheplane/pretable/issues/457) — second cycle. +**Builds on:** `docs/superpowers/specs/2026-08-17-sort-fast-path-design.md` (cycle 1, +implemented on this branch: classifier, synchronous rebuild, setQuery wiring). + +**Date:** 2026-08-18 + +## Problem + +Cycle 1 made the 50k S2 sort complete and settle (`completed ×3`, settle +16–25ms, previously never) but landed at ~390–409ms interaction latency vs +TanStack's ~37ms — ~9.5x against a ~2x bar. Two profiles attribute the +residual precisely: + +- **Node CPU profile** (`sort-residual-profile.md`): of ~250ms per 50k + setQuery, ordering work (Array.sort + comparator) is only ~35ms — already + TanStack-parity. ~86% is rebuilding immutable per-row state a sort does not + logically change: new metadata + frozen records (~46ms carryover + freeze), + a full new rows HAMT (~30ms + GC share), the visible tree (~39ms), + sourceOrder traversal (~22ms), ~12% GC. +- **Browser trace** (`sort-browser-attribution.md`, interaction window): + 192ms synchronous setQuery task + **~215ms of layout-core row-height-index + re-ingest** of all 50k rows (identity hash + HAMT insert + frozen rowRef + per row, in 8ms cooperative slices) before the first changed frame + presents. React render/commit is 5.3ms — innocent. The re-ingest happens + because every query commit publishes a change-journal **barrier**, and the + row-layout controller answers any barrier with a full replacement. + +Two independent ~200ms levers. Both are in scope. + +## Success criteria + +Measured in-browser (`bench:matrix`), one variable, dist rebuilt per side: + +1. S2 sort, target scale (50k): `completed ×3`, `interaction_latency_ms` + within **~2x of same-run TanStack**. +2. S2 sort, hypothesis scale (3k): at or below the current 50–59ms band. +3. Grouped gate (`rebuild_slice_max_ms`, updates-grouped) green. +4. Work-based assertions (these hold even if the wall-clock bar wiggles): + a sort-only change preserves the rows-map root by identity, and the + layout controller's reorder path reports **zero** rows re-measured and + zero identity re-ingests. + +**Reserve lever** (named, not implemented): if measurements land at the top +of the ceiling range, replace the post-sort AVL visible tree with a flat +O(n) order container (~35ms floor). Only reached for if bars 1–3 miss. + +## Decisions + +- End-to-end bar; whole stack in scope (row model + journal + layout-core + + renderer-dom). +- One spec, two phases: **A** (row model) then **B** (renderer). A is + independently measurable in Node before B starts. +- The held PR accumulates these commits; nothing merges until the whole bar + is evaluated. +- `resortRecordMetadata` (cycle 1) is **deleted**, tests included — the + sort-key store replaces carryover. One mechanism, no deprecation aliases + (pre-1.0, no external consumers). + +## Workstream A — sort keys move from records to the plan + +### Ownership + +`CompiledRowMetadata` loses `sortKeys`. The aggregate leaf `dependency` +shrinks to `{ sourceOrder }`. Each `CompiledQueryPlan` owns a **sort-key +store**: `WeakMap`. + +- **Eager fill:** `evaluate` writes the row's keys into the store at the + point it computes them today — same work, different home. (Rows are + replaced by object identity on update, so the WeakMap invalidates + naturally; a stale-keys-after-update bug is structurally impossible.) +- **Swap fill:** on a sort-only plan change, the new plan's store is filled + once per row during the rebuild pass — value carried from the OLD plan's + store where the column sets overlap, accessor run only for newly-active + sort columns (same carry rule as cycle 1, relocated). +- **Resolution rule:** all internal consumers obtain keys via the plan, never + from metadata. A missing store entry on a live row is a defect, not a + lazy-fill opportunity — resolution throws in that case (fail loud; the + fill points above are exhaustive). + +### Consumer reroutes (the complete list — audited 2026-08-18) + +1. `compareRows` (compiled-query.ts): reads the plan's own store. Signature + consequence: it can no longer compare bare metadata from two different + plans; its inputs become row records (or rowId+row), and the tree + comparator in `visible-index.ts` adapts. +2. `transaction-draft.ts:348` — the moved-row check (`sameKeyValues` on old + vs new metadata sortKeys) resolves old keys from the OLD plan's store and + new keys from the current plan's store. +3. `group-index.ts:897-921` — aggregate-leaf ordering reads dependency + sortKeys today; reroutes to plan resolution. +4. `persistent/aggregate-tree.ts:480` — dependency identity check. With + sortKeys gone from the dependency, a sort-only change leaves every + dependency identical, so aggregate subtrees **reuse instead of rebuild** + — a correctness-preserving improvement (aggregation is order-independent; + the invariant tests below pin it). + +No consumer outside `packages/row-model` reads `sortKeys` or `dependency` +(grep-audited; grid-core/renderer-dom matches are unrelated prose). + +### sort-rebuild v2 + +On a sort-only change (same classifier gate as cycle 1): + +- **No record rebuild. No rows transient. No metadata construction.** The + new root's `rows` IS the captured root's `rows` (identity). Records, + `publicRow`, `integrity`, metadata all carry. +- Fill the new plan's store (carry-or-accessor, once per row, during the + same pass that collects filter-passing records). +- `Array.sort` the record refs with the new plan's comparator; bulk-build + the visible tree (`createOrderStatisticTreeFromSortedEntries`, unchanged). +- Publish through `publishCommittedRoot` as today, except the journal entry + (workstream B). + +Error semantics unchanged from cycle 1 (accessor failure → same error +status shape; state untouched on throw). + +## Workstream B — order-only journal signal + permutation layout path + +### Journal + +New entry kind `reorder`: `{ kind: "reorder", fromRevision, toRevision }`, +asserting the visible row SET and every row identity/height-relevant fact +is unchanged — only order. Published by the sort-only fast path instead of +`appendBarrier`. Every other commit path is untouched (barriers remain +barriers). + +`changesSince` surfaces reorder entries to consumers. Consumers that do not +understand them treat them exactly like barriers (fail-closed: the sequence +validator in any consumer that predates the kind must classify unknown +kinds as "cannot enumerate" — verify this is already true of the row-layout +controller's `validateChanges` and any other journal consumer; if a +consumer would misparse rather than reject, that is a blocking finding). + +### Layout-core + +`RowHeightIndex` gains `reorder(source)` (same `{rowCount, entryAt}` source +shape as `beginReplacement`): rebuilds the ordered structure **from existing +measurement entries by key** — no re-measure, no identity re-hash for keys +already known, recomputed prefix sums only. Synchronous (the whole point; +~10–20ms at 50k). A key present in the new order but missing from the +existing entries (should be impossible under the reorder contract) throws — +the controller catches and falls back to full replacement. + +### Row-layout controller + +In `synchronize`, when `changesSince(observedRevision)` yields a sequence +that is exclusively reorder entries (plus the revision bookkeeping), +take the permutation path instead of `startReplacement`: + +- `rowHeights.reorder(...)` with the new snapshot's order. +- Anchor/scroll restoration IDENTICAL in semantics to `startReplacement`'s + (capture anchor, restore against the new order) — a sort change today + re-anchors; the permutation path must not change that UX. +- `publishReady` with the rebuilt root. +- ANY doubt — mixed sequence, validation failure, reorder() throw, + revision gap — falls back to `startReplacement`. Conservative default, + same philosophy as the cycle-1 classifier. + +### Identity dependency between A and B + +The controller's `entryAt` keys rows via `rowRef(target.rowAt(index))`. +Workstream A preserves `publicRow` identity across a sort-only change, +which is what makes "existing entry by key" lookups exact. B therefore +lands after A and its tests assert the identity chain explicitly. + +## Instrumentation + +- Row-model: `synchronousRebuilds`/`synchronousRebuildMs` keep their + meaning; add `work.sortKeyCarries` / `work.sortKeyEvaluations` (carry vs + accessor counts — the swap-fill efficiency is observable). +- Layout-core/renderer: reorder-path counters — entries reused, entries + re-measured (expected 0), reorder fallbacks (expected 0 on the happy + path; a nonzero fallback count in the bench is a finding). + +## Testing + +TDD throughout; mutation-hardened per house standard (every new assertion +demonstrated to fail under a seeded defect before it ships). + +**Workstream A:** + +1. The stale-hazard test (the trap that made cycle 1 rebuild records): a + `setRows` update AFTER a sort-only fast path re-ranks the updated row + correctly, and a non-key update does NOT move it (moved-row check + resolves through the store, both plans' stores agree where they must). +2. Cold-model equivalence extended beyond cycle 1's: visible order, + aggregates (grouped and ungrouped), group ordering, distinct values — + after sort-only change vs fresh model, AND after sort-only change + followed by mutations. +3. Identity: rows-map root `toBe` across sort-only change; every + `publicRow` `toBe`; selection-bearing consumers unaffected (row-model + level: record identity is the proxy). +4. Aggregate-reuse improvement pinned: dependency identity stable across + sort-only change (and the aggregate values still correct — assert the + positive twin, not just the reuse). +5. Grouped path still correct with rerouted key resolution (grouped sort + equivalence vs cold model — grouped does NOT take the fast path but DOES + use the rerouted consumers). +6. Delete `resortRecordMetadata` + its describe block; cycle-1 fast-path + tests updated to the new invariants (identity assertions replace + metadata-content assertions where metadata no longer changes). + +**Workstream B:** 7. Journal: sort-only commit emits `reorder`, not a barrier; every other +commit kind unchanged; an unknown-kind consumer rejects (fail-closed +check). 8. Layout: `reorder()` produces a rank→offset table identical to a full +replacement over the same order (equivalence oracle); reuse counter = +row count, re-measure counter = 0; missing-key throw → controller +fallback engages (fault-injection test). 9. Controller: permutation path preserves anchor semantics (same +scroll-restoration observable as `startReplacement` for the same +scenario); mixed/invalid sequences fall back. 10. e2e (website or bench-level): sorted 50k grid's first changed frame no +longer waits for ingest — assert via the bench's frames-to-first-change +or the reorder counters, not wall-clock alone. + +**Final gate:** browser A/B per the one-variable protocol (merge-base vs +branch, rebuild between sides, same-run TanStack), all four success +criteria evaluated, numbers recorded before any merge decision. + +## Out of scope + +- Publish-early/refine-later layout (would help filter latency; separate + issue if pursued). +- Filter-change fast path (unchanged from cycle 1's deferral). +- The flat O(n) order container (reserve lever; only on a missed bar). +- Progress publishes (still deferred). diff --git a/packages/core/core.api.md b/packages/core/core.api.md index 629e8e8bb..7d0adb164 100644 --- a/packages/core/core.api.md +++ b/packages/core/core.api.md @@ -264,7 +264,7 @@ export type PretableChangeSequence = { } | { readonly kind: "reset"; readonly toRevision: number; - readonly reason: "unknown-revision" | "journal-evicted" | "bulk-replace"; + readonly reason: "unknown-revision" | "journal-evicted" | "bulk-replace" | "reorder"; }; // @public (undocumented) diff --git a/packages/grid-core/src/__tests__/indexed-selection.test.ts b/packages/grid-core/src/__tests__/indexed-selection.test.ts index 62a258b37..665ee1c91 100644 --- a/packages/grid-core/src/__tests__/indexed-selection.test.ts +++ b/packages/grid-core/src/__tests__/indexed-selection.test.ts @@ -774,6 +774,67 @@ describe("indexed row selection", () => { expect(dataRowReads).toBe(100_000); }); + test('handles a "reorder" reset exactly as a "bulk-replace" reset', () => { + // Fail-closed pin: grid-core is deliberately reorder-UNAWARE. A reset + // whose reason is "reorder" (or any reason this suite has never heard + // of) must take the same full-rebuild path as "bulk-replace" — the + // reason field is advisory for consumers that opt into it, never a + // requirement for correctness. + const rows = [1, 2, 3, 4].map((id) => ({ id, team: "a", score: id })); + const model = createLocalRowModel({ + rows, + columns, + getRowId: (row) => row.id, + }); + const previous = model.getState().snapshot; + const selected = selectIndexedRowRange( + createEmptyIndexedSelection(), + 1, + 2, + previous, + ); + model.setRows([rows[3]!, rows[2]!, rows[1]!, rows[0]!]); + const snapshot = model.getState().snapshot; + + const project = (reason: "reorder" | "bulk-replace" | "unknown-revision") => + projectIndexedSelection(selected, previous, snapshot, { + kind: "reset", + toRevision: snapshot.revision, + reason, + }); + const viaReorder = project("reorder"); + const viaBulkReplace = project("bulk-replace"); + + for (const projected of [viaReorder, viaBulkReplace]) { + expect(getIndexedSelectionSummary(projected, snapshot)).toEqual( + getIndexedSelectionSummary(viaBulkReplace, snapshot), + ); + for (const rowId of [1, 2, 3, 4]) + expect( + isIndexedRowSelected(projected, { kind: "data", rowId }, snapshot), + ).toBe( + isIndexedRowSelected( + viaBulkReplace, + { kind: "data", rowId }, + snapshot, + ), + ); + } + // The rebuild is semantic, not positional: rows 1 and 2 stay selected + // by identity even though the reorder moved them. + expect(getIndexedSelectionSummary(viaReorder, snapshot)).toEqual({ + state: "some", + selectedCount: 2, + visibleCount: 4, + }); + expect( + isIndexedRowSelected(viaReorder, { kind: "data", rowId: 1 }, snapshot), + ).toBe(true); + expect( + isIndexedRowSelected(viaReorder, { kind: "data", rowId: 2 }, snapshot), + ).toBe(true); + }); + test("drops a covered subtree of 50k disjoint exclusions without scanning it", () => { const source = createModel().getState().snapshot; let dataIndexReads = 0; diff --git a/packages/layout-core/src/__tests__/row-height-index.test.ts b/packages/layout-core/src/__tests__/row-height-index.test.ts index 0b788b0e2..2b0e91daf 100644 --- a/packages/layout-core/src/__tests__/row-height-index.test.ts +++ b/packages/layout-core/src/__tests__/row-height-index.test.ts @@ -821,7 +821,11 @@ describe("persistent row-height index", () => { const rows = Array.from({ length: 1_000 }, (_, index) => entry(data(String(index)), 20), ); - const base = createIndex(rows); + // Measured so the base holds retained state: an unmeasured base now takes + // the synchronous bulk path, and this test's subject is the COOPERATIVE + // slicing of the no-op scan. The measurement does not disturb the no-op — + // that predicate reads identities and estimates only. + const base = createIndex(rows).measure(0, data("0"), 41); const builder = base.beginReplacement({ rowCount: rows.length, entryAt: (index) => entry(data(String(index)), 20), @@ -1321,3 +1325,400 @@ describe("persistent row-height index", () => { } }, 30_000); }); + +describe("synchronous reorder over existing height entries", () => { + /** + * A base index with mixed measured and estimated entries: rows 0..N-1 with + * varied estimates (including `undefined` → default height), every third row + * measured to a height its estimate could not predict. + */ + function reorderFixture(count = 25) { + const keys = Array.from({ length: count }, (_, index) => + index % 5 === 0 ? group(String(index)) : data(String(index)), + ); + const estimates = keys.map((_, index) => + index % 4 === 3 ? undefined : 18 + (index % 7) * 3, + ); + let base = createIndex( + keys.map((key, index) => entry(key, estimates[index])), + 30, + ); + for (let index = 0; index < count; index += 3) { + base = base.measure(index, keys[index]!, 51 + index); + } + return { keys, estimates, base, count }; + } + + function sourceFor( + keys: readonly Key[], + estimates?: readonly (number | undefined)[], + ): RowHeightReplacementSource { + return { + rowCount: keys.length, + entryAt: (index) => entry(keys[index]!, estimates?.[index]), + }; + } + + /** Every rank's offset and height, plus the total: the full geometry. */ + function rankTable(index: RowHeightIndex) { + return { + rowCount: index.rowCount, + total: index.getTotalHeight(), + offsets: Array.from({ length: index.rowCount + 1 }, (_, rank) => + index.getOffsetForIndex(rank), + ), + heights: Array.from({ length: index.rowCount }, (_, rank) => + index.getHeight(rank), + ), + }; + } + + function permutations(count: number): Record { + const identity = Array.from({ length: count }, (_, index) => index); + const reversal = [...identity].reverse(); + const swap = [...identity]; + [swap[3], swap[17]] = [swap[17]!, swap[3]!]; + return { reversal, swap, identity }; + } + + test("matches a full replacement oracle for reversal, swap, and identity", () => { + const { keys, estimates, base, count } = reorderFixture(); + for (const order of Object.values(permutations(count))) { + const orderedKeys = order.map((rank) => keys[rank]!); + const orderedEstimates = order.map((rank) => estimates[rank]); + const reordered = base.reorder(sourceFor(orderedKeys)); + const replaced = base.replace( + orderedKeys.map((key, index) => entry(key, orderedEstimates[index])), + ); + expect(rankTable(reordered)).toEqual(rankTable(replaced)); + } + }); + + test("reuses every existing entry and re-measures none", () => { + const { keys, base, count } = reorderFixture(); + const reordered = base.reorder(sourceFor([...keys].reverse())); + expect(getRowHeightIndexDiagnosticsForTesting(reordered)).toMatchObject({ + reorderEntriesReused: count, + reorderEntriesRemeasured: 0, + }); + }); + + test("an identity-order reorder is a no-op returning the same index", () => { + const { keys, base } = reorderFixture(); + expect(base.reorder(sourceFor(keys))).toBe(base); + }); + + test("a key absent from the existing rows throws instead of fabricating", () => { + const { keys, base } = reorderFixture(); + const foreign = [...keys]; + foreign[6] = data("not-an-existing-row"); + let thrown: unknown; + try { + base.reorder(sourceFor(foreign)); + } catch (error) { + thrown = error; + } + expect(thrown).toBeInstanceOf(Error); + expect((thrown as Error).message).toMatch(/existing row/i); + expect((thrown as Error).message).toContain("not-an-existing-row"); + }); + + test("a key duplicated in the new order throws", () => { + const { keys, base } = reorderFixture(); + const duplicated = [...keys]; + duplicated[6] = duplicated[7]!; + expect(() => base.reorder(sourceFor(duplicated))).toThrow(/existing row/i); + }); + + test("a row-count mismatch throws in both directions", () => { + const { keys, base } = reorderFixture(); + for (const rowCount of [keys.length - 1, keys.length + 1]) { + let thrown: unknown; + try { + base.reorder({ rowCount, entryAt: (index) => entry(keys[index]!) }); + } catch (error) { + thrown = error; + } + expect(thrown).toBeInstanceOf(RangeError); + expect((thrown as Error).message).toMatch(/row count/i); + } + expect(() => + base.reorder({ rowCount: 0.5, entryAt: () => entry(keys[0]!) }), + ).toThrow(RangeError); + }); + + test("keeps estimates and measurements intact, ignoring source estimates", () => { + const { keys, base, count } = reorderFixture(); + // Rank 1 is estimated (estimate 21), rank 3 is measured (54). Hand the + // source wildly different estimates for every row: a reorder must not + // re-estimate or re-measure, so the original heights survive verbatim. + const reversedKeys = [...keys].reverse(); + const lyingEstimates = reversedKeys.map(() => 999); + const reordered = base.reorder(sourceFor(reversedKeys, lyingEstimates)); + for (let rank = 0; rank < count; rank += 1) { + expect(reordered.getHeight(rank)).toBe(base.getHeight(count - 1 - rank)); + } + expect(reordered.getTotalHeight()).toBe(base.getTotalHeight()); + }); + + test("leaves the old index untouched", () => { + const { keys, base } = reorderFixture(); + const before = rankTable(base); + const beforeDiagnostics = getRowHeightIndexDiagnosticsForTesting(base); + const reordered = base.reorder(sourceFor([...keys].reverse())); + expect(reordered).not.toBe(base); + expect(rankTable(base)).toEqual(before); + expect(getRowHeightIndexDiagnosticsForTesting(base)).toEqual( + beforeDiagnostics, + ); + }); + + test("post-reorder mutations behave exactly like a replace-built index", () => { + const { keys, estimates, base } = reorderFixture(); + const reversedKeys = [...keys].reverse(); + const reversedEntries = reversedKeys.map((key, index) => + entry(key, estimates[keys.length - 1 - index]), + ); + const viaReorder = base.reorder(sourceFor(reversedKeys)); + const viaReplace = base.replace(reversedEntries); + + // A measurement update lands identically on both. + const target = reversedKeys[4]!; + const measuredReorder = viaReorder.measure(4, target, 77); + const measuredReplace = viaReplace.measure(4, target, 77); + expect(rankTable(measuredReorder)).toEqual(rankTable(measuredReplace)); + expect(measuredReorder.getHeight(4)).toBe(77); + + // A subsequent full replacement lands identically on both. + const nextRows = [ + ...reversedEntries.slice(5), + entry(data("fresh-a"), 22), + entry(data("fresh-b")), + ]; + expect(rankTable(measuredReorder.replace(nextRows))).toEqual( + rankTable(measuredReplace.replace(nextRows)), + ); + }); +}); + +describe("bulk replacement when the base holds no retained state", () => { + /** Every rank's offset and height, plus the total: the full geometry. */ + function rankTable(index: RowHeightIndex) { + return { + rowCount: index.rowCount, + total: index.getTotalHeight(), + keys: Array.from({ length: index.rowCount }, (_, rank) => + index.keyAt(rank), + ), + offsets: Array.from({ length: index.rowCount + 1 }, (_, rank) => + index.getOffsetForIndex(rank), + ), + heights: Array.from({ length: index.rowCount }, (_, rank) => + index.getHeight(rank), + ), + }; + } + + /** Mixed estimates: undefined (→ default) interleaved with varied numbers. */ + function mixedRows(count: number): RowHeightEntry[] { + return Array.from({ length: count }, (_, index) => + entry( + index % 5 === 0 ? group(String(index)) : data(String(index)), + index % 4 === 3 ? undefined : 18 + (index % 7) * 3, + ), + ); + } + + function sourceOf(rows: readonly RowHeightEntry[]) { + return { + rowCount: rows.length, + entryAt: (index: number) => rows[index]!, + }; + } + + /** + * Drives the COOPERATIVE builder over `source`. The base carries one + * measurement on an identity disjoint from every source row, which forces + * the retained-state path without affecting any produced height: the ingest + * lookup misses for every source identity, and the pinned measurement only + * lands in the result's tombstones, which `rankTable` never observes. + */ + function cooperativeResult( + source: RowHeightReplacementSource, + ): RowHeightIndex { + const pin = data("__cooperative-pin__"); + const base = createIndex([entry(pin)]).measure(0, pin, 77); + expect(base.hasRetainedState).toBe(true); + const builder = base.beginReplacement(source); + const first = builder.advance({ maxUnits: 1, now: () => 0 }); + expect(first.done).toBe(false); + while (!builder.done) builder.advance({ maxUnits: 256, now: () => 0 }); + return builder.finish(); + } + + test("hasRetainedState is false for empty and never-measured indexes", () => { + const empty = createIndex([]); + expect(empty.hasRetainedState).toBe(false); + + // 50k-shaped case in miniature: entries exist, but none carries a + // measurement, so a replacement's retained-state lookups would all miss. + const populated = createIndex(mixedRows(64)); + expect(populated.hasRetainedState).toBe(false); + + const replaced = populated.replace(mixedRows(32)); + expect(replaced.hasRetainedState).toBe(false); + }); + + test("hasRetainedState turns true with a measurement and with tombstones", () => { + const rows = mixedRows(8); + const measured = createIndex(rows).measure(1, rows[1]!.key, 44); + expect(measured.hasRetainedState).toBe(true); + + // Removing the measured row converts the measurement into a tombstone + + // retention-order entry; all three retained categories are now non-empty. + const tombstoned = measured.apply([ + { kind: "remove", ref: rows[1]!.key, previousIndex: 1 }, + ]); + expect( + getRowHeightIndexDiagnosticsForTesting(tombstoned).tombstoneCount, + ).toBe(1); + expect(tombstoned.hasRetainedState).toBe(true); + + // `retainMeasurement` on an absent key is the other tombstone producer. + const retained = createIndex(rows).retainMeasurement(data("gone"), 51); + expect(retained.hasRetainedState).toBe(true); + }); + + test("hasRetainedState returns to false when retention is disabled", () => { + // With maxRetainedMeasurements 0 a removal deletes the measurement instead + // of tombstoning it, so the index can empty back out. + const rows = mixedRows(4); + const measured = createIndex(rows, 30, 0).measure(2, rows[2]!.key, 44); + expect(measured.hasRetainedState).toBe(true); + const emptied = measured.apply([ + { kind: "remove", ref: rows[2]!.key, previousIndex: 2 }, + ]); + expect( + getRowHeightIndexDiagnosticsForTesting(emptied).measurementCacheCount, + ).toBe(0); + expect(emptied.hasRetainedState).toBe(false); + }); + + test("a no-retained-state replacement completes on its first advance", () => { + for (const count of [0, 1, 32, 1_000]) { + const base = createIndex(mixedRows(Math.max(0, count - 7))); + expect(base.hasRetainedState).toBe(false); + const rows = mixedRows(count); + const builder = base.beginReplacement(sourceOf(rows)); + const first = builder.advance({ maxUnits: 1, now: () => 0 }); + expect(first.done).toBe(true); + expect(first.phase).toBe("done"); + expect(first.sourceRowsIngested).toBe(count); + const result = builder.finish(); + expect(result.rowCount).toBe(count); + } + }); + + test("bulk geometry equals the cooperative builder's at every rank", () => { + for (const count of [0, 1, 32, 1_000]) { + const rows = mixedRows(count); + const base = createIndex([]); + const builder = base.beginReplacement(sourceOf(rows)); + builder.advance({ maxUnits: 1, now: () => 0 }); + const bulk = builder.finish(); + expect(rankTable(bulk)).toEqual( + rankTable(cooperativeResult(sourceOf(rows))), + ); + } + }); + + test("a bulk replacement with an identical source is the same no-op", () => { + const rows = mixedRows(24); + const base = createIndex(rows); + const builder = base.beginReplacement(sourceOf(rows)); + const first = builder.advance({ maxUnits: 1, now: () => 0 }); + expect(first.done).toBe(true); + expect(builder.finish()).toBe(base); + }); + + test("a duplicate source identity fails exactly like the cooperative path", () => { + const duplicated = { + rowCount: 3, + entryAt: (index: number) => + entry(data(index === 2 ? "0" : String(index))), + }; + const bulkBuilder = createIndex([]).beginReplacement(duplicated); + let bulkError: unknown; + try { + bulkBuilder.advance({ maxUnits: 1, now: () => 0 }); + } catch (error) { + bulkError = error; + } + expect(bulkError).toBeInstanceOf(Error); + expect((bulkError as Error).message).toMatch( + /Duplicate stable row-height key/, + ); + expectReplacementLifecycleError( + () => bulkBuilder.advance({ maxUnits: 1 }), + "failed", + ); + + let cooperativeError: unknown; + try { + cooperativeResult(duplicated); + } catch (error) { + cooperativeError = error; + } + expect((cooperativeError as Error).message).toBe( + (bulkError as Error).message, + ); + }); + + test("post-bulk mutations behave exactly like a cooperatively built twin", () => { + const rows = mixedRows(40); + const builder = createIndex([]).beginReplacement(sourceOf(rows)); + builder.advance({ maxUnits: 1, now: () => 0 }); + const bulk = builder.finish(); + const cooperative = cooperativeResult(sourceOf(rows)); + + // A measurement lands identically on both. + const bulkMeasured = bulk.measure(4, rows[4]!.key, 91); + const cooperativeMeasured = cooperative.measure(4, rows[4]!.key, 91); + expect(rankTable(bulkMeasured)).toEqual(rankTable(cooperativeMeasured)); + expect(bulkMeasured.getHeight(4)).toBe(91); + + // A subsequent full replacement lands identically on both. Both twins now + // carry a measurement, so both take the cooperative path. + const nextRows = [ + ...rows.slice(9), + entry(data("fresh-a"), 22), + entry(data("fresh-b")), + ]; + expect(rankTable(bulkMeasured.replace(nextRows))).toEqual( + rankTable(cooperativeMeasured.replace(nextRows)), + ); + + // A permutation lands identically on both. + const reversed = { + rowCount: rows.length, + entryAt: (index: number) => ({ + key: rows[rows.length - 1 - index]!.key, + }), + }; + expect(rankTable(bulkMeasured.reorder(reversed))).toEqual( + rankTable(cooperativeMeasured.reorder(reversed)), + ); + }); + + test("any retained measurement disables the bulk path", () => { + const rows = mixedRows(16); + const base = createIndex(rows).measure(0, rows[0]!.key, 63); + const builder = base.beginReplacement(sourceOf(mixedRows(48))); + const first = builder.advance({ maxUnits: 1, now: () => 0 }); + expect(first.done).toBe(false); + expect(first.phase).toBe("ingest"); + while (!builder.done) builder.advance({ maxUnits: 256, now: () => 0 }); + const result = builder.finish(); + expect(result.getHeight(0)).toBe(63); + }); +}); diff --git a/packages/layout-core/src/row-height-index.ts b/packages/layout-core/src/row-height-index.ts index bcac386a5..41222521f 100644 --- a/packages/layout-core/src/row-height-index.ts +++ b/packages/layout-core/src/row-height-index.ts @@ -18,6 +18,8 @@ interface Work { measurementEntriesScanned: number; previousEntriesScanned: number; sortComparisons: number; + reorderEntriesReused: number; + reorderEntriesRemeasured: number; } function createWork(entriesVisited = 0): Work { @@ -29,6 +31,8 @@ function createWork(entriesVisited = 0): Work { measurementEntriesScanned: 0, previousEntriesScanned: 0, sortComparisons: 0, + reorderEntriesReused: 0, + reorderEntriesRemeasured: 0, }; } @@ -149,6 +153,15 @@ export interface RowHeightIndexDiagnostics { readonly previousEntriesScanned: number; /** Comparator calls from sorting; bulk replacement deliberately performs none. */ readonly sortComparisons: number; + /** Existing height entries relinked as-is by `reorder` — no re-measure. */ + readonly reorderEntriesReused: number; + /** + * Entries whose height `reorder` recomputed instead of reusing. The reorder + * path reuses every entry object verbatim, so this counts increments at + * height-recompute sites — of which the path has none — rather than + * asserting zero by fiat. + */ + readonly reorderEntriesRemeasured: number; readonly visibleMeasurementCount: number; readonly tombstoneCount: number; readonly measurementCacheCount: number; @@ -413,6 +426,28 @@ function sequenceAt( return undefined; } +/** + * One-pass balanced build over an already-ordered value array — the + * synchronous counterpart of the replacement builder's `build-sequence` phase + * (same midpoint convention, so both produce the same shape). Depth is + * logarithmic, so the recursion is stack-safe at any realistic row count. + */ +function buildBalancedSequence( + values: readonly HeightValue[], + start: number, + end: number, + work: Work, +): SequenceNode | null { + if (start >= end) return null; + const middle = Math.floor((start + end) / 2); + return sequenceNode( + values[middle]!, + buildBalancedSequence(values, start, middle, work), + buildBalancedSequence(values, middle + 1, end, work), + work, + ); +} + function hashIdentity(identity: string): number { let hash = 0x811c9dc5; for (let index = 0; index < identity.length; index += 1) { @@ -1074,6 +1109,8 @@ class PersistentRowHeightIndex implements RowHeightIndex { measurementEntriesScanned: options.work.measurementEntriesScanned, previousEntriesScanned: options.work.previousEntriesScanned, sortComparisons: options.work.sortComparisons, + reorderEntriesReused: options.work.reorderEntriesReused, + reorderEntriesRemeasured: options.work.reorderEntriesRemeasured, visibleMeasurementCount: hashCount(options.measurements) - hashCount(options.tombstones), tombstoneCount: hashCount(options.tombstones), @@ -1085,6 +1122,37 @@ class PersistentRowHeightIndex implements RowHeightIndex { return nodeCount(this.#root); } + /** + * Derivation — what must be empty for a bulk replacement to be + * byte-equivalent to the cooperative one. The cooperative builder consults + * prior state in exactly three places: + * + * 1. The ingest's measured-height lookup reads `#measurements` — a hit makes + * a produced entry `measured` with the cached height. + * 2. The `scan-retained` walk reads `#tombstoneOrder` (whose entries mirror + * `#tombstones`) to carry removed-row measurements forward. + * 3. The `scan-visible` walk retains a prior VISIBLE entry's measurement + * only when `value.measured` — and a visible entry is `measured` only + * while its identity is in `#measurements` (`measure` sets both, + * `apply`'s re-estimate clears both), so `#measurements` empty makes this + * branch unreachable. + * + * `#visibleKeys`, `#nextTicket`, and the sequence entries themselves are + * never read for values — the builder rebuilds the key set and sequence from + * the source — so a populated but never-measured index (e.g. 50k rows at + * true mount) has NO retained state: every lookup above would miss, and a + * from-scratch bulk build over the source produces the identical index. + * The predicate is therefore exactly "measurements, tombstones, and + * retention order are all empty". + */ + get hasRetainedState(): boolean { + return ( + hashCount(this.#measurements) > 0 || + hashCount(this.#tombstones) > 0 || + this.#tombstoneOrder !== null + ); + } + getHeight(index: number): number { if (!Number.isSafeInteger(index)) return 0; return sequenceAt(this.#root, index)?.height ?? 0; @@ -1386,6 +1454,94 @@ class PersistentRowHeightIndex implements RowHeightIndex { return builder.finish(); } + /** + * Synchronous BY DESIGN: a sort-only commit reorders EXISTING rows whose + * heights are already known, so only the ordered structure and its prefix + * sums change — no re-measure, no re-estimate, no identity re-hash beyond + * the per-row key encoding. Measured ~10–20ms at 50k rows versus ~215ms of + * cooperative `beginReplacement` re-ingest (identity hash + HAMT insert + + * frozen rowRef per row, in 8ms slices), which is the whole point of the + * method. The caller (renderer-dom's row-layout controller) falls back to a + * full replacement on ANY throw, so violations of the permutation contract + * fail loud here rather than fabricating entries. + */ + reorder(source: RowHeightReplacementSource): RowHeightIndex { + const rowCount = source.rowCount; + if (!Number.isSafeInteger(rowCount) || rowCount < 0) { + throw new RangeError( + "Reorder source rowCount must be a non-negative safe integer.", + ); + } + const entryAt = source.entryAt; + if (typeof entryAt !== "function") { + throw new TypeError("Reorder source entryAt must be a function."); + } + if (rowCount !== this.rowCount) { + throw new RangeError( + `Reorder source rowCount (${rowCount}) must equal the current row ` + + `count (${this.rowCount}); reorder permutes the existing rows only.`, + ); + } + if (rowCount === 0) return this; + const boundEntryAt = entryAt.bind(source); + const work = createWork(); + + // One in-order pass over the current sequence: the by-identity lookup + // table for the walk below, and the old order for no-op detection. + const previousValues: HeightValue[] = []; + const unconsumed = new Map>(); + { + const stack: SequenceNode[] = []; + let cursor = this.#root; + while (cursor !== null || stack.length > 0) { + while (cursor !== null) { + stack.push(cursor); + cursor = cursor.left; + } + const node = stack.pop()!; + previousValues.push(node.value); + unconsumed.set(node.value.identity, node.value); + work.previousEntriesScanned += 1; + cursor = node.right; + } + } + + // Walk the new order, relinking each EXISTING entry verbatim. Estimates + // and measurements ride along untouched inside the reused entry objects, + // so the source's `estimatedHeight`s are deliberately ignored. + const values: HeightValue[] = new Array>(rowCount); + let unchanged = true; + for (let index = 0; index < rowCount; index += 1) { + const row = boundEntryAt(index); + const identity = this.#identity(row.key); + work.entriesVisited += 1; + work.identityLookups += 1; + const value = unconsumed.get(identity); + if (value === undefined) { + throw new Error( + `Reorder key does not match an existing row (missing, or ` + + `duplicated in the new order): ${identity}`, + ); + } + unconsumed.delete(identity); + values[index] = value; + work.reorderEntriesReused += 1; + if (value !== previousValues[index]) unchanged = false; + } + if (unchanged) return this; + + const root = buildBalancedSequence(values, 0, rowCount, work); + return this.#next( + root, + this.#visibleKeys, + this.#measurements, + this.#tombstones, + this.#tombstoneOrder, + this.#nextTicket, + work, + ); + } + beginReplacement( source: RowHeightReplacementSource, ): RowHeightReplacementBuilder { @@ -1400,6 +1556,10 @@ class PersistentRowHeightIndex implements RowHeightIndex { throw new TypeError("Replacement source entryAt must be a function."); } return new PersistentRowHeightReplacementBuilder({ + // With no retained state every ingest lookup would miss (see + // `hasRetainedState`'s derivation), so the builder may build everything + // in one synchronous O(n) pass instead of cooperative phases. + bulk: !this.hasRetainedState, base: { index: this, defaultHeight: this.#defaultHeight, @@ -1533,25 +1693,32 @@ class PersistentRowHeightReplacementBuilder< done: 0, }; readonly #totalUnits: number; + readonly #bulk: boolean; readonly #work = createWork(); constructor(options: { readonly base: ReplacementBase; readonly rowCount: number; readonly entryAt: (index: number) => RowHeightEntry; + readonly bulk: boolean; }) { this.#base = options.base; this.#entryAt = options.entryAt; this.#sourceRowCount = options.rowCount; this.#measurements = options.base.measurements; this.#nextTicket = options.base.nextTicket; - this.#totalUnits = Math.min( - Number.MAX_SAFE_INTEGER, - options.rowCount * 4 + - nodeCount(options.base.root) * 8 + - hashCount(options.base.tombstones) * 8 + - 8, - ); + this.#bulk = options.bulk; + // The bulk pass is one unit of work by construction; the cooperative + // estimate below deliberately over-counts so progress never regresses. + this.#totalUnits = options.bulk + ? 1 + : Math.min( + Number.MAX_SAFE_INTEGER, + options.rowCount * 4 + + nodeCount(options.base.root) * 8 + + hashCount(options.base.tombstones) * 8 + + 8, + ); } get done(): boolean { @@ -1641,6 +1808,29 @@ class PersistentRowHeightReplacementBuilder< } observedAt = startedAt; } + if (this.#bulk && this.#status === "pending") { + // Synchronous BY DESIGN (spec C2a): with no retained state the + // cooperative phases degenerate — every measured-height lookup misses, + // scan-retained walks nothing, and eviction/tombstone/retention builds + // are all empty — so the entire replacement is one O(n) pass: + // duplicate-checked ingest + `buildBalancedSequence`. Measured + // ~15-20ms at 50k rows (B3's balanced build) versus ~450ms of sliced + // cooperative re-ingest during which a mounting grid paints nothing. + // `maxUnits`/`deadline` are deliberately not consulted: the pass is a + // single unit, and slicing it would recreate the blank mount. + this.#stepBulk(); + this.#phaseUnits.ingest += 1; + units = 1; + this.#completedUnits += 1; + if (options.now !== undefined) { + observedAt = options.now(); + if (!Number.isFinite(observedAt)) { + throw new RangeError( + "Replacement clock must return a finite number.", + ); + } + } + } while (units < maxUnits && this.#status === "pending") { const phase = this.#phase; this.#step(); @@ -1765,6 +1955,91 @@ class PersistentRowHeightReplacementBuilder< } } + /** + * The whole replacement in one pass, valid only when the base has no + * retained state (`hasRetainedState === false`, checked by + * `beginReplacement`). Reproduces the cooperative ingest exactly, with the + * measured-height lookup resolved by the gate itself: `#measurements` is + * empty, so every lookup would miss and height is always + * `estimatedHeight ?? defaultHeight` with `measured: false`. The duplicate + * identity check and its error are the cooperative ingest's, verbatim. + * Semantic no-op detection is preserved too: an identical visible sequence + * (same identities and estimates, in order) finishes to the base index. + */ + #stepBulk(): void { + const base = this.#base!; + const values = this.#values!; + const identities = this.#identities!; + const entryAt = this.#entryAt!; + while (this.#ingestIndex < this.#sourceRowCount) { + const row = entryAt(this.#ingestIndex); + const identity = encodeStableKey(base.getKey(row.key)); + this.#work.entriesVisited += 1; + this.#work.identityLookups += 1; + if (identities.has(identity)) { + throw new Error(`Duplicate stable row-height key: ${identity}`); + } + identities.add(identity); + const estimatedHeight = + row.estimatedHeight === undefined + ? base.defaultHeight + : normalizeHeight(row.estimatedHeight, "Estimated row height"); + this.#visibleKeys = hashSet( + this.#visibleKeys, + identity, + true, + this.#work, + ); + values.push({ + ref: row.key, + identity, + estimatedHeight: row.estimatedHeight, + height: estimatedHeight, + measured: false, + }); + this.#ingestIndex += 1; + } + this.#entryAt = null; + + // Same no-op predicate as `#stepVisibleTraversal`: every prior visible + // entry matches its candidate's identity and estimate, and the counts + // agree. A count mismatch skips the scan entirely, so a true mount (empty + // base, populated source) pays nothing here. + if (nodeCount(base.root) === values.length) { + let equal = true; + const stack: SequenceNode[] = []; + let cursor = base.root; + let position = 0; + while (equal && (cursor !== null || stack.length > 0)) { + while (cursor !== null) { + stack.push(cursor); + cursor = cursor.left; + } + const node = stack.pop()!; + const candidate = values[position]!; + this.#work.previousEntriesScanned += 1; + if ( + candidate.identity !== node.value.identity || + candidate.estimatedHeight !== node.value.estimatedHeight + ) { + equal = false; + } + position += 1; + cursor = node.right; + } + if (equal) { + this.#noOp = true; + this.#phase = "done"; + this.#status = "done"; + return; + } + } + + this.#root = buildBalancedSequence(values, 0, values.length, this.#work); + this.#phase = "done"; + this.#status = "done"; + } + #stepIngest(): void { const base = this.#base!; const values = this.#values!; diff --git a/packages/layout-core/src/types.ts b/packages/layout-core/src/types.ts index f4e59ed61..567b2d9f9 100644 --- a/packages/layout-core/src/types.ts +++ b/packages/layout-core/src/types.ts @@ -146,6 +146,17 @@ export interface RowHeightReplacementBuilder { * @internal */ export interface RowHeightIndex extends RowMetricsReader { + /** + * True iff the index holds ANY state a replacement could retain: a cached + * measurement, a tombstoned (removed-row) measurement, or retention-order + * bookkeeping. Visible entries alone are NOT retained state — a populated but + * never-measured index reports `false`, because a replacement's retained-state + * lookups would all miss and rebuilding from the source alone is exact. When + * this is `false`, `beginReplacement` returns a builder that completes in a + * single `advance` (the synchronous bulk path); callers may run it to + * completion without cooperative slicing. + */ + readonly hasRetainedState: boolean; keyAt(index: number): TKey | undefined; hasMeasurement(ref: TKey): boolean; /** @@ -167,6 +178,14 @@ export interface RowHeightIndex extends RowMetricsReader { retainMeasurement(ref: TKey, height: number): RowHeightIndex; apply(operations: readonly RowHeightOperation[]): RowHeightIndex; replace(rows: readonly RowHeightEntry[]): RowHeightIndex; + /** + * Rebuilds the ordered structure from the EXISTING entries in a new order — + * a permutation of the current rows, synchronously. No entry is re-measured + * or re-estimated (source `estimatedHeight`s are ignored); only the sequence + * and its prefix sums are recomputed. Throws when the source is not an exact + * permutation of the current rows; callers fall back to `beginReplacement`. + */ + reorder(source: RowHeightReplacementSource): RowHeightIndex; beginReplacement( source: RowHeightReplacementSource, ): RowHeightReplacementBuilder; diff --git a/packages/react/react.api.md b/packages/react/react.api.md index 9fb99f7e6..56322bee5 100644 --- a/packages/react/react.api.md +++ b/packages/react/react.api.md @@ -532,7 +532,7 @@ export type PretableChangeSequence = { } | { readonly kind: "reset"; readonly toRevision: number; - readonly reason: "unknown-revision" | "journal-evicted" | "bulk-replace"; + readonly reason: "unknown-revision" | "journal-evicted" | "bulk-replace" | "reorder"; }; // @public (undocumented) diff --git a/packages/react/src/__tests__/indexed-rendering.test.tsx b/packages/react/src/__tests__/indexed-rendering.test.tsx index 98d29dd7a..312b00292 100644 --- a/packages/react/src/__tests__/indexed-rendering.test.tsx +++ b/packages/react/src/__tests__/indexed-rendering.test.tsx @@ -1260,13 +1260,21 @@ describe("indexed PretableSurface", () => { ).toBe(true), ); - // The commit this test exists for has to have happened, or the assertion - // below is vacuous: rows loaded, plan not yet caught up. + // Before the synchronous bulk mount (layout-core's no-retained-state + // replacement path), the block landed against the EMPTY plan and this + // asserted that stale commit existed — rows loaded, `totalHeight` 0 — so + // the no-gap check below could not pass vacuously. The bulk path now + // rebuilds and publishes the plan synchronously inside the same commit, + // so rows and their geometry arrive together and the stale interval is + // gone from the mount sequence. Pin THAT instead: no commit ever shows + // loaded rows against the empty plan. The no-gap assertion below is still + // exercised across the whole sequence, including the pre-load commits the + // `waitFor` above observed. expect( telemetry.some( (entry) => entry.loadedRowCount === 30 && entry.totalHeight === 0, ), - ).toBe(true); + ).toBe(false); // Nobody scrolled, so no gap is reported at any point. expect(telemetry.map((entry) => entry.windowGap)).toEqual( telemetry.map(() => undefined), @@ -1275,6 +1283,117 @@ describe("indexed PretableSurface", () => { view.unmount(); }); + test("windowGap stays silent when the plan matches the prop but not the model", async () => { + // The suppression's model-count clause (`plannedRowCount !== + // rowModelSnapshot.sourceRowCount` in pretable-surface.tsx) is the ONLY + // guard left in one production interval. Every rendered row is measured, + // which makes the next rows replacement COOPERATIVE: the plan keeps + // drawing the OLD block while the model has already ingested the NEW one. + // The prop-count clause covers that interval too — until the consumer + // flips `rows` back to the old block before the rebuild publishes. Now + // prop and plan agree again (the `loadedRowCount` clause passes) while + // the model still holds the new set, and without the model-count clause + // the gap arithmetic mixes the stale plan's boundary with the model's + // trailing count into an "after" gap about no window anybody committed. + const telemetry: { + readonly loadedRowCount: number; + readonly totalHeight: number; + readonly windowGap: unknown; + }[] = []; + const onTelemetryChange = (next: { + loadedRowCount: number; + totalHeight: number; + windowGap?: unknown; + }) => { + telemetry.push({ + loadedRowCount: next.loadedRowCount, + totalHeight: next.totalHeight, + windowGap: next.windowGap, + }); + }; + const Harness = (props: { readonly block: readonly Row[] }) => ( + row.id} + onQueryChange={() => undefined} + onTelemetryChange={onTelemetryChange} + overscan={0} + processing={{ filter: "external", sort: "external" }} + query={{ filters: [], sort: [], rowGroups: [] }} + resultMeta={{ + total: { kind: "exact", count: 1_000 }, + window: { start: 0, hasMore: true }, + }} + rows={props.block} + viewportHeight={168} + /> + ); + const blockA = rows.slice(0, 10); + const blockB = rows.slice(0, 30); + + const view = render(); + await waitFor(() => + expect( + telemetry.some( + (entry) => entry.loadedRowCount === 10 && entry.totalHeight > 0, + ), + ).toBe(true), + ); + const steadyTotalA = telemetry[telemetry.length - 1]!.totalHeight; + + // Scroll so the body viewport's bottom edge passes block A's loaded + // extent (10 rows x 44px = 440px; 400 + the 132px body viewport = 532px + // > 440px). At A's steady state that is a GENUINE "after" gap — the + // positive control that the gap machinery works against exactly this + // geometry, so the silence asserted below is the clause's doing, not a + // viewport that never reached the boundary. + const viewport = view.getByRole("grid", { + name: "windowed grid stale model", + }); + fireEvent.scroll(viewport, { target: { scrollTop: 400 } }); + await waitFor(() => + expect(telemetry[telemetry.length - 1]!.windowGap).toEqual({ + direction: "after", + rowCount: 990, + }), + ); + + // Both rerenders in one synchronous block: the cooperative rebuild's + // first slice is a macrotask, so the second rerender lands while the + // 30-row replacement is still active and the plan still draws block A — + // including the render where the prop has already flipped back (prop 10, + // plan 10, model 30), which only the model-count clause suppresses. + telemetry.length = 0; + view.rerender(); + view.rerender(); + + // The stale interval this test exists for has to have happened, or the + // silence assertion below is vacuous: the model ingested the 30-row block + // while the plan still drew the 10-row one. A synchronous (bulk) rebuild + // here would mean the measured-base seeding failed. + expect( + telemetry.some( + (entry) => + entry.loadedRowCount === 30 && entry.totalHeight === steadyTotalA, + ), + ).toBe(true); + // No commit in the stale interval ever agreed on one window, so every one + // of them stays silent — even though the stale boundary (440px) sits + // above the scrolled viewport's bottom (468px) and the model's trailing + // count (970) is begging to be reported. (Block A's own steady states + // legitimately report their gap — see the control above — so the filter + // keys on the model count that defines the interval.) + const staleEntries = telemetry.filter( + (entry) => entry.loadedRowCount === 30, + ); + expect(staleEntries.map((entry) => entry.windowGap)).toEqual( + staleEntries.map(() => undefined), + ); + + view.unmount(); + }); + test("windowGap telemetry refreshes from a resultMeta-only update without a rows/viewport change", async () => { const onTelemetryChange = vi.fn(); const windowedRows = rows.slice(500, 530); diff --git a/packages/renderer-dom/src/__tests__/indexed-renderer.test.ts b/packages/renderer-dom/src/__tests__/indexed-renderer.test.ts index 4fcb32996..b9d60e72f 100644 --- a/packages/renderer-dom/src/__tests__/indexed-renderer.test.ts +++ b/packages/renderer-dom/src/__tests__/indexed-renderer.test.ts @@ -1212,9 +1212,11 @@ describe("indexed DOM row layout controller", () => { now: () => 0, maxUnitsPerSlice: 256, }); - const pending = controller.getState(); - expect(pending.status.kind).toBe("rebuilding"); - expect(pending.observedRevision).toBeNull(); + // The mount base holds no retained state, so the replacement bulk-builds + // and publishes inside the constructor: no "rebuilding" frame, and no + // scheduled slices for `flushAll` to drain. + expect(scheduler.tasks.length).toBe(0); + expect(controller.getState().status.kind).toBe("ready"); scheduler.flushAll(); const state = controller.getState(); @@ -1594,6 +1596,9 @@ describe("indexed DOM row layout controller", () => { })); const model = createModel(initial, { journalCapacity: 0 }); const { controller, scheduler } = createReadyController(model); + // Retained state keeps the reset on the cooperative path under test; an + // unmeasured base would bulk-build the 100k reset synchronously. + controller.measure(data(0), 45); const prior = controller.getState(); const next = Array.from({ length: 100_000 }, (_, index) => ({ id: index, @@ -1631,6 +1636,8 @@ describe("indexed DOM row layout controller", () => { })), ); const { controller, scheduler } = createReadyController(model); + // Retained state keeps the reset cooperative (the path under test). + controller.measure(data(0), 45); const beforeStarts = getRowLayoutControllerDiagnosticsForTesting( controller, @@ -1700,6 +1707,8 @@ describe("indexed DOM row layout controller", () => { })), ); const { controller, scheduler } = createReadyController(model); + // Retained state keeps both resets cooperative (the path under test). + controller.measure(data(0), 45); const beforeStarts = getRowLayoutControllerDiagnosticsForTesting( controller, @@ -1761,6 +1770,8 @@ describe("indexed DOM row layout controller", () => { { journalCapacity: 1 }, ); const { controller, scheduler } = createReadyController(model); + // Retained state keeps the reset cooperative (the path under test). + controller.measure(data(0), 45); const beforeStarts = getRowLayoutControllerDiagnosticsForTesting( controller, @@ -1811,6 +1822,8 @@ describe("indexed DOM row layout controller", () => { })), ); const { controller, scheduler } = createReadyController(model); + // Retained state keeps the reset cooperative (the path under test). + controller.measure(data(0), 45); const published = controller.getState(); model.setRows( Array.from({ length: 10_000 }, (_, index) => ({ @@ -2038,6 +2051,9 @@ describe("indexed DOM row layout controller", () => { now: () => 0, }); scheduler.flushAll(); + // Retained state keeps the reset cooperative, so it is still active when + // the hostile revision getter fires. + controller.measure(data(0), 45); source.setRows( Array.from({ length: 10_000 }, (_, index) => ({ id: index, @@ -2118,6 +2134,9 @@ describe("indexed DOM row layout controller", () => { })), ); const { controller, scheduler } = createReadyController(model); + // Retained state keeps the reset cooperative, so there is a rebuilding + // interval for the viewport request to be deferred into. + controller.measure(data(0), 45); const notifications = vi.fn(); controller.subscribe(notifications); model.setRows( @@ -2178,6 +2197,9 @@ describe("indexed DOM row layout controller", () => { now: () => 0, }); scheduler.flushAll(); + // Retained state keeps the reset cooperative, so the viewport request + // below is deferred into an ACTIVE replacement — the rollback under test. + controller.measure(data(1), 44); failNextEstimate = true; model.setRows( Array.from({ length: 40 }, (_, index) => ({ @@ -2258,6 +2280,9 @@ describe("indexed DOM row layout controller", () => { now: () => 0, }); scheduler.flushAll(); + // Retained state keeps the reset cooperative, so the hostile rowAt fires + // from a SCHEDULED slice — the stale-replacement flow under test. + controller.measure(data(0), 45); const statuses: string[] = []; controller.subscribe(() => statuses.push(controller.getState().status.kind), @@ -2308,6 +2333,9 @@ describe("indexed DOM row layout controller", () => { now: () => 0, }); queue.flushAll(); + // Retained state keeps the reset cooperative, so it schedules — which is + // where the reentrant-then-throwing scheduler under test can fire. + controller.measure(data(0), 45); const statuses: string[] = []; controller.subscribe(() => statuses.push(controller.getState().status.kind), @@ -2616,6 +2644,9 @@ describe("indexed DOM row layout controller", () => { ]); const scheduler = new ManualScheduler(new Error("cancel exploded")); const { controller } = createReadyController(model, scheduler); + // Retained state keeps the fallback replacements cooperative, so they + // schedule — the hostile scheduling/cancellation surface under test. + controller.measure(data(1), 45); const realChangesSince = model.changesSince.bind(model); vi.spyOn(model, "changesSince").mockImplementation((revision) => { const actual = realChangesSince(revision); @@ -2647,6 +2678,12 @@ describe("indexed DOM row layout controller", () => { viewport: { scrollTop: 0, viewportHeight: 88, overscan: 0 }, scheduler: throwingScheduler, }); + // The mount itself no longer schedules (bulk path), so a throwing + // scheduler cannot fail it. The hazard now lives where scheduling still + // happens: a cooperative replacement over retained state. + expect(failed.getState().status).toMatchObject({ kind: "ready" }); + failed.measure(data(4), 45); + model.setRows([{ id: 6, team: "D", score: 6, label: "six" }]); expect(failed.getState().status).toMatchObject({ kind: "error" }); expect( getRowLayoutControllerDiagnosticsForTesting(failed).retainedBuilderCount, @@ -2661,6 +2698,9 @@ describe("indexed DOM row layout controller", () => { { id: 2, team: "A", score: 2, label: "two" }, ]); const { controller, scheduler } = createReadyController(model); + // Retained state keeps the fallback replacement cooperative, preserving + // the rebuilding interval the atomicity assertions below observe. + controller.measure(data(1), 45); const before = controller.getState(); const actualChangesSince = model.changesSince.bind(model); vi.spyOn(model, "changesSince").mockImplementation((revision) => { @@ -2726,6 +2766,9 @@ describe("indexed DOM row layout controller", () => { { id: 2, team: "A", score: 2, label: "two" }, ]); const { controller, scheduler } = createReadyController(model); + // Retained state keeps the reset cooperative, so the measurement below + // arrives while it is ACTIVE and must be staged — the flow under test. + controller.measure(data(2), 40); const before = controller.getState(); model.setRows([{ id: 2, team: "B", score: 2, label: "two reset" }]); const startsAfterReset = @@ -2932,6 +2975,9 @@ describe("indexed DOM row layout controller", () => { })), ); const { controller } = createReadyController(model); + // Retained state keeps the reset cooperative, so the catch-up queue and + // staged measurement below accumulate against an ACTIVE replacement. + controller.measure(data(0), 40); model.setRows( Array.from({ length: 10_000 }, (_, index) => ({ id: index, @@ -2978,6 +3024,9 @@ describe("indexed DOM row layout controller", () => { { id: 2, team: "A", score: 2, label: "two" }, ]); const { controller, scheduler } = createReadyController(model); + // Retained state keeps the reset cooperative, so the measurement below is + // staged against an ACTIVE replacement whose target removed the row. + controller.measure(data(2), 40); model.setRows([{ id: 2, team: "B", score: 2, label: "two reset" }]); controller.measure(data(1), 93); scheduler.flushAll(); @@ -3130,6 +3179,16 @@ describe("indexed DOM row layout controller", () => { now: () => 0, }); expect(getState).toHaveBeenCalledTimes(1); + // The bulk mount publishes inside the constructor without scheduling. + expect(controller.getState().status.kind).toBe("ready"); + // The synchronous-callback hazard now lives where scheduling still + // happens: a cooperative replacement over retained state. Seed a + // measurement, then force a rebuild through the synchronous scheduler. + controller.measure(data(1), 91); + controller.setColumns([ + { id: "label", wrap: true, widthPx: 120 }, + { id: "score", widthPx: 80 }, + ]); expect(controller.getState().status.kind).toBe("rebuilding"); vi.runAllTimers(); expect(controller.getState()).toMatchObject({ @@ -3270,4 +3329,629 @@ describe("indexed DOM row layout controller", () => { ).toThrow(); expect(controller.getState()).toBe(before); }); + + describe("sort-only reorder permutation path", () => { + const tenRows = Array.from({ length: 10 }, (_, index) => ({ + id: index + 1, + team: "A", + score: index + 1, + label: "r", + })); + // Per-row distinct measured heights: 41..50 by id. The oracle tests below + // measure EVERY row so both controllers run on measurements alone — + // estimates are where the two paths legitimately differ (the permutation + // carries refined estimates for rows that have left the viewport, while a + // full replacement re-derives offscreen rows from the default), so an + // estimate-bearing fixture would flag that intended difference as a + // defect. Distinct heights make every rank's offset sensitive to the + // permutation: misplacing ANY row moves the table. + const measuredHeightOf = (rowId: number): number => 40 + rowId; + const allMeasurements: ReadonlyArray = + tenRows.map((row) => [row.id, measuredHeightOf(row.id)] as const); + const descQuery = { + filters: [], + sort: [{ columnId: "score" as const, direction: "desc" as const }], + rowGroups: [], + }; + + // `diagnostics` is a public property of layout-core's concrete index but + // deliberately absent from the `RowHeightIndex` interface, and layout-core's + // own seam (`getRowHeightIndexDiagnosticsForTesting`) is a direct-module + // export the barrel alias in `vitest.config.ts` cannot reach. The + // structural cast reads the same frozen object the seam returns. + const heightIndexDiagnostics = ( + index: unknown, + ): { reorderEntriesReused: number; reorderEntriesRemeasured: number } => + ( + index as { + diagnostics: { + reorderEntriesReused: number; + reorderEntriesRemeasured: number; + }; + } + ).diagnostics; + + /** + * The replacement oracle: an identical controller whose model reports the + * same commit as a `"bulk-replace"` reset, so the cooperative replacement + * path — the pre-permutation behavior — produces the reference state. + */ + function createReplacementOracle( + rows: readonly Row[], + measurements: ReadonlyArray = [], + viewport?: { + scrollTop: number; + viewportHeight: number; + overscan: number; + }, + ) { + const model = createModel(rows); + const ready = createReadyController(model); + for (const [rowId, height] of measurements) { + ready.controller.measure(data(rowId), height); + } + if (viewport !== undefined) ready.controller.setViewport(viewport); + const realChangesSince = model.changesSince.bind(model); + vi.spyOn(model, "changesSince").mockImplementation((revision) => { + const sequence = realChangesSince(revision); + return sequence.kind === "reset" + ? { ...sequence, reason: "bulk-replace" as const } + : sequence; + }); + return { model, ...ready }; + } + + test("a sort-only commit permutes existing heights without a replacement", () => { + const model = createModel(tenRows); + const { controller } = createReadyController(model); + for (const [rowId, height] of allMeasurements) { + controller.measure(data(rowId), height); + } + const before = controller.getState(); + const reorderSpy = vi.spyOn(before.rowHeights, "reorder"); + const beforeDiagnostics = + getRowLayoutControllerDiagnosticsForTesting(controller); + + model.setQuery(descQuery); + + // Synchronous: ready again with no scheduler flush and no replacement. + const after = controller.getState(); + expect(after.status.kind).toBe("ready"); + expect(after.observedRevision).toBe(model.getState().snapshot.revision); + const diagnostics = + getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.replacementStartCount).toBe( + beforeDiagnostics.replacementStartCount, + ); + expect(diagnostics.reorderPathCount).toBe( + beforeDiagnostics.reorderPathCount + 1, + ); + expect(diagnostics.reorderFallbackCount).toBe( + beforeDiagnostics.reorderFallbackCount, + ); + expect(reorderSpy).toHaveBeenCalledTimes(1); + const permuted = reorderSpy.mock.results[0]!.value as unknown; + expect(after.rowHeights).toBe(permuted); + expect(heightIndexDiagnostics(permuted).reorderEntriesReused).toBe( + tenRows.length, + ); + + // Equivalence oracle: the published rank -> offset table is exactly what + // a full replacement over the same target produces. + const oracle = createReplacementOracle(tenRows, allMeasurements); + oracle.model.setQuery(descQuery); + oracle.scheduler.flushAll(); + const reference = oracle.controller.getState(); + expect(reference.status.kind).toBe("ready"); + expect(after.rowHeights.rowCount).toBe(reference.rowHeights.rowCount); + const rankOffsets = ( + heights: (typeof after)["rowHeights"], + ): readonly number[] => + Array.from({ length: tenRows.length }, (_, rank) => + heights.getOffsetForIndex(rank), + ); + expect(rankOffsets(after.rowHeights)).toEqual( + rankOffsets(reference.rowHeights), + ); + expect(after.totalHeight).toBe(reference.totalHeight); + // The measured heights moved with their rows, by identity. + for (const [rowId, height] of allMeasurements) { + expect(after.rowHeights.hasMeasurement(data(rowId))).toBe(true); + expect( + after.rowHeights.getHeight( + model.getState().snapshot.indexOf(data(rowId)), + ), + ).toBe(height); + } + }); + + test("restores the scroll anchor exactly as the replacement path does", () => { + const model = createModel(tenRows); + const { controller } = createReadyController(model); + for (const [rowId, height] of allMeasurements) { + controller.measure(data(rowId), height); + } + // Anchor inside row 4 (ascending rank 3, offsets 126..170 under the + // 41..50 measured heights): 4px below its top. Descending moves row 4 to + // rank 6, whose offset is 50+49+48+47+46+45 = 285, so an anchored + // viewport lands at 289 — a position the un-anchored scrollTop (130) and + // the identity permutation (offset 126) both miss. + controller.setViewport({ + scrollTop: 130, + viewportHeight: 88, + overscan: 0, + }); + expect(controller.getState().scrollTop).toBe(130); + + model.setQuery(descQuery); + + const after = controller.getState(); + expect(after.status.kind).toBe("ready"); + expect(after.scrollTop).toBe(289); + + const oracle = createReplacementOracle(tenRows, allMeasurements, { + scrollTop: 130, + viewportHeight: 88, + overscan: 0, + }); + oracle.model.setQuery(descQuery); + oracle.scheduler.flushAll(); + expect(oracle.controller.getState().scrollTop).toBe(after.scrollTop); + }); + + test('a "bulk-replace" reset still takes the replacement path', () => { + const oracle = createReplacementOracle(tenRows); + const before = getRowLayoutControllerDiagnosticsForTesting( + oracle.controller, + ); + + oracle.model.setQuery(descQuery); + oracle.scheduler.flushAll(); + + const state = oracle.controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.snapshot?.rowAt(0)).toMatchObject({ rowId: 10 }); + const diagnostics = getRowLayoutControllerDiagnosticsForTesting( + oracle.controller, + ); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount + 1, + ); + expect(diagnostics.reorderPathCount).toBe(before.reorderPathCount); + expect(diagnostics.reorderFallbackCount).toBe( + before.reorderFallbackCount, + ); + }); + + test("a reorder reset with a misaligned revision falls back to replacement", () => { + const model = createModel(tenRows); + const { controller, scheduler } = createReadyController(model); + const before = getRowLayoutControllerDiagnosticsForTesting(controller); + vi.spyOn(model, "changesSince").mockImplementation((revision) => ({ + kind: "reset" as const, + // One short of the committed revision: the range this reset claims to + // cover does not reach the snapshot the controller is looking at. + toRevision: revision, + reason: "reorder" as const, + })); + + model.setQuery(descQuery); + scheduler.flushAll(); + + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.snapshot?.rowAt(0)).toMatchObject({ rowId: 10 }); + expect(state.observedRevision).toBe(model.getState().snapshot.revision); + const diagnostics = + getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount + 1, + ); + expect(diagnostics.reorderPathCount).toBe(before.reorderPathCount); + expect(diagnostics.reorderFallbackCount).toBe( + before.reorderFallbackCount + 1, + ); + }); + + test("a reorder() throw falls back to replacement without publishing an error", () => { + const model = createModel(tenRows); + const { controller, scheduler } = createReadyController(model); + const before = getRowLayoutControllerDiagnosticsForTesting(controller); + const statuses: string[] = []; + controller.subscribe(() => { + statuses.push(controller.getState().status.kind); + }); + const realChangesSince = model.changesSince.bind(model); + let lie = false; + vi.spyOn(model, "changesSince").mockImplementation((revision) => { + if (!lie) return realChangesSince(revision); + // A perfectly aligned reorder reset over a commit that ADDED a row: + // the target's rowCount no longer matches the index, so `reorder` + // itself rejects the permutation contract. + return { + kind: "reset" as const, + toRevision: model.getState().snapshot.revision, + reason: "reorder" as const, + }; + }); + + lie = true; + model.applyTransaction({ + add: [{ id: 11, team: "B", score: 11, label: "row 11" }], + }); + scheduler.flushAll(); + + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(statuses).not.toContain("error"); + expect(state.rowHeights.rowCount).toBe(11); + expect(state.observedRevision).toBe(model.getState().snapshot.revision); + const diagnostics = + getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount + 1, + ); + expect(diagnostics.reorderPathCount).toBe(before.reorderPathCount); + expect(diagnostics.reorderFallbackCount).toBe( + before.reorderFallbackCount + 1, + ); + + // The published state matches what a plain replacement produces. + const oracle = createReplacementOracle(tenRows); + oracle.model.applyTransaction({ + add: [{ id: 11, team: "B", score: 11, label: "row 11" }], + }); + oracle.scheduler.flushAll(); + const reference = oracle.controller.getState(); + expect(state.totalHeight).toBe(reference.totalHeight); + for (let rank = 0; rank < 11; rank += 1) { + expect(state.rowHeights.getOffsetForIndex(rank)).toBe( + reference.rowHeights.getOffsetForIndex(rank), + ); + } + }); + + test("a reorder arriving mid-replacement composes into the replacement", () => { + const model = createModel(tenRows); + const { controller, scheduler } = createReadyController(model); + // Retained state keeps the reset cooperative, so the reorder below + // really does arrive MID-replacement. + controller.measure(data(2), 77); + model.setRows( + Array.from({ length: 600 }, (_, index) => ({ + id: index + 1, + team: "A", + score: index + 1, + label: `reset ${index + 1}`, + })), + ); + expect(controller.getState().status.kind).toBe("rebuilding"); + + // The sort-only fast path publishes its reorder barrier while the + // controller is mid-replacement; the active-replacement flow owns it. + model.setQuery(descQuery); + expect( + getRowLayoutControllerDiagnosticsForTesting(controller) + .reorderPathCount, + ).toBe(0); + scheduler.flushAll(); + + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.snapshot?.rowAt(0)).toMatchObject({ rowId: 600 }); + expect(state.observedRevision).toBe(model.getState().snapshot.revision); + expect( + getRowLayoutControllerDiagnosticsForTesting(controller) + .reorderPathCount, + ).toBe(0); + expect( + getRowLayoutControllerDiagnosticsForTesting(controller) + .reorderComposeCount, + ).toBe(1); + }); + + test("a permutation reuses every entry and re-measures none", () => { + const model = createModel(tenRows); + const { controller } = createReadyController(model); + controller.measure(data(2), 77); + const heights = controller.getState().rowHeights; + const reorderSpy = vi.spyOn(heights, "reorder"); + + model.setQuery(descQuery); + + expect(reorderSpy).toHaveBeenCalledTimes(1); + const diagnostics = heightIndexDiagnostics( + reorderSpy.mock.results[0]!.value, + ); + expect(diagnostics.reorderEntriesReused).toBe(tenRows.length); + expect(diagnostics.reorderEntriesRemeasured).toBe(0); + }); + + describe("composition into an active replacement", () => { + // Same ids and scores, different labels: a reset commit whose + // replacement retains every measurement by key, so the composed finish + // and the from-scratch oracle both run on measurements alone. + const relabeledRows = tenRows.map((row) => ({ ...row, label: "x" })); + const ascQuery = { + filters: [], + sort: [{ columnId: "score" as const, direction: "asc" as const }], + rowGroups: [], + }; + + /** + * A model with a fully measured base and a relabeling `setRows` reset + * mid-flight: retained state keeps the replacement cooperative (C2a), + * so with a manual scheduler it is genuinely ACTIVE until the flush. + */ + function beginMeasuredReplacement() { + const model = createModel(tenRows); + const { controller, scheduler } = createReadyController(model); + for (const [rowId, height] of allMeasurements) { + controller.measure(data(rowId), height); + } + model.setRows(relabeledRows); + expect(controller.getState().status.kind).toBe("rebuilding"); + return { model, controller, scheduler }; + } + + test("a mid-replacement reorder composes at finish without a restart", () => { + const { model, controller, scheduler } = beginMeasuredReplacement(); + const before = getRowLayoutControllerDiagnosticsForTesting(controller); + + model.setQuery(descQuery); + // Staged while the reorder is pending: must survive into the final + // index at the row's FINAL rank. + controller.measure(data(5), 95); + scheduler.flushAll(); + + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.snapshot?.rowAt(0)).toMatchObject({ rowId: 10 }); + expect(state.observedRevision).toBe(model.getState().snapshot.revision); + const diagnostics = + getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount, + ); + expect(diagnostics.reorderComposeCount).toBe( + before.reorderComposeCount + 1, + ); + expect(diagnostics.reorderComposeFallbackCount).toBe( + before.reorderComposeFallbackCount, + ); + expect(diagnostics.reorderPathCount).toBe(before.reorderPathCount); + + // Staged measurement survived composition, at the final rank. + expect(state.rowHeights.hasMeasurement(data(5))).toBe(true); + const rankOf5 = model.getState().snapshot.indexOf(data(5)); + expect(state.rowHeights.getHeight(rankOf5)).toBe(95); + + // From-scratch oracle: replacement over the same rows, measurements + // (the 95 overwrite included), and final sort. + const oracle = createReplacementOracle(relabeledRows, [ + ...allMeasurements, + [5, 95], + ]); + oracle.model.setQuery(descQuery); + oracle.scheduler.flushAll(); + const reference = oracle.controller.getState(); + expect(reference.status.kind).toBe("ready"); + const rankOffsets = ( + heights: (typeof state)["rowHeights"], + ): readonly number[] => + Array.from({ length: tenRows.length }, (_, rank) => + heights.getOffsetForIndex(rank), + ); + expect(rankOffsets(state.rowHeights)).toEqual( + rankOffsets(reference.rowHeights), + ); + expect(state.totalHeight).toBe(reference.totalHeight); + }); + + test("changes after a pending reorder fail closed to a restart", () => { + const { model, controller, scheduler } = beginMeasuredReplacement(); + const before = getRowLayoutControllerDiagnosticsForTesting(controller); + + model.setQuery(descQuery); + model.applyTransaction({ + update: [{ id: 7, changes: { label: "changed" } }], + }); + scheduler.flushAll(); + + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.snapshot?.rowAt(0)).toMatchObject({ rowId: 10 }); + expect(state.observedRevision).toBe(model.getState().snapshot.revision); + const diagnostics = + getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount + 1, + ); + expect(diagnostics.reorderComposeCount).toBe( + before.reorderComposeCount, + ); + expect(diagnostics.reorderComposeFallbackCount).toBe( + before.reorderComposeFallbackCount + 1, + ); + }); + + test("a newer aligned reorder replaces the pending one — last wins", () => { + const { model, controller, scheduler } = beginMeasuredReplacement(); + const before = getRowLayoutControllerDiagnosticsForTesting(controller); + + model.setQuery(descQuery); + model.setQuery(ascQuery); + scheduler.flushAll(); + + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + // The SECOND sort's order. Dropping the second retarget would publish + // descending (rowId 10 first); restarting would bump the counter. + expect(state.snapshot?.rowAt(0)).toMatchObject({ rowId: 1 }); + expect(state.observedRevision).toBe(model.getState().snapshot.revision); + const diagnostics = + getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount, + ); + expect(diagnostics.reorderComposeCount).toBe( + before.reorderComposeCount + 1, + ); + expect(diagnostics.reorderComposeFallbackCount).toBe( + before.reorderComposeFallbackCount, + ); + }); + + test("a compose-time reorder() throw restarts without publishing an error", () => { + const { model, controller, scheduler } = beginMeasuredReplacement(); + const before = getRowLayoutControllerDiagnosticsForTesting(controller); + const statuses: string[] = []; + controller.subscribe(() => { + statuses.push(controller.getState().status.kind); + }); + const realChangesSince = model.changesSince.bind(model); + let lie = false; + vi.spyOn(model, "changesSince").mockImplementation((revision) => { + if (!lie) return realChangesSince(revision); + // A perfectly aligned reorder reset over a commit that ADDED a + // row: the retarget is accepted, and the compose-time `reorder()` + // rejects the permutation contract (11 rows over a 10-row + // candidate). + return { + kind: "reset" as const, + toRevision: model.getState().snapshot.revision, + reason: "reorder" as const, + }; + }); + + lie = true; + model.applyTransaction({ + add: [{ id: 11, team: "B", score: 11, label: "x" }], + }); + lie = false; + scheduler.flushAll(); + + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(statuses).not.toContain("error"); + expect(state.rowHeights.rowCount).toBe(11); + expect(state.observedRevision).toBe(model.getState().snapshot.revision); + const diagnostics = + getRowLayoutControllerDiagnosticsForTesting(controller); + expect(diagnostics.replacementStartCount).toBe( + before.replacementStartCount + 1, + ); + expect(diagnostics.reorderComposeCount).toBe( + before.reorderComposeCount, + ); + expect(diagnostics.reorderComposeFallbackCount).toBe( + before.reorderComposeFallbackCount + 1, + ); + }); + + test("a composed finish restores the anchor against the final order", () => { + const model = createModel(tenRows); + const { controller, scheduler } = createReadyController(model); + for (const [rowId, height] of allMeasurements) { + controller.measure(data(rowId), height); + } + // Same geometry as the B4 anchor test: anchor inside row 4 + // (ascending rank 3, offsets 126..170 under the 41..50 measured + // heights), 4px below its top; descending puts row 4 at rank 6 + // (offset 285), so the anchored viewport lands at 289. + controller.setViewport({ + scrollTop: 130, + viewportHeight: 88, + overscan: 0, + }); + expect(controller.getState().scrollTop).toBe(130); + + model.setRows(relabeledRows); + expect(controller.getState().status.kind).toBe("rebuilding"); + model.setQuery(descQuery); + scheduler.flushAll(); + + const after = controller.getState(); + expect(after.status.kind).toBe("ready"); + expect(after.scrollTop).toBe(289); + // Restart-based handling would land on the same scrollTop, so pin + // that this scenario really went through the compose path. + expect( + getRowLayoutControllerDiagnosticsForTesting(controller) + .reorderComposeCount, + ).toBe(1); + + const oracle = createReplacementOracle(relabeledRows, allMeasurements, { + scrollTop: 130, + viewportHeight: 88, + overscan: 0, + }); + oracle.model.setQuery(descQuery); + oracle.scheduler.flushAll(); + expect(oracle.controller.getState().scrollTop).toBe(after.scrollTop); + }); + }); + }); +}); + +describe("synchronous bulk mount", () => { + const mountRows = (count: number): Row[] => + Array.from({ length: count }, (_, index) => ({ + id: index, + team: index % 2 === 0 ? "A" : "B", + score: index, + label: `row ${index}`, + })); + + test("a mount over the eager limit publishes during activation's synchronize pass", () => { + // 1000 rows, far over the 32-row `eagerInitialRowLimit` that used to be + // the only synchronous mount path. The base index holds no retained state + // (nothing has ever been measured), so the replacement builds in one bulk + // pass and publishes before the constructor returns — no scheduler entry, + // no blank "rebuilding" frame. + const model = createModel(mountRows(1_000)); + const scheduler = new ManualScheduler(); + const controller = createRowLayoutController({ + model, + columns: renderColumns, + viewport: { scrollTop: 0, viewportHeight: 88, overscan: 1 }, + scheduler, + now: () => 0, + budgetMs: 5, + maxUnitsPerSlice: 256, + }); + + expect(scheduler.tasks.length).toBe(0); + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.observedRevision).toBe(model.getState().snapshot.revision); + expect(state.window.length).toBeGreaterThan(0); + expect(state.rowHeights.rowCount).toBe(1_000); + expect( + getRowLayoutControllerDiagnosticsForTesting(controller) + .scheduledCallbackCount, + ).toBe(0); + }); + + test("a replacement over a measured base still slices cooperatively", () => { + // Pins the unchanged behavior: retained state (one DOM measurement) keeps + // the cooperative path, because the bulk rebuild would discard it. + const model = createModel(mountRows(6)); + const { controller, scheduler } = createReadyController(model); + controller.measure(data(2), 91); + expect(controller.getState().rowHeights.hasRetainedState).toBe(true); + + controller.setColumns([ + { id: "label", wrap: true, widthPx: 120 }, + { id: "score", widthPx: 80 }, + ]); + + expect(controller.getState().status.kind).toBe("rebuilding"); + expect(scheduler.tasks.length).toBeGreaterThan(0); + scheduler.flushAll(); + + const state = controller.getState(); + expect(state.status.kind).toBe("ready"); + expect(state.rowHeights.hasMeasurement(data(2))).toBe(true); + }); }); diff --git a/packages/renderer-dom/src/row-layout-controller.ts b/packages/renderer-dom/src/row-layout-controller.ts index 70d04580c..d4e3743b7 100644 --- a/packages/renderer-dom/src/row-layout-controller.ts +++ b/packages/renderer-dom/src/row-layout-controller.ts @@ -5,6 +5,7 @@ import { type RowHeightIndex, type RowHeightOperation, type RowHeightReplacementBuilder, + type RowHeightReplacementSource, } from "@pretable-internal/layout-core"; // One emission of the engine's declarations — see the note in `./types.ts`. import type { @@ -64,6 +65,23 @@ export interface RowLayoutControllerDiagnostics { readonly lastPublishedRangeRows: number; readonly anchorSearchUnits: number; readonly replacementStartCount: number; + /** Sort-only commits absorbed by permuting the height index in place. */ + readonly reorderPathCount: number; + /** + * Reorder resets that ended in a full replacement anyway — a misaligned + * revision, or a `reorder()` contract violation. Expected 0 on the happy + * path; a nonzero count under a bench run is a finding. + */ + readonly reorderFallbackCount: number; + /** Reorders that arrived mid-replacement and composed into its finish. */ + readonly reorderComposeCount: number; + /** + * Pending mid-replacement reorders abandoned for a restart — a later + * non-reorder wake, or a compose-time `reorder()` contract violation. + * Expected 0 on the happy path; a nonzero count under a bench run is a + * finding. + */ + readonly reorderComposeFallbackCount: number; readonly pendingCatchUpChangeSetCount: number; readonly pendingCatchUpOperationCount: number; readonly retainedCatchUpSnapshotCount: number; @@ -401,6 +419,31 @@ interface ActiveReplacement< candidate: RowHeightIndex> | undefined; searchDistance: number; searchPrevious: boolean; + /** + * A reorder reset accepted mid-replacement, composed wholesale at finish. + * + * Reorders queue NO changesets — the permutation is order-only, so the + * catch-up queue's terminus stays at `fromApplied`, the `capturedRevision` + * at which the FIRST pending reorder was accepted, while `capturedRevision` + * itself advances to the reorder's revision. Every "has catch-up drained?" + * comparison must therefore measure `appliedRevision` against `fromApplied` + * while this is set (see `catchUpTargetOf`), and staged measurements must + * resolve their indexes against `appliedTarget` — the snapshot at + * `fromApplied`, whose order the candidate is actually in — never against + * `latestTarget`, which is already permuted. + */ + pendingReorder: + | { + /** The final (permuted) snapshot `finishReplacement` composes to. */ + readonly target: PretableRowModelSnapshot; + readonly fromApplied: number; + readonly appliedTarget: PretableRowModelSnapshot< + TRow, + TRowId, + TColumns + >; + } + | undefined; } interface StagedMeasurement { @@ -560,6 +603,10 @@ export function createRowLayoutController< let lastPublishedRangeRows = 0; let anchorSearchUnits = 0; let replacementStartCount = 0; + let reorderPathCount = 0; + let reorderFallbackCount = 0; + let reorderComposeCount = 0; + let reorderComposeFallbackCount = 0; let catchUpUnits = 0; let maxCatchUpUnitsPerSlice = 0; let deferredViewportWithoutAnchor = false; @@ -969,6 +1016,18 @@ export function createRowLayoutController< return nearest; }; + /** + * The revision the catch-up queue must reach before the candidate is + * complete. A pending reorder advances `capturedRevision` WITHOUT queuing + * changesets — the permutation is applied wholesale at finish — so while + * one is set, catch-up is complete at `fromApplied`, not at + * `capturedRevision`. Compose reconciles the two before publish. + */ + const catchUpTargetOf = ( + replacement: ActiveReplacement, + ): number => + replacement.pendingReorder?.fromApplied ?? replacement.capturedRevision; + const finishReplacement = ( replacement: ActiveReplacement, resolvedAnchor: PretableVisibleRowRef | undefined, @@ -982,13 +1041,38 @@ export function createRowLayoutController< } if ( replacement.pending[replacement.pendingHead] !== undefined || - replacement.appliedRevision !== replacement.capturedRevision || + replacement.appliedRevision !== catchUpTargetOf(replacement) || replacement.capturedWakeVersion !== modelWakeVersion || stagedMeasurementHead < stagedMeasurementKeys.length ) { scheduleReplacement(replacement); return; } + const pendingReorder = replacement.pendingReorder; + if (pendingReorder !== undefined) { + // Catch-up and staged replay have drained against the pre-reorder + // order; now permute the finished candidate to the final order in one + // synchronous pass. ANY throw — the target lying about the row set, + // a hostile snapshot — falls back to a restart on the final target; + // the fallback IS the error handling, nothing publishes an error here. + let composed: RowHeightIndex>; + try { + composed = replacement.candidate.reorder( + replacementSourceOf(pendingReorder.target), + ); + } catch { + reorderComposeFallbackCount += 1; + startReplacement(replacement.latestTarget, true); + return; + } + replacement.candidate = composed; + replacement.pendingReorder = undefined; + // The candidate now IS the permuted commit: reconcile the applied + // revision so the publish gate below (and any post-publish wake) sees + // an ordinary, fully caught-up replacement. + replacement.appliedRevision = replacement.capturedRevision; + reorderComposeCount += 1; + } const candidate = replacement.candidate; const target = replacement.latestTarget; const expectedWakeVersion = replacement.capturedWakeVersion; @@ -1152,7 +1236,7 @@ export function createRowLayoutController< continue; } - if (replacement.appliedRevision !== replacement.capturedRevision) { + if (replacement.appliedRevision !== catchUpTargetOf(replacement)) { throw new CatchUpSequenceError( "The queued row-layout changes do not reach the captured revision.", ); @@ -1164,7 +1248,17 @@ export function createRowLayoutController< measurement !== undefined && measurement.appliedToken !== replacement.token ) { - const index = replacement.latestTarget.indexOf(measurement.ref); + // With a pending reorder the candidate is still in the + // PRE-reorder order (`appliedTarget`), and `measure` asserts the + // ref's identity at the index it is given — resolving against the + // already-permuted `latestTarget` would place heights on the + // wrong rank or throw. Composition relinks entries by key, so a + // measurement applied at its pre-reorder rank rides to its final + // one. + const stagedTarget = + replacement.pendingReorder?.appliedTarget ?? + replacement.latestTarget; + const index = stagedTarget.indexOf(measurement.ref); if (active !== replacement || disposed) return; stagedMeasurementHead += 1; if (index >= 0) { @@ -1322,6 +1416,29 @@ export function createRowLayoutController< } }; + /** + * The `{rowCount, entryAt}` source both re-ingest paths hand the height + * index: `startReplacement` feeds it to `beginReplacement`, and the sort-only + * permutation path feeds the SAME shape to `reorder`, so a snapshot that + * omits a visible row fails identically on either path. Reads + * `visibleRowCount` eagerly — construct inside a try. + */ + const replacementSourceOf = ( + target: PretableRowModelSnapshot, + ): RowHeightReplacementSource> => ({ + rowCount: target.visibleRowCount, + entryAt(index) { + const row = target.rowAt(index); + if (row === undefined) { + throw new RowLayoutControllerError( + "layout-failed", + `The row-model snapshot omitted visible row ${index}.`, + ); + } + return { key: rowRef(row) }; + }, + }); + const startReplacement = ( target: PretableRowModelSnapshot, shouldNotify: boolean, @@ -1333,19 +1450,7 @@ export function createRowLayoutController< let targetRevision: number; try { targetRevision = target.revision; - builder = state.rowHeights.beginReplacement({ - rowCount: target.visibleRowCount, - entryAt(index) { - const row = target.rowAt(index); - if (row === undefined) { - throw new RowLayoutControllerError( - "layout-failed", - `The row-model snapshot omitted visible row ${index}.`, - ); - } - return { key: rowRef(row) }; - }, - }); + builder = state.rowHeights.beginReplacement(replacementSourceOf(target)); } catch (error) { clearStagedMeasurements(); rollbackDeferredViewport(); @@ -1376,6 +1481,7 @@ export function createRowLayoutController< candidate: undefined, searchDistance: 1, searchPrevious: false, + pendingReorder: undefined, }; stagedMeasurementHead = 0; active = replacement; @@ -1388,8 +1494,17 @@ export function createRowLayoutController< }); if (shouldNotify) notify(); if ( - state.observedRevision === null && - target.visibleRowCount <= eagerInitialRowLimit + // A base with no retained state (no measurement, tombstone, or retention + // order — see `RowHeightIndex.hasRetainedState`) hands out a builder + // that completes in ONE `advance`, so running the slice inline is a + // synchronous burst of ~20ms at 50k rows (spec C2a's accepted trade) — + // versus a cooperatively sliced mount whose ~450ms of slices publish + // nothing, leaving the grid blank. This supersedes `eagerInitialRowLimit` + // for the no-retained-state case; the limit keeps its documented role for + // retained-state replacements, whose cooperative builders it bounds. + !state.rowHeights.hasRetainedState || + (state.observedRevision === null && + target.visibleRowCount <= eagerInitialRowLimit) ) { runReplacementSlice(replacement, true); } else { @@ -1437,6 +1552,49 @@ export function createRowLayoutController< return true; } const sequence = options.model.changesSince(replacement.capturedRevision); + if ( + sequence.kind === "reset" && + sequence.reason === "reorder" && + sequence.toRevision === targetRevision + ) { + // A sort-only commit mid-replacement: accepted as a RETARGET and + // composed wholesale at finish, instead of restarting the build. + // A newer aligned reorder simply replaces a pending one's target — + // reorders are wholesale, so only the last matters — while + // `fromApplied`/`appliedTarget` keep the FIRST acceptance's values: + // the catch-up queue still ends where it ended then. + replacement.pendingReorder = { + target, + fromApplied: + replacement.pendingReorder?.fromApplied ?? + replacement.capturedRevision, + appliedTarget: + replacement.pendingReorder?.appliedTarget ?? + replacement.latestTarget, + }; + replacement.capturedRevision = targetRevision; + replacement.capturedWakeVersion = modelWakeVersion; + replacement.latestTarget = target; + state = Object.freeze({ + ...state, + status: Object.freeze({ + kind: "rebuilding" as const, + targetRevision, + }), + }); + notify(); + return true; + } + if (replacement.pendingReorder !== undefined) { + // Conservative composition rule: a pending reorder is FINAL. Any + // revision-advancing wake other than a newer aligned reorder — + // changes, another reset reason, a misaligned reorder — fails closed + // to a restart rather than reasoning about index-based operations + // applied across a permutation. (Same-revision wakes never reach + // here; they are absorbed by the early equal-revision return.) + reorderComposeFallbackCount += 1; + return false; + } if ( sequence.kind !== "changes" || sequence.fromRevision !== replacement.capturedRevision || @@ -1479,6 +1637,42 @@ export function createRowLayoutController< return root; }; + /** + * Resolves a captured anchor into the scroll request a synchronous publish + * uses: nearest surviving ref in the new order, then the row's new offset + * plus the anchor's intra-row offset. The incremental journal path and the + * sort-only permutation path share it so their anchor semantics cannot + * drift; the cooperative replacement path implements the same resolution + * against its own staged candidate in `finishReplacement`. + */ + const restoreAnchorRequest = ( + target: PretableRowModelSnapshot, + root: RowHeightIndex>, + anchor: CapturedAnchor | undefined, + ): ScrollRequest => { + if (anchor !== undefined) { + const resolved = target.nearestVisibleRef(anchor.heightAnchor.ref); + if (resolved !== undefined) { + const index = target.indexOf(resolved); + if (index >= 0) { + return localScroll( + Math.max( + 0, + root.restoreAnchor( + { + ref: resolved, + offset: anchor.heightAnchor.offset, + }, + index, + ), + ), + ); + } + } + } + return globalScroll(viewport.scrollTop); + }; + const synchronize = (): void => { if (disposed) return; modelWakeVersion += 1; @@ -1517,6 +1711,52 @@ export function createRowLayoutController< let sequence: PretableChangeSequence; try { sequence = options.model.changesSince(state.observedRevision); + if (sequence.kind === "reset" && sequence.reason === "reorder") { + // A sort-only commit: the visible row SET and every height-relevant + // fact are unchanged, only the order moved, so the height index is + // permuted synchronously instead of re-ingested row by row. The + // reset carries no `fromRevision` — `changesSince` was called with + // `state.observedRevision`, so the range's start is pinned by the + // argument and only the target side needs to line up. + // + // No staged/pending lifecycle: this path runs only when no + // replacement is active (the `active` branch above owns everything + // else), the permutation is synchronous, and with no active + // replacement `measure` applies immediately, so the staged + // measurement queue is empty and stays untouched. + // + // ANY doubt — misaligned revision, a `reorder()` contract + // violation, a publish failure — falls back to the full + // replacement. The fallback IS the error handling; nothing here + // publishes an error state of its own. + if (sequence.toRevision === target.revision) { + try { + const anchor = deferredViewportWithoutAnchor + ? undefined + : captureAnchor(); + const root = state.rowHeights.reorder( + replacementSourceOf(target), + ); + publishReady( + target, + root, + restoreAnchorRequest(target, root, anchor), + ); + // Mirrors `finishReplacement`'s commit: a deferred viewport is + // applied by the publish above (it reads the live `viewport`), + // so the flag must not survive into the next capture. + deferredViewportWithoutAnchor = false; + reorderPathCount += 1; + } catch { + reorderFallbackCount += 1; + startReplacement(target, true); + } + } else { + reorderFallbackCount += 1; + startReplacement(target, true); + } + continue; + } if ( !validateChanges(sequence, state.observedRevision, target.revision) ) { @@ -1530,30 +1770,11 @@ export function createRowLayoutController< try { const previousAnchor = captureAnchor(); const root = applyChanges(sequence); - let request = globalScroll(viewport.scrollTop); - if (previousAnchor !== undefined) { - const resolved = target.nearestVisibleRef( - previousAnchor.heightAnchor.ref, - ); - if (resolved !== undefined) { - const index = target.indexOf(resolved); - if (index >= 0) { - request = localScroll( - Math.max( - 0, - root.restoreAnchor( - { - ref: resolved, - offset: previousAnchor.heightAnchor.offset, - }, - index, - ), - ), - ); - } - } - } - publishReady(target, root, request); + publishReady( + target, + root, + restoreAnchorRequest(target, root, previousAnchor), + ); } catch (error) { if (error instanceof RowLayoutControllerError) { publishError( @@ -1841,6 +2062,10 @@ export function createRowLayoutController< lastPublishedRangeRows, anchorSearchUnits, replacementStartCount, + reorderPathCount, + reorderFallbackCount, + reorderComposeCount, + reorderComposeFallbackCount, pendingCatchUpChangeSetCount: active?.pendingChangeSetCount ?? 0, pendingCatchUpOperationCount: active?.pendingOperationCount ?? 0, retainedCatchUpSnapshotCount: retainedSnapshots.size, diff --git a/packages/row-model/src/__tests__/change-journal.test.ts b/packages/row-model/src/__tests__/change-journal.test.ts index a031d2f5b..ff4de5c62 100644 --- a/packages/row-model/src/__tests__/change-journal.test.ts +++ b/packages/row-model/src/__tests__/change-journal.test.ts @@ -476,6 +476,79 @@ describe("bounded revision change journal", () => { }); }); + test('a range of only "reorder" barriers resets with reason "reorder"', () => { + const journal = createChangeJournal(4); + journal.appendBarrier(0, 1, "reorder"); + + expect(journal.changesSince(0, 1)).toEqual({ + kind: "reset", + toRevision: 1, + reason: "reorder", + }); + + journal.appendBarrier(1, 2, "reorder"); + expect(journal.changesSince(0, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "reorder", + }); + // A sub-range that is still all-reorder reports "reorder" too. + expect(journal.changesSince(1, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "reorder", + }); + }); + + test('a changes entry in the range demotes "reorder" to "bulk-replace" (both orders)', () => { + const changesFirst = createChangeJournal(4); + changesFirst.appendChanges(0, 1, [ + { kind: "insert", ref: data(1), index: 0 }, + ]); + changesFirst.appendBarrier(1, 2, "reorder"); + expect(changesFirst.changesSince(0, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "bulk-replace", + }); + + const reorderFirst = createChangeJournal(4); + reorderFirst.appendBarrier(0, 1, "reorder"); + reorderFirst.appendChanges(1, 2, [ + { kind: "insert", ref: data(1), index: 0 }, + ]); + expect(reorderFirst.changesSince(0, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "bulk-replace", + }); + // Resuming from PAST the reorder barrier replays the plain changes. + expect(reorderFirst.changesSince(1, 2)).toMatchObject({ + kind: "changes", + changes: [{ previousRevision: 1, revision: 2 }], + }); + }); + + test('a non-"reorder" barrier in the range wins over "reorder" (both orders)', () => { + const reorderFirst = createChangeJournal(4); + reorderFirst.appendBarrier(0, 1, "reorder"); + reorderFirst.appendBarrier(1, 2); + expect(reorderFirst.changesSince(0, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "bulk-replace", + }); + + const barrierFirst = createChangeJournal(4); + barrierFirst.appendBarrier(0, 1); + barrierFirst.appendBarrier(1, 2, "reorder"); + expect(barrierFirst.changesSince(0, 2)).toEqual({ + kind: "reset", + toRevision: 2, + reason: "bulk-replace", + }); + }); + test("rejects malformed or non-contiguous append pairs without changing retained state", () => { const journal = createChangeJournal(1); journal.appendChanges(0, 1, [{ kind: "insert", ref: data(1), index: 0 }]); @@ -537,9 +610,12 @@ describe("bounded revision change journal", () => { reason: "bulk-replace", }); + // A FILTER change: a sort-only change would take the synchronous fast + // path and journal a "reorder" barrier instead (pinned in + // sort-fast-path.test.ts). const query = flat.setQuery({ - filters: [], - sort: [{ columnId: "score", direction: "desc" }], + filters: [{ columnId: "team", operator: "equals", value: "Z" }], + sort: [{ columnId: "score", direction: "asc" }], rowGroups: [], }); await query.finished; diff --git a/packages/row-model/src/__tests__/compiled-query.test.ts b/packages/row-model/src/__tests__/compiled-query.test.ts index 929cff79c..e9abbf11e 100644 --- a/packages/row-model/src/__tests__/compiled-query.test.ts +++ b/packages/row-model/src/__tests__/compiled-query.test.ts @@ -1,10 +1,12 @@ import { describe, expect, test, vi } from "vitest"; import { + compareRecordRows, CompiledQueryComparatorError, CompiledQueryValidationError, compileQuery, createColumnHelper, + sortKeysOf, type CompiledAggregateLeaf, type PretableAggregator, type PretableQueryFor, @@ -89,11 +91,12 @@ describe("compileQuery", () => { sourceOrder: 3, filterPasses: true, groupPath: [{ columnId: "sector", value: "Tech" }], - sortKeys: [ - { columnId: "quantity", value: 20 }, - { columnId: "label", value: "item 2" }, - ], }); + // Sort keys live in the plan's store, not on metadata. + expect(sortKeysOf(plan, metadata)).toEqual([ + { columnId: "quantity", value: 20 }, + { columnId: "label", value: "item 2" }, + ]); expect(metadata.aggregateLeaves.map((leaf) => leaf.columnId)).toEqual([ "quantity", "label", @@ -212,7 +215,9 @@ describe("compileQuery", () => { ]; expect( - numeric.sort(quantityPlan.compareRows).map((row) => row.rowId), + numeric + .sort((left, right) => compareRecordRows(quantityPlan, left, right)) + .map((row) => row.rowId), ).toEqual([3, 2, 1, 4]); const labelPlan = compileQuery({ @@ -231,7 +236,9 @@ describe("compileQuery", () => { }), ); expect( - labels.sort(labelPlan.compareRows).map((row) => row.row.label), + labels + .sort((left, right) => compareRecordRows(labelPlan, left, right)) + .map((row) => row.row.label), ).toEqual(["Item 2", "item 2", "item 10"]); }); @@ -274,10 +281,10 @@ describe("compileQuery", () => { row: { id: 2, sector: null, quantity: null, label: "", ignored: "" }, }); - expect(defaultLast.compareRows(defined, missing)).toBeLessThan(0); - expect(nullFirst.compareRows(firstDefined, firstMissing)).toBeGreaterThan( - 0, - ); + expect(compareRecordRows(defaultLast, defined, missing)).toBeLessThan(0); + expect( + compareRecordRows(nullFirst, firstDefined, firstMissing), + ).toBeGreaterThan(0); expect( defaultLast.compareGroupKeys( 0, @@ -829,7 +836,7 @@ describe("compileQuery", () => { }); let caught: unknown; try { - plan.compareRows(left, right); + compareRecordRows(plan, left, right); } catch (error) { caught = error; } @@ -1224,7 +1231,7 @@ describe("compileQuery", () => { for (const invalid of ["invalid", Number.NaN]) { result = invalid; - expect(() => plan.compareRows(left, right)).toThrowError( + expect(() => compareRecordRows(plan, left, right)).toThrowError( expect.objectContaining({ name: "CompiledQueryComparatorError", columnId: "value", @@ -1233,7 +1240,7 @@ describe("compileQuery", () => { ); } result = Number.POSITIVE_INFINITY; - expect(plan.compareRows(left, right)).toBe(Number.POSITIVE_INFINITY); + expect(compareRecordRows(plan, left, right)).toBe(Number.POSITIVE_INFINITY); }); test("includes group values when a custom group comparator returns NaN", () => { diff --git a/packages/row-model/src/__tests__/order-statistic-tree.test.ts b/packages/row-model/src/__tests__/order-statistic-tree.test.ts index ad0161f7c..f5729c22c 100644 --- a/packages/row-model/src/__tests__/order-statistic-tree.test.ts +++ b/packages/row-model/src/__tests__/order-statistic-tree.test.ts @@ -1,8 +1,10 @@ import { describe, expect, test } from "vitest"; import { PoisonedTransientOrderStatisticTreeError, + compareOrderStatisticTreeIds, createDeferredMeasureTransientOrderStatisticTree, createOrderStatisticTree, + createOrderStatisticTreeFromSortedEntries, getOrderStatisticTreeDiagnosticsForTesting, type OrderStatisticTree, type OrderStatisticTreeNodeDiagnostic, @@ -584,3 +586,143 @@ describe("OrderStatisticTree", () => { } }); }); + +describe("createOrderStatisticTreeFromSortedEntries", () => { + const compositeCompare = (left: Item, right: Item) => + left.score - right.score || compareOrderStatisticTreeIds(left.id, right.id); + + function sortedEntries(count: number): Item[] { + const entries = Array.from({ length: count }, (_, id) => + item(id, adversarialOrder(id) % 97, (id % 13) + 1), + ); + entries.sort(compositeCompare); + return entries; + } + + test("matches incremental construction observably", () => { + const entries = sortedEntries(1_000); + const bulk = createOrderStatisticTreeFromSortedEntries( + createTree(), + entries, + ); + const incremental = entries.reduce( + (tree, entry) => tree.insertOrReplace(entry), + createTree(), + ); + + expect(bulk.size).toBe(incremental.size); + for (let rank = 0; rank < entries.length; rank += 25) { + const entry = incremental.entryAt(rank)!; + expect(bulk.entryAt(rank)).toBe(entry); + expect(bulk.rankOf(entry.id)).toBe(rank); + } + expect(bulk.entryAt(entries.length - 1)).toBe( + incremental.entryAt(entries.length - 1), + ); + expect(bulk.measure).toBe(incremental.measure); + }); + + test("builds an AVL-balanced tree at awkward sizes", () => { + for (const size of [0, 1, 2, 3, 7, 8, 9, 1_000]) { + const bulk = createOrderStatisticTreeFromSortedEntries( + createTree(), + sortedEntries(size), + ); + const diagnostics = getOrderStatisticTreeDiagnosticsForTesting(bulk); + expect(diagnostics.balanced).toBe(true); + expect(diagnostics.count).toBe(size); + } + }); + + test("supports later incremental mutation", () => { + const entries = sortedEntries(100); + let tree = createOrderStatisticTreeFromSortedEntries(createTree(), entries); + + const inserted = item("zzz-new", -1, 4); + tree = tree.insertOrReplace(inserted); + expect(tree.size).toBe(101); + expect(tree.rankOf("zzz-new")).toBe(0); + expect(tree.entryAt(0)).toBe(inserted); + + const victim = entries[50]!; + tree = tree.remove(victim.id); + expect(tree.size).toBe(100); + expect(tree.get(victim.id)).toBeUndefined(); + expect(getOrderStatisticTreeDiagnosticsForTesting(tree).balanced).toBe( + true, + ); + }); + + test("throws TypeError on comparator-order violations", () => { + expect(() => + createOrderStatisticTreeFromSortedEntries(createTree(), [ + item("a", 1), + item("b", 3), + item("c", 2), + ]), + ).toThrow(TypeError); + }); + + test("throws when equal-compare entries have misordered IDs", () => { + const first = item("beta", 5); + const second = item("alpha", 5); + expect(compareOrderStatisticTreeIds(first.id, second.id)).toBeGreaterThan( + 0, + ); + expect(() => + createOrderStatisticTreeFromSortedEntries(createTree(), [first, second]), + ).toThrow(TypeError); + }); + + test("throws on duplicate IDs", () => { + const duplicate = item("alpha", 1); + expect(() => + createOrderStatisticTreeFromSortedEntries(createTree(), [ + duplicate, + duplicate, + ]), + ).toThrow(TypeError); + }); + + test("builds an empty tree from empty input", () => { + const bulk = createOrderStatisticTreeFromSortedEntries(createTree(), []); + expect(bulk.size).toBe(0); + expect(bulk.measure).toBe(0); + expect([...bulk.entries()]).toEqual([]); + }); + + test("throws TypeError for a foreign tree object", () => { + const foreign = { + size: 0, + measure: 0, + } as unknown as OrderStatisticTree; + expect(() => + createOrderStatisticTreeFromSortedEntries(foreign, []), + ).toThrow(TypeError); + }); + + test("caches ordered noncommutative measures identically to incremental", () => { + const options = { + getId: (entry: Item) => entry.id, + compare: (left: Item, right: Item) => left.score - right.score, + measure: { + empty: "", + fromEntry: (entry: Item) => `${entry.label}|`, + combine: (left: string, right: string) => left + right, + }, + }; + const entries = sortedEntries(63); + const bulk = createOrderStatisticTreeFromSortedEntries( + createOrderStatisticTree(options), + entries, + ); + const incremental = entries.reduce( + (tree, entry) => tree.insertOrReplace(entry), + createOrderStatisticTree(options), + ); + expect(bulk.measure).toBe(incremental.measure); + expect(bulk.measure).toBe( + entries.map((entry) => `${entry.label}|`).join(""), + ); + }); +}); diff --git a/packages/row-model/src/__tests__/properties.test.ts b/packages/row-model/src/__tests__/properties.test.ts index 010083bf3..2af7efdaf 100644 --- a/packages/row-model/src/__tests__/properties.test.ts +++ b/packages/row-model/src/__tests__/properties.test.ts @@ -617,17 +617,45 @@ describe("incremental row-model properties", () => { async ({ rows, first, second, concurrent }) => { const scheduler = new ManualScheduler(); const model = propertyModel(rows, initialQuery, "sum", scheduler); - const firstTransition = model.setQuery(first); - const firstOutcome = firstTransition.finished.catch( - (error: unknown) => error, - ); - const secondTransition = model.setQuery(second); const state: PropertyMachineState = { rows: [...rows], query: initialQuery, derivations: "sum", revision: 0, }; + const sameJson = (a: unknown, b: unknown) => + JSON.stringify(a) === JSON.stringify(b); + // Mirrors the #457 fast path: a sort-only change on an ungrouped + // query commits synchronously, so its revision must be accounted + // BEFORE the concurrent mutations assert their previousRevision. + const commitsSynchronously = ( + from: PropertyQuery, + to: PropertyQuery, + ) => + !sameJson(from.sort, to.sort) && + sameJson(from.filters, to.filters) && + sameJson(from.rowGroups, to.rowGroups) && + to.rowGroups.length === 0; + let committed: PropertyQuery = initialQuery; + let cooperativePending = false; + const firstTransition = model.setQuery(first); + const firstOutcome = firstTransition.finished.catch( + (error: unknown) => error, + ); + if (commitsSynchronously(committed, first)) { + committed = first; + state.revision += 1; + } else if (!sameJson(committed, first)) { + cooperativePending = true; + } + const secondTransition = model.setQuery(second); + if (commitsSynchronously(committed, second)) { + committed = second; + state.revision += 1; + cooperativePending = false; + } else { + cooperativePending = !sameJson(committed, second); + } for (const operation of concurrent) { await applyPropertyOperation(model, state, operation, scheduler); } @@ -635,9 +663,7 @@ describe("incremental row-model properties", () => { await firstOutcome; await secondTransition.finished; state.query = second; - if (JSON.stringify(second) !== JSON.stringify(initialQuery)) { - state.revision += 1; - } + if (cooperativePending) state.revision += 1; assertPropertySnapshot(model, state.rows, state.revision, second); const reference = propertyModel(state.rows, second, "sum"); diff --git a/packages/row-model/src/__tests__/query-delta.test.ts b/packages/row-model/src/__tests__/query-delta.test.ts new file mode 100644 index 000000000..b4340aeb3 --- /dev/null +++ b/packages/row-model/src/__tests__/query-delta.test.ts @@ -0,0 +1,246 @@ +import { describe, expect, test } from "vitest"; + +import { + compileQuery, + createColumnHelper, + isSortOnlyChange, + type PretableQueryFor, +} from "../index"; + +interface Holding { + id: string; + sector: string; + customer: string; + quantity: number; +} + +const helper = createColumnHelper(); +const columns = [ + helper.accessor("sector", { type: "text" }), + helper.accessor("customer", { type: "text" }), + helper.accessor("quantity", { type: "number", aggregate: "sum" }), +] as const; + +/** + * Checks a query literal against a column tuple, exactly as + * `compiled-query.test.ts` does. `PretableQueryFor` is not an + * inference site, so the tuple type is named once here. + */ +function queryFor( + value: PretableQueryFor, +): PretableQueryFor { + return value; +} + +const ASC_QUANTITY = queryFor({ + filters: [], + sort: [{ columnId: "quantity", direction: "asc" }], + rowGroups: [], +}); + +const DESC_QUANTITY = queryFor({ + filters: [], + sort: [{ columnId: "quantity", direction: "desc" }], + rowGroups: [], +}); + +describe("isSortOnlyChange", () => { + test("true when only the sort differs", () => { + const previous = compileQuery({ + derivations: columns, + query: ASC_QUANTITY, + }); + const next = compileQuery({ derivations: columns, query: DESC_QUANTITY }); + + expect(isSortOnlyChange(previous, next)).toBe(true); + }); + + test.each([ + { + name: "direction flip", + prevSort: [{ columnId: "quantity", direction: "asc" }], + nextSort: [{ columnId: "quantity", direction: "desc" }], + }, + { + name: "added sort column", + prevSort: [{ columnId: "quantity", direction: "asc" }], + nextSort: [ + { columnId: "quantity", direction: "asc" }, + { columnId: "sector", direction: "asc" }, + ], + }, + { + name: "removal to unsorted", + prevSort: [{ columnId: "quantity", direction: "asc" }], + nextSort: [], + }, + ] as const)("true for $name", ({ prevSort, nextSort }) => { + const previous = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [], + sort: prevSort, + rowGroups: [], + }), + }); + const next = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [], + sort: nextSort, + rowGroups: [], + }), + }); + + expect(isSortOnlyChange(previous, next)).toBe(true); + }); + + test("false when the sort is identical", () => { + // Two structurally-equal plans compiled independently (no `previous` + // passed to `compileQuery`), so they are distinct objects. + const previous = compileQuery({ + derivations: columns, + query: ASC_QUANTITY, + }); + const next = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [], + sort: [{ columnId: "quantity", direction: "asc" }], + rowGroups: [], + }), + }); + + expect(previous).not.toBe(next); + expect(isSortOnlyChange(previous, next)).toBe(false); + }); + + test("false when filters also changed", () => { + const previous = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [{ columnId: "sector", operator: "contains", value: "Tech" }], + sort: [{ columnId: "quantity", direction: "asc" }], + rowGroups: [], + }), + }); + const next = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [ + { columnId: "sector", operator: "contains", value: "Energy" }, + ], + sort: [{ columnId: "quantity", direction: "desc" }], + rowGroups: [], + }), + }); + + expect(isSortOnlyChange(previous, next)).toBe(false); + }); + + test("false when rowGroups also changed", () => { + const previous = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [], + sort: [{ columnId: "quantity", direction: "asc" }], + rowGroups: [{ columnId: "sector", direction: "asc" }], + }), + }); + const next = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [], + sort: [{ columnId: "quantity", direction: "desc" }], + rowGroups: [{ columnId: "customer", direction: "asc" }], + }), + }); + + expect(isSortOnlyChange(previous, next)).toBe(false); + }); + + test("false when derivations changed for an active column", () => { + const quantityA = (row: Holding) => row.quantity; + const quantityB = (row: Holding) => row.quantity; + const columnsA = [ + helper.accessor("sector", { type: "text" }), + helper.accessor("customer", { type: "text" }), + helper.accessor("quantity", quantityA, { + type: "number", + aggregate: "sum", + }), + ] as const; + const columnsB = [ + helper.accessor("sector", { type: "text" }), + helper.accessor("customer", { type: "text" }), + helper.accessor("quantity", quantityB, { + type: "number", + aggregate: "sum", + }), + ] as const; + + const previous = compileQuery({ + derivations: columnsA, + query: ASC_QUANTITY as unknown as PretableQueryFor, + }); + const next = compileQuery({ + derivations: columnsB, + query: DESC_QUANTITY as unknown as PretableQueryFor, + }); + + expect(isSortOnlyChange(previous, next)).toBe(false); + }); + + test("false when filterAuthority differs between plans", () => { + const previous = compileQuery({ + derivations: columns, + query: ASC_QUANTITY, + }); + const next = compileQuery({ + derivations: columns, + query: DESC_QUANTITY, + filterAuthority: "external", + }); + + expect(isSortOnlyChange(previous, next)).toBe(false); + }); + + test("false when sortAuthority differs between plans", () => { + const previous = compileQuery({ + derivations: columns, + query: ASC_QUANTITY, + }); + const next = compileQuery({ + derivations: columns, + query: DESC_QUANTITY, + sortAuthority: "external", + }); + + expect(isSortOnlyChange(previous, next)).toBe(false); + }); + + test("false when both plans are external sort authority and only the public sort differs", () => { + const previous = compileQuery({ + derivations: columns, + query: ASC_QUANTITY, + sortAuthority: "external", + }); + const next = compileQuery({ + derivations: columns, + query: DESC_QUANTITY, + sortAuthority: "external", + }); + + // Runtime sort is [] for both under external authority, so there is no + // runtime-level change at all. + expect(isSortOnlyChange(previous, next)).toBe(false); + }); + + test("false for foreign plan objects in either position", () => { + const real = compileQuery({ derivations: columns, query: ASC_QUANTITY }); + const foreign = { query: DESC_QUANTITY, derivations: columns }; + + expect(isSortOnlyChange(foreign as never, real)).toBe(false); + expect(isSortOnlyChange(real, foreign as never)).toBe(false); + }); +}); diff --git a/packages/row-model/src/__tests__/retention.test.ts b/packages/row-model/src/__tests__/retention.test.ts index 21edc373a..29fce84b8 100644 --- a/packages/row-model/src/__tests__/retention.test.ts +++ b/packages/row-model/src/__tests__/retention.test.ts @@ -64,7 +64,9 @@ describe("instrumented local row-model retention", () => { }); const transition = instrumented.model.setQuery({ - filters: [], + // Filter change keeps the query off the #457 sort-only fast path; + // these tests exercise cooperative scheduler ownership. + filters: [{ columnId: "score", operator: "gte", value: 5 }], sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); @@ -112,7 +114,9 @@ describe("instrumented local row-model retention", () => { notifications += 1; }); const transition = instrumented.model.setQuery({ - filters: [], + // Filter change keeps the query off the #457 sort-only fast path; + // these tests exercise cooperative scheduler ownership. + filters: [{ columnId: "score", operator: "gte", value: 5 }], sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); @@ -161,7 +165,9 @@ describe("instrumented local row-model retention", () => { transitionBudgetMs: 1, }); const synchronousTransition = synchronous.model.setQuery({ - filters: [], + // Filter change keeps the query off the #457 sort-only fast path; + // these tests exercise cooperative scheduler ownership. + filters: [{ columnId: "score", operator: "gte", value: 5 }], sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); @@ -205,12 +211,16 @@ describe("instrumented local row-model retention", () => { transitionBudgetMs: 1, }); const firstTransition = first.model.setQuery({ - filters: [], + // Filter change keeps the query off the #457 sort-only fast path; + // these tests exercise cooperative scheduler ownership. + filters: [{ columnId: "score", operator: "gte", value: 5 }], sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); const secondTransition = second.model.setQuery({ - filters: [], + // Filter change keeps the query off the #457 sort-only fast path; + // these tests exercise cooperative scheduler ownership. + filters: [{ columnId: "score", operator: "gte", value: 5 }], sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); diff --git a/packages/row-model/src/__tests__/sort-fast-path.test.ts b/packages/row-model/src/__tests__/sort-fast-path.test.ts new file mode 100644 index 000000000..b5ca10c5b --- /dev/null +++ b/packages/row-model/src/__tests__/sort-fast-path.test.ts @@ -0,0 +1,1175 @@ +import { describe, expect, test, vi } from "vitest"; + +import { + compileQuery, + createColumnHelper, + PretableReentrantMutationError, + PretableRowModelError, + PretableTransitionCancelledError, + type PretableQueryFor, +} from "../index"; +import { + compareRecordRows, + sortKeysOf, + type CompiledQuery, +} from "../compiled-query"; +import type { CooperativeTransitionScheduler } from "../cooperative-transition"; +import { createInstrumentedLocalRowModel } from "../diagnostics"; +import type { LocalRowModelInstrumentation } from "../diagnostics"; +import type { RevisionRoot } from "../internal-types"; +import { compareOrderStatisticTreeIds } from "../persistent/order-statistic-tree"; +import { createPersistentMap } from "../persistent/persistent-map"; +import { buildRowStore } from "../row-store"; +import { rebuildRootForSortOnlyChange } from "../sort-rebuild"; +import type { PretableGroupId } from "../types"; +import { createVisibleIndex } from "../visible-index"; + +interface Holding { + id: string; + team: string; + score: number; + note: string; + label: string; +} + +const helper = createColumnHelper(); + +/** + * Checks a query literal against a column tuple, exactly as + * `query-delta.test.ts` does. + */ +function queryFor( + value: PretableQueryFor, +): PretableQueryFor { + return value; +} + +/** + * Builds the shared fixture: spied accessors on every column so tests can + * assert exactly which accessors a carryover rebuild runs. `label` is inactive + * in BOTH plans (never sorted, filtered, grouped, or aggregated), so its spy + * count must be 0 throughout — including the setup `evaluate`. + */ +function createFixture() { + const teamAccessor = vi.fn((row: Holding) => row.team); + const scoreAccessor = vi.fn((row: Holding) => row.score); + const noteAccessor = vi.fn((row: Holding) => row.note); + const labelAccessor = vi.fn((row: Holding) => row.label); + const columns = [ + helper.accessor("team", teamAccessor, { type: "text" }), + helper.accessor("score", scoreAccessor, { + type: "number", + aggregate: "sum", + }), + helper.accessor("note", noteAccessor, { type: "text" }), + helper.accessor("label", labelAccessor, { type: "text" }), + ] as const; + return { columns, teamAccessor, scoreAccessor, noteAccessor, labelAccessor }; +} + +type FixtureColumns = ReturnType["columns"]; + +const SCORE_DESC_TEAM_FILTER = queryFor({ + filters: [{ columnId: "team", operator: "equals", value: "Alpha" }], + sort: [{ columnId: "score", direction: "desc" }], + rowGroups: [], +}); + +const SCORE_ASC_TEAM_FILTER = queryFor({ + filters: [{ columnId: "team", operator: "equals", value: "Alpha" }], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [], +}); + +const NOTE_ASC_TEAM_FILTER = queryFor({ + filters: [{ columnId: "team", operator: "equals", value: "Alpha" }], + sort: [{ columnId: "note", direction: "asc" }], + rowGroups: [], +}); + +/** + * Seven rows chosen so the three orders that matter are pairwise-distinct + * permutations (asserted below): source order, score-desc order, note-asc + * order. `h3` fails the team filter; `h4`/`h5` tie on `note`, and `h5` + * appears BEFORE `h4` in source order while its id sorts AFTER — so the + * engine's real tie resolution (compareRecordRows falls through to sourceOrder) + * and an id-based one produce OPPOSITE orders for the tied pair, and the + * expectations below can disprove either mistake. + */ +const ROOT_ROWS: readonly Holding[] = Object.freeze([ + { id: "h1", team: "Alpha", score: 50, note: "delta", label: "u" }, + { id: "h2", team: "Alpha", score: 10, note: "alpha", label: "u" }, + { id: "h3", team: "Beta", score: 99, note: "aaaa", label: "u" }, + { id: "h5", team: "Alpha", score: 70, note: "bravo", label: "u" }, + { id: "h4", team: "Alpha", score: 30, note: "bravo", label: "u" }, + { id: "h6", team: "Alpha", score: 20, note: "echo", label: "u" }, + { id: "h7", team: "Alpha", score: 60, note: "charlie", label: "u" }, +]); + +const SOURCE_VISIBLE_ORDER = ["h1", "h2", "h5", "h4", "h6", "h7"] as const; +const OLD_VISIBLE_ORDER = ["h5", "h7", "h1", "h4", "h6", "h2"] as const; +const NEW_VISIBLE_ORDER = ["h2", "h5", "h4", "h7", "h1", "h6"] as const; + +function createRoot( + queryPlan: CompiledQuery, + rows: readonly Holding[], +): RevisionRoot { + const store = buildRowStore({ + rows, + getRowId: (row) => row.id, + queryPlan, + }); + const defaultPolicy = Object.freeze({ kind: "expanded" as const }); + const expansion = Object.freeze({ + default: defaultPolicy, + overrides: createPersistentMap(), + state: Object.freeze({ default: defaultPolicy, overrideCount: 0 }), + }); + return Object.freeze({ + revision: 0, + parentRevision: null, + rows: store.rows, + sourceOrder: store.sourceOrder, + visible: createVisibleIndex( + store.records, + queryPlan, + false, + expansion.overrides, + ), + queryPlan, + expansion, + cause: Object.freeze({ kind: "initial" as const }), + }); +} + +function rankedIds( + visible: RevisionRoot["visible"], +): readonly string[] { + const ids: string[] = []; + for (let index = 0; index < visible.rows.size; index += 1) { + ids.push(visible.rows.entryAt(index)!.record.rowId); + } + return ids; +} + +function testInstrumentation(): LocalRowModelInstrumentation { + return { + work: { + rowsEvaluated: 0, + hamtNodesCopied: 0, + orderNodesCopied: 0, + groupNodesCopied: 0, + aggregateMerges: 0, + transitionRows: 0, + snapshotOutputRowsRead: 0, + synchronousRebuilds: 0, + synchronousRebuildMs: 0, + sortKeyCarries: 0, + sortKeyEvaluations: 0, + schedulerSliceDurations: [], + }, + snapshotRoots: new WeakMap(), + retainedSnapshots: new Map(), + scheduledCallbacks: new Set(), + currentRevisionRoot: undefined, + model: undefined, + }; +} + +describe("rebuildRootForSortOnlyChange", () => { + function createRebuildFixture() { + const fixture = createFixture(); + const previousPlan = compileQuery({ + derivations: fixture.columns, + query: SCORE_DESC_TEAM_FILTER, + }); + const nextPlan = compileQuery({ + derivations: fixture.columns, + query: NOTE_ASC_TEAM_FILTER, + }); + const captured = createRoot(previousPlan, ROOT_ROWS); + // Fixture controls: the three orders must be pairwise-distinct + // permutations, or a rebuild that ignores the sort could still pass. + expect(rankedIds(captured.visible)).toEqual(OLD_VISIBLE_ORDER); + expect(OLD_VISIBLE_ORDER).not.toEqual(NEW_VISIBLE_ORDER); + expect(OLD_VISIBLE_ORDER).not.toEqual(SOURCE_VISIBLE_ORDER); + expect(NEW_VISIBLE_ORDER).not.toEqual(SOURCE_VISIBLE_ORDER); + expect([...OLD_VISIBLE_ORDER].sort()).toEqual( + [...NEW_VISIBLE_ORDER].sort(), + ); + // Tiebreak control: the note-tied pair's source order OPPOSES its id + // order, so ties resolved by rowId instead of the engine's sourceOrder + // fallthrough (compareRecordRows' final clause) cannot pass by luck — and the + // NEW_VISIBLE_ORDER expectation pins the sourceOrder resolution (h5 + // before h4). + expect(SOURCE_VISIBLE_ORDER.indexOf("h5")).toBeLessThan( + SOURCE_VISIBLE_ORDER.indexOf("h4"), + ); + expect(compareOrderStatisticTreeIds("h4", "h5")).toBeLessThan(0); + return { fixture, previousPlan, nextPlan, captured }; + } + + test("the rebuilt root's visible order equals a cold build under nextPlan", () => { + const { fixture, nextPlan, captured } = createRebuildFixture(); + + const rebuilt = rebuildRootForSortOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + }); + + // Oracle: an identical plan compiled WITHOUT `previous` chaining (cold + // cache), evaluated from scratch, sorted with the same composite order + // the visible tree maintains. + const twinPlan = compileQuery({ + derivations: fixture.columns, + query: NOTE_ASC_TEAM_FILTER, + }); + expect(twinPlan).not.toBe(nextPlan); + const expected = ROOT_ROWS.map((row, sourceOrder) => ({ + rowId: row.id, + input: { rowId: row.id, row, sourceOrder }, + metadata: twinPlan.evaluate({ rowId: row.id, row, sourceOrder }), + })) + .filter((entry) => entry.metadata.filterPasses) + .sort( + (left, right) => + compareRecordRows(twinPlan, left.input, right.input) || + compareOrderStatisticTreeIds(left.rowId, right.rowId), + ) + .map((entry) => entry.rowId); + expect(expected).toEqual([...NEW_VISIBLE_ORDER]); + expect(rankedIds(rebuilt.visible)).toEqual(expected); + }); + + test("filtered-out rows stay out of visible but keep updated records in rows", () => { + const { nextPlan, captured } = createRebuildFixture(); + + const rebuilt = rebuildRootForSortOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + }); + + expect(rebuilt.visible.rows.rankOf("h3")).toBeUndefined(); + const record = rebuilt.rows.get("h3"); + expect(record).toBeDefined(); + expect(record!.metadata.filterPasses).toBe(false); + // The NEW plan's store was filled for the filtered-out row too: sort keys + // resolve under nextPlan as note, not score. + expect(sortKeysOf(nextPlan, record!)).toEqual([ + { columnId: "note", value: "aaaa" }, + ]); + expect(rebuilt.rows.size).toBe(ROOT_ROWS.length); + expect(rebuilt.visible.rows.size).toBe(NEW_VISIBLE_ORDER.length); + }); + + test("revision, parentRevision, queryPlan, and cause are the requested values", () => { + const { nextPlan, captured } = createRebuildFixture(); + + const rebuilt = rebuildRootForSortOnlyChange({ + captured, + nextPlan, + revision: 5, + now: () => 0, + }); + + expect(rebuilt.revision).toBe(5); + expect(rebuilt.parentRevision).toBe(4); + expect(rebuilt.queryPlan).toBe(nextPlan); + expect(rebuilt.cause).toEqual({ kind: "set-query" }); + }); + + test("sourceOrder and expansion are carried by reference from the captured root", () => { + const { nextPlan, captured } = createRebuildFixture(); + + const rebuilt = rebuildRootForSortOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + }); + + expect(rebuilt.sourceOrder).toBe(captured.sourceOrder); + expect(rebuilt.expansion).toBe(captured.expansion); + }); + + test("the rows map and every record carry by IDENTITY", () => { + const { nextPlan, captured } = createRebuildFixture(); + + const rebuilt = rebuildRootForSortOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + }); + + // The entire point of sort-rebuild v2: no record rebuild, no rows + // transient — the committed root's rows map IS the captured one. + expect(rebuilt.rows).toBe(captured.rows); + for (const row of ROOT_ROWS) { + const before = captured.rows.get(row.id)!; + const after = rebuilt.rows.get(row.id)!; + expect(after).toBe(before); + expect(after.publicRow).toBe(before.publicRow); + expect(after.integrity).toBe(before.integrity); + expect(after.row).toBe(before.row); + expect(after.sourceOrder).toBe(before.sourceOrder); + } + // Identity carried, order still changed — the positive twin. + expect(rankedIds(rebuilt.visible)).toEqual([...NEW_VISIBLE_ORDER]); + }); + + test("aggregate-leaf dependencies carry by identity and values stay correct", () => { + const { nextPlan, captured } = createRebuildFixture(); + + const rebuilt = rebuildRootForSortOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => 0, + }); + + // Record identity implies leaf and dependency identity; asserted on a + // concrete leaf anyway so a future record rebuild cannot silently start + // dirtying aggregate leaves on sort-only changes. + for (const row of ROOT_ROWS) { + const before = captured.rows.get(row.id)!.metadata.aggregateLeaves[0]; + const after = rebuilt.rows.get(row.id)!.metadata.aggregateLeaves[0]; + expect(after).toBe(before); + expect(after.allLeaf.dependency).toBe(before.allLeaf.dependency); + // Positive twin: the carried leaf still holds the row's real value. + expect(after.allLeaf.value).toBe(row.score); + } + }); + + test("instrumentation counts one rebuild and the measured duration", () => { + const { nextPlan, captured } = createRebuildFixture(); + const instrumentation = testInstrumentation(); + const ticks = [0, 7]; + let call = 0; + + rebuildRootForSortOnlyChange({ + captured, + nextPlan, + revision: 1, + now: () => ticks[call++] ?? 7, + instrumentation, + }); + + expect(instrumentation.work.synchronousRebuilds).toBe(1); + expect(instrumentation.work.synchronousRebuildMs).toBe(7); + }); + + test("throws TypeError when the plans are not a sort-only change", () => { + const { fixture, captured } = createRebuildFixture(); + const filterChangedPlan = compileQuery({ + derivations: fixture.columns, + query: queryFor({ + filters: [{ columnId: "team", operator: "equals", value: "Beta" }], + sort: [{ columnId: "note", direction: "asc" }], + rowGroups: [], + }), + }); + + expect(() => + rebuildRootForSortOnlyChange({ + captured, + nextPlan: filterChangedPlan, + revision: 1, + now: () => 0, + }), + ).toThrowError( + new TypeError("Synchronous rebuild requires a sort-only plan change."), + ); + }); + + test("throws TypeError for a grouped next plan", () => { + const fixture = createFixture(); + const groupedPrevious = compileQuery({ + derivations: fixture.columns, + query: queryFor({ + filters: [{ columnId: "team", operator: "equals", value: "Alpha" }], + sort: [{ columnId: "score", direction: "desc" }], + rowGroups: [{ columnId: "team", direction: "asc" }], + }), + }); + const groupedNext = compileQuery({ + derivations: fixture.columns, + query: queryFor({ + filters: [{ columnId: "team", operator: "equals", value: "Alpha" }], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [{ columnId: "team", direction: "asc" }], + }), + }); + const captured = createRoot(groupedPrevious, ROOT_ROWS); + + expect(() => + rebuildRootForSortOnlyChange({ + captured, + nextPlan: groupedNext, + revision: 1, + now: () => 0, + }), + ).toThrowError( + new TypeError("Synchronous rebuild requires an ungrouped query."), + ); + }); +}); + +/** + * Minimal deterministic scheduler, duplicated from `transitions.test.ts` + * (which exports nothing; test files here do not import from each other). + */ +class ManualScheduler implements CooperativeTransitionScheduler { + readonly entries: { readonly task: () => void; cancelled: boolean }[] = []; + + schedule(task: () => void): () => void { + const entry = { task, cancelled: false }; + this.entries.push(entry); + return () => { + entry.cancelled = true; + }; + } + + flushAll(limit = 1_000_000): void { + let count = 0; + for (;;) { + const entry = this.entries.shift(); + if (entry === undefined) return; + if (!entry.cancelled) entry.task(); + count += 1; + if (count > limit) throw new Error("Manual scheduler did not settle."); + } + } +} + +/** + * `h8` extends the shared seven-row fixture to eight rows; it sorts last + * under source order, score-desc, and note-asc alike, so the pairwise + * distinctness of the three orders (and the h5/h4 tie control) is preserved. + */ +const MODEL_ROWS: readonly Holding[] = Object.freeze([ + ...ROOT_ROWS, + { id: "h8", team: "Alpha", score: 5, note: "zulu", label: "u" }, +]); + +const MODEL_SOURCE_ORDER = [...SOURCE_VISIBLE_ORDER, "h8"] as const; +const MODEL_OLD_ORDER = [...OLD_VISIBLE_ORDER, "h8"] as const; +const MODEL_NEW_ORDER = [...NEW_VISIBLE_ORDER, "h8"] as const; +const MODEL_SCORE_ASC_ORDER = [ + "h8", + "h2", + "h6", + "h4", + "h1", + "h7", + "h5", +] as const; + +type AnyModel = ReturnType["model"]; + +function snapshotIds(model: { + getState(): { snapshot: { range(a: number, b: number): readonly unknown[] } }; +}): readonly string[] { + return model + .getState() + .snapshot.range(0, Number.MAX_SAFE_INTEGER) + .flatMap((row) => + (row as { kind: string }).kind === "data" + ? [String((row as { rowId: unknown }).rowId)] + : [], + ); +} + +describe("setQuery sort-only fast path", () => { + /** + * Ticking clock + 1ms budget force the cooperative path to yield after + * every unit, so any scheduler entry is proof the cooperative machinery + * ran — and an empty queue is proof the fast path bypassed it. + */ + function createModelFixture(options?: { + readonly columns?: FixtureColumns; + readonly rows?: readonly Holding[]; + }) { + const scheduler = new ManualScheduler(); + const fixture = createFixture(); + let tick = 0; + const instrumented = createInstrumentedLocalRowModel({ + rows: options?.rows ?? MODEL_ROWS, + columns: options?.columns ?? fixture.columns, + query: SCORE_DESC_TEAM_FILTER, + transitionScheduler: scheduler, + transitionClock: () => tick++, + transitionBudgetMs: 1, + }); + const model = instrumented.model; + // Fixture controls: three pairwise-distinct permutations of one row set, + // and the tied pair's id order opposes its source order. + expect(snapshotIds(model)).toEqual([...MODEL_OLD_ORDER]); + expect([...MODEL_OLD_ORDER]).not.toEqual([...MODEL_NEW_ORDER]); + expect([...MODEL_OLD_ORDER]).not.toEqual([...MODEL_SOURCE_ORDER]); + expect([...MODEL_NEW_ORDER]).not.toEqual([...MODEL_SOURCE_ORDER]); + expect([...MODEL_OLD_ORDER].sort()).toEqual([...MODEL_NEW_ORDER].sort()); + expect(MODEL_SOURCE_ORDER.indexOf("h5")).toBeLessThan( + MODEL_SOURCE_ORDER.indexOf("h4"), + ); + expect(compareOrderStatisticTreeIds("h4", "h5")).toBeLessThan(0); + return { + model, + diagnostics: instrumented.diagnostics, + scheduler, + fixture, + }; + } + + test("resolves synchronously without any scheduler task", async () => { + const { model, diagnostics, scheduler } = createModelFixture(); + + const transition = model.setQuery(NOTE_ASC_TEAM_FILTER); + + expect(scheduler.entries).toHaveLength(0); + expect(model.getState().status).toEqual({ kind: "ready" }); + expect(snapshotIds(model)).toEqual([...MODEL_NEW_ORDER]); + expect(diagnostics.read().work.synchronousRebuilds).toBe(1); + await expect(transition.finished).resolves.toBe(1); + }); + + test("mutation twin: a filter change takes the cooperative path", () => { + const { model, diagnostics, scheduler } = createModelFixture(); + + model.setQuery({ + filters: [{ columnId: "team", operator: "equals", value: "Beta" }], + sort: [{ columnId: "score", direction: "desc" }], + rowGroups: [], + }); + + expect( + scheduler.entries.length > 0 || + model.getState().status.kind === "rebuilding", + ).toBe(true); + expect(diagnostics.read().work.synchronousRebuilds).toBe(0); + }); + + test("sorting still sorts: full permutation, ties by source order", () => { + const { model } = createModelFixture(); + + model.setQuery(NOTE_ASC_TEAM_FILTER); + + const ids = snapshotIds(model); + expect(ids).toEqual([...MODEL_NEW_ORDER]); + // The note-tied pair resolves by SOURCE order (h5 before h4), which is + // the opposite of its id order — asserted as a fixture control above. + expect(ids.indexOf("h5")).toBeLessThan(ids.indexOf("h4")); + }); + + test("supersedes an in-flight cooperative transition", async () => { + const { model, scheduler } = createModelFixture(); + const first = model.setQuery({ + filters: [{ columnId: "team", operator: "equals", value: "Beta" }], + sort: [{ columnId: "score", direction: "desc" }], + rowGroups: [], + }); + expect(model.getState().status.kind).toBe("rebuilding"); + + const second = model.setQuery(NOTE_ASC_TEAM_FILTER); + + await expect(first.finished).rejects.toMatchObject({ + name: "PretableTransitionCancelledError", + reason: "superseded", + }); + await expect(first.finished).rejects.toBeInstanceOf( + PretableTransitionCancelledError, + ); + await expect(second.finished).resolves.toBe(1); + // The fast path rebuilt from the last COMMITTED root: OLD filter (Alpha) + // + NEW sort. Every Alpha row the abandoned Beta filter would have + // removed is still present, and h3 (Beta) is still filtered out. + expect(snapshotIds(model)).toEqual([...MODEL_NEW_ORDER]); + expect(model.getState().status).toEqual({ kind: "ready" }); + scheduler.flushAll(); + // Abandoned cooperative tasks must not resurrect the superseded query. + expect(snapshotIds(model)).toEqual([...MODEL_NEW_ORDER]); + }); + + test("notifies subscribers exactly once", () => { + const { model } = createModelFixture(); + let calls = 0; + model.subscribe(() => { + calls += 1; + }); + + model.setQuery(NOTE_ASC_TEAM_FILTER); + + expect(calls).toBe(1); + }); + + test("snapshot.query and requestedQuery report the new sort", () => { + const { model } = createModelFixture(); + + const transition = model.setQuery(NOTE_ASC_TEAM_FILTER); + + expect(transition.requestedQuery.sort).toEqual([ + { columnId: "note", direction: "asc" }, + ]); + const snapshot = model.getState().snapshot; + expect(snapshot.query.sort).toEqual([ + { columnId: "note", direction: "asc" }, + ]); + expect(snapshot.query.filters).toEqual(SCORE_DESC_TEAM_FILTER.filters); + }); + + test("setRows immediately after a fast setQuery applies incrementally", () => { + const { model, diagnostics, scheduler } = createModelFixture(); + model.setQuery(NOTE_ASC_TEAM_FILTER); + expect(diagnostics.read().work.synchronousRebuilds).toBe(1); + + // "aardvark" sorts before every other note, so h6 must move from + // second-to-last to first under the NEW plan. + const moved = MODEL_ROWS.map((row) => + row.id === "h6" ? { ...row, note: "aardvark" } : row, + ); + model.setRows(moved); + + expect(snapshotIds(model)).toEqual([ + "h6", + "h2", + "h5", + "h4", + "h7", + "h1", + "h8", + ]); + // Parity with normal incremental setRows: synchronous, no scheduler + // task, no additional whole-root rebuild. + expect(model.getState().status).toEqual({ kind: "ready" }); + expect(scheduler.entries).toHaveLength(0); + expect(diagnostics.read().work.synchronousRebuilds).toBe(1); + }); + + test('the fast path journals a reset with reason "reorder"', () => { + const { model } = createModelFixture(); + const before = model.getState().snapshot.revision; + + model.setQuery(NOTE_ASC_TEAM_FILTER); + + expect(model.changesSince(before)).toEqual({ + kind: "reset", + toRevision: before + 1, + reason: "reorder", + }); + }); + + test('mutation twin: a cooperative filter setQuery journals "bulk-replace"', async () => { + const { model, scheduler } = createModelFixture(); + const before = model.getState().snapshot.revision; + + const transition = model.setQuery({ + filters: [{ columnId: "team", operator: "equals", value: "Beta" }], + sort: [{ columnId: "score", direction: "desc" }], + rowGroups: [], + }); + scheduler.flushAll(); + await expect(transition.finished).resolves.toBe(before + 1); + + expect(model.changesSince(before)).toEqual({ + kind: "reset", + toRevision: before + 1, + reason: "bulk-replace", + }); + }); + + test('setRows after a fast sort spans a mixed range: NOT "reorder"', () => { + const { model } = createModelFixture(); + const before = model.getState().snapshot.revision; + model.setQuery(NOTE_ASC_TEAM_FILTER); + + const moved = MODEL_ROWS.map((row) => + row.id === "h6" ? { ...row, note: "aardvark" } : row, + ); + model.setRows(moved); + + // The range [reorder barrier, setRows barrier] must NOT collapse to + // "reorder" — the setRows changed row content, not just order. + expect(model.changesSince(before)).toEqual({ + kind: "reset", + toRevision: before + 2, + reason: "bulk-replace", + }); + // And the setRows commit alone is a plain barrier. + expect(model.changesSince(before + 1)).toEqual({ + kind: "reset", + toRevision: before + 2, + reason: "bulk-replace", + }); + }); + + test('same-reference-mutation recompile setRows journals "bulk-replace", never "reorder"', () => { + // The A2-review-flagged case: the recompile path swaps the plan exactly + // like the fast path does, but it changes row CONTENT — its barrier must + // stay a plain one. + // Non-extensible rows: the dev integrity guard fingerprints them instead + // of freezing, which is what makes an in-place mutation observable. + const rows = MODEL_ROWS.map((row) => Object.preventExtensions({ ...row })); + const { model } = createModelFixture({ rows }); + model.setQuery(NOTE_ASC_TEAM_FILTER); + const afterSort = model.getState().snapshot.revision; + + // Mutate one row IN PLACE and hand back the same references, which is + // what forces the same-reference-mutation recompile. + const mutated = rows.find((row) => row.id === "h6")!; + mutated.note = "aardvark"; + model.setRows(rows); + + expect(snapshotIds(model)[0]).toBe("h6"); + expect(model.changesSince(afterSort)).toEqual({ + kind: "reset", + toRevision: afterSort + 1, + reason: "bulk-replace", + }); + }); + + test("every publicRow carries by identity across the sort-only change", () => { + const { model } = createModelFixture(); + const snapshotBefore = model.getState().snapshot; + const before = new Map(); + for (let index = 0; index < snapshotBefore.visibleRowCount; index += 1) { + const row = snapshotBefore.rowAt(index)!; + expect(row.kind).toBe("data"); + if (row.kind === "data") before.set(String(row.rowId), row); + } + + model.setQuery(NOTE_ASC_TEAM_FILTER); + + const after = model.getState().snapshot; + expect(after.visibleRowCount).toBe(snapshotBefore.visibleRowCount); + // Order changed (fixture control: distinct permutations)... + expect(snapshotIds(model)).toEqual([...MODEL_NEW_ORDER]); + // ...while every published row object is the SAME object as before — + // selection/focus consumers keyed by row identity survive the change. + for (let index = 0; index < after.visibleRowCount; index += 1) { + const row = after.rowAt(index)!; + expect(row.kind).toBe("data"); + if (row.kind === "data") { + expect(row).toBe(before.get(String(row.rowId))); + } + } + }); + + test("stale-hazard: key updates re-rank after the fast path, non-key updates do not move", () => { + const { model, scheduler } = createModelFixture(); + model.setQuery(NOTE_ASC_TEAM_FILTER); + expect(snapshotIds(model)).toEqual([...MODEL_NEW_ORDER]); + + // A sort-KEY update after the fast path: "aardvark" precedes every other + // note, so h6 must re-rank from fifth to first. Hand-computed: the + // remaining rows keep their note-asc relative order. + const keyUpdated = MODEL_ROWS.map((row) => + row.id === "h6" ? { ...row, note: "aardvark" } : row, + ); + model.setRows(keyUpdated); + const afterKeyUpdate = ["h6", "h2", "h5", "h4", "h7", "h1", "h8"] as const; + expect(snapshotIds(model)).toEqual([...afterKeyUpdate]); + + // A NON-key update (label is inactive in every plan): the updated row + // must NOT move — sameFlatOrder resolves both sides through the store + // and sees identical keys. + const nonKeyUpdated = keyUpdated.map((row) => + row.id === "h5" ? { ...row, label: "renamed" } : row, + ); + model.setRows(nonKeyUpdated); + expect(snapshotIds(model)).toEqual([...afterKeyUpdate]); + expect(model.getState().status).toEqual({ kind: "ready" }); + expect(scheduler.entries).toHaveLength(0); + }); + + test("work counters split carries from evaluations per sort-column entry", () => { + // Counting contract: `fillSortKeysFromPrevious` bumps ONE counter per + // (row, next-plan sort column) pair — `sortKeyCarries` when the value + // came from the previous plan's store, `sortKeyEvaluations` when the + // accessor ran. Rows already in the next store count nothing. + + // Overlap-heavy: score desc -> score asc shares its single sort column, + // so every one of the 8 captured rows carries: carries == rowCount, + // evaluations == 0. + const overlap = createModelFixture(); + overlap.diagnostics.resetWork(); + overlap.model.setQuery(SCORE_ASC_TEAM_FILTER); + expect(overlap.diagnostics.read().work.synchronousRebuilds).toBe(1); + expect(overlap.diagnostics.read().work.sortKeyCarries).toBe( + MODEL_ROWS.length, + ); + expect(overlap.diagnostics.read().work.sortKeyEvaluations).toBe(0); + + // New-column: score desc -> note asc has NO overlap; note's accessor + // runs once per row: evaluations == rowCount, carries == 0. + const fresh = createModelFixture(); + fresh.diagnostics.resetWork(); + fresh.model.setQuery(NOTE_ASC_TEAM_FILTER); + expect(fresh.diagnostics.read().work.synchronousRebuilds).toBe(1); + expect(fresh.diagnostics.read().work.sortKeyEvaluations).toBe( + MODEL_ROWS.length, + ); + expect(fresh.diagnostics.read().work.sortKeyCarries).toBe(0); + }); + + test("equivalence with a cold model built directly under the next query", () => { + const { model: warm, fixture } = createModelFixture(); + warm.setQuery(NOTE_ASC_TEAM_FILTER); + const cold = createInstrumentedLocalRowModel({ + rows: MODEL_ROWS, + columns: fixture.columns, + query: NOTE_ASC_TEAM_FILTER, + }).model; + + const warmSnapshot = warm.getState().snapshot; + const coldSnapshot = cold.getState().snapshot; + expect(warmSnapshot.visibleRowCount).toBe(coldSnapshot.visibleRowCount); + for (let index = 0; index < warmSnapshot.visibleRowCount; index += 1) { + const warmRow = warmSnapshot.rowAt(index)!; + const coldRow = coldSnapshot.rowAt(index)!; + expect(warmRow.kind).toBe("data"); + expect(warmRow.kind === "data" && coldRow.kind === "data").toBe(true); + if (warmRow.kind === "data" && coldRow.kind === "data") { + expect(warmRow.rowId).toBe(coldRow.rowId); + expect(warmRow.row).toBe(coldRow.row); + } + } + expect(warmSnapshot.query).toEqual(coldSnapshot.query); + }); + + function throwingNoteColumns(boom: Error): FixtureColumns { + return [ + helper.accessor("team", (row: Holding) => row.team, { type: "text" }), + helper.accessor("score", (row: Holding) => row.score, { + type: "number", + aggregate: "sum", + }), + helper.accessor( + "note", + (row: Holding): string => { + // h6 sits sixth in source order, so several rows succeed before the + // throw — partial work would be visible if state leaked. + if (row.id === "h6") throw boom; + return row.note; + }, + { type: "text" }, + ), + helper.accessor("label", (row: Holding) => row.label, { type: "text" }), + ] as unknown as FixtureColumns; + } + + function expectAccessorFailureShape( + model: AnyModel, + transitionId: number, + boom: Error, + ): PretableRowModelError { + const status = model.getState().status; + expect(status.kind).toBe("error"); + if (status.kind !== "error") throw new Error("unreachable"); + expect(status.transitionId).toBe(transitionId); + expect(status.error).toBeInstanceOf(PretableRowModelError); + const error = status.error as PretableRowModelError; + expect(error.code).toBe("accessor-failed"); + expect(error.cause).toBe(boom); + return error; + } + + test("accessor failure on the SLOW path pins the error shape", async () => { + const boom = new Error("boom"); + const { model, scheduler } = createModelFixture({ + columns: throwingNoteColumns(boom), + }); + + // Filter AND sort change: not sort-only, so the cooperative path runs the + // throwing accessor. + const transition = model.setQuery({ + filters: [], + sort: [{ columnId: "note", direction: "asc" }], + rowGroups: [], + }); + scheduler.flushAll(); + + const error = expectAccessorFailureShape(model, transition.id, boom); + await expect(transition.finished).rejects.toBe(error); + // Root unchanged: the OLD committed order is still published. + expect(snapshotIds(model)).toEqual([...MODEL_OLD_ORDER]); + }); + + test("accessor failure on the fast path matches the slow path's shape", async () => { + const boom = new Error("boom"); + const { model, scheduler, diagnostics } = createModelFixture({ + columns: throwingNoteColumns(boom), + }); + + const transition = model.setQuery(NOTE_ASC_TEAM_FILTER); + + // Must not throw synchronously, must not schedule cooperative work. + expect(scheduler.entries).toHaveLength(0); + const error = expectAccessorFailureShape(model, transition.id, boom); + await expect(transition.finished).rejects.toBe(error); + expect(snapshotIds(model)).toEqual([...MODEL_OLD_ORDER]); + expect(diagnostics.read().work.synchronousRebuilds).toBe(0); + + // A subsequent valid sort-only setQuery recovers to ready. + const recovery = model.setQuery({ + filters: SCORE_DESC_TEAM_FILTER.filters, + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [], + }); + expect(model.getState().status).toEqual({ kind: "ready" }); + expect(snapshotIds(model)).toEqual([...MODEL_SCORE_ASC_ORDER]); + await expect(recovery.finished).resolves.toBe(1); + }); + + test("a reentrant mutation from a sort accessor surfaces the reentrancy error", async () => { + const modelRef: { current: AnyModel | undefined } = { current: undefined }; + const columns = [ + helper.accessor("team", (row: Holding) => row.team, { type: "text" }), + helper.accessor("score", (row: Holding) => row.score, { + type: "number", + aggregate: "sum", + }), + helper.accessor( + "note", + (row: Holding): string => { + if (row.id === "h6") modelRef.current!.setRows([]); + return row.note; + }, + { type: "text" }, + ), + helper.accessor("label", (row: Holding) => row.label, { type: "text" }), + ] as unknown as FixtureColumns; + const scheduler = new ManualScheduler(); + const instrumented = createInstrumentedLocalRowModel({ + rows: MODEL_ROWS, + columns, + query: SCORE_DESC_TEAM_FILTER, + transitionScheduler: scheduler, + }); + modelRef.current = instrumented.model; + + const transition = instrumented.model.setQuery(NOTE_ASC_TEAM_FILTER); + + expect(scheduler.entries).toHaveLength(0); + const status = instrumented.model.getState().status; + expect(status.kind).toBe("error"); + if (status.kind !== "error") throw new Error("unreachable"); + expect(status.transitionId).toBe(transition.id); + expect(status.error).toBeInstanceOf(PretableReentrantMutationError); + await expect(transition.finished).rejects.toBe(status.error); + expect(snapshotIds(instrumented.model)).toEqual([...MODEL_OLD_ORDER]); + }); + + test("cancel() on the already-resolved fast transition is a no-op", async () => { + const { model } = createModelFixture(); + const transition = model.setQuery(NOTE_ASC_TEAM_FILTER); + await expect(transition.finished).resolves.toBe(1); + + transition.cancel(); + + expect(model.getState().status).toEqual({ kind: "ready" }); + expect(snapshotIds(model)).toEqual([...MODEL_NEW_ORDER]); + }); +}); + +describe("aggregates under the slimmed {sourceOrder} dependency", () => { + interface Deal { + id: string; + team: string; + score: number; + note: string; + label: string; + } + const dealHelper = createColumnHelper(); + + test("sort-key-only updates keep aggregate values correct while re-ranking", () => { + const columns = [ + dealHelper.accessor("team", { type: "text" }), + dealHelper.accessor("score", { type: "number", aggregate: "sum" }), + dealHelper.accessor("note", { type: "text" }), + ] as const; + const rows: Deal[] = [ + { id: "r1", team: "A", score: 5, note: "x", label: "u" }, + { id: "r2", team: "A", score: 3, note: "y", label: "u" }, + { id: "r3", team: "B", score: 8, note: "x", label: "u" }, + { id: "r4", team: "B", score: 1, note: "y", label: "u" }, + { id: "r5", team: "A", score: 7, note: "x", label: "u" }, + { id: "r6", team: "B", score: 9, note: "z", label: "u" }, + ]; + const model = createInstrumentedLocalRowModel({ + rows, + columns, + initialExpansion: { kind: "expanded" }, + query: { + filters: [], + sort: [{ columnId: "note", direction: "asc" }], + rowGroups: [{ columnId: "team", direction: "asc" }], + }, + }).model; + const shape = () => + model + .getState() + .snapshot.range(0, 100) + .map((row) => + row.kind === "group" + ? `group:${String(row.value)}:sum=${String( + (row.aggregates as { score: unknown }).score, + )}` + : String(row.rowId), + ); + // Hand derivation: A = {r1, r2, r5} sum 15; B = {r3, r4, r6} sum 18. + // note asc within groups, note ties by source order. + expect(shape()).toEqual([ + "group:A:sum=15", + "r1", + "r5", + "r2", + "group:B:sum=18", + "r3", + "r4", + "r6", + ]); + + // Update ONLY r5's sort key (note). By design the slimmed dependency no + // longer dirties aggregate leaves for sort-key changes — the sums must + // still be right (the positive twin), and the row must re-rank. + model.setRows( + rows.map((row) => (row.id === "r5" ? { ...row, note: "a" } : row)), + ); + expect(shape()).toEqual([ + "group:A:sum=15", + "r5", + "r1", + "r2", + "group:B:sum=18", + "r3", + "r4", + "r6", + ]); + }); + + test("applyTransaction updating ONLY an aggregated value recomputes that group's sum", () => { + const columns = [ + dealHelper.accessor("team", { type: "text" }), + dealHelper.accessor("score", { type: "number", aggregate: "sum" }), + dealHelper.accessor("note", { type: "text" }), + ] as const; + const rows: Deal[] = [ + { id: "r1", team: "A", score: 5, note: "x", label: "u" }, + { id: "r2", team: "A", score: 3, note: "y", label: "u" }, + { id: "r3", team: "B", score: 8, note: "x", label: "u" }, + { id: "r4", team: "B", score: 1, note: "y", label: "u" }, + { id: "r5", team: "A", score: 7, note: "x", label: "u" }, + { id: "r6", team: "B", score: 9, note: "z", label: "u" }, + ]; + const model = createInstrumentedLocalRowModel({ + rows, + columns, + initialExpansion: { kind: "expanded" }, + query: { + filters: [], + sort: [{ columnId: "note", direction: "asc" }], + rowGroups: [{ columnId: "team", direction: "asc" }], + }, + }).model; + const shape = () => + model + .getState() + .snapshot.range(0, 100) + .map((row) => + row.kind === "group" + ? `group:${String(row.value)}:sum=${String( + (row.aggregates as { score: unknown }).score, + )}` + : String(row.rowId), + ); + expect(shape()).toEqual([ + "group:A:sum=15", + "r1", + "r5", + "r2", + "group:B:sum=18", + "r3", + "r4", + "r6", + ]); + + // Update ONLY r2's aggregated VALUE: score is not sorted and the group + // key is untouched, so sourceOrder and sort keys are identical before + // and after — the value comparison in `sameGroupIndexContribution` is + // the ONLY thing standing between this update and a stale sum. + model.applyTransaction({ update: [{ id: "r2", changes: { score: 30 } }] }); + + expect(shape()).toEqual([ + "group:A:sum=42", + "r1", + "r5", + "r2", + "group:B:sum=18", + "r3", + "r4", + "r6", + ]); + }); + + test("aggregate leaves tying on ALL sort keys order by sourceOrder, not id", () => { + // Order-revealing custom aggregator: concatenates each leaf's label in + // the aggregate tree's traversal order. Associative (merge preserves + // left-right order), so the aggregator law holds; NOT commutative, so + // the output exposes the leaf ordering. + const concat = { + init: () => "", + accumulate: (acc: string, value: string) => acc + value, + merge: (left: string, right: string) => left + right, + finalize: (acc: string) => acc, + }; + const columns = [ + dealHelper.accessor("team", { type: "text" }), + dealHelper.accessor("score", { type: "number" }), + dealHelper.accessor("note", { type: "text" }), + dealHelper.accessor("label", { type: "text", aggregate: concat }), + ] as const; + // t3 and t2 tie on the ONLY sort key (note "x"); t3 precedes t2 in + // source order while its id sorts AFTER t2's — sourceOrder resolution + // yields "3" before "2", id resolution the opposite. t1 sorts FIRST by + // note ("m") but LAST in source order, so an ordering that ignores the + // keys (stale or empty decoration falling through to sourceOrder) + // produces "321", not "132" — the fixture can disprove key loss, not + // just tie direction. + const rows: Deal[] = [ + { id: "t3", team: "A", score: 2, note: "x", label: "3" }, + { id: "t2", team: "A", score: 3, note: "x", label: "2" }, + { id: "t1", team: "A", score: 1, note: "m", label: "1" }, + { id: "t4", team: "B", score: 4, note: "a", label: "4" }, + ]; + const model = createInstrumentedLocalRowModel({ + rows, + columns, + initialExpansion: { kind: "expanded" }, + query: { + filters: [], + sort: [{ columnId: "note", direction: "asc" }], + rowGroups: [{ columnId: "team", direction: "asc" }], + }, + }).model; + + const groups = model + .getState() + .snapshot.range(0, 100) + .flatMap((row) => + row.kind === "group" + ? [ + `${String(row.value)}:${String( + (row.aggregates as { label: unknown }).label, + )}`, + ] + : [], + ); + // A traverses note asc = t1("1"), then the tie by SOURCE order: t3("3") + // before t2("2"). + expect(groups).toEqual(["A:132", "B:4"]); + }); +}); diff --git a/packages/row-model/src/__tests__/sort-key-store.test.ts b/packages/row-model/src/__tests__/sort-key-store.test.ts new file mode 100644 index 000000000..ee5a0e18b --- /dev/null +++ b/packages/row-model/src/__tests__/sort-key-store.test.ts @@ -0,0 +1,784 @@ +import { describe, expect, test, vi } from "vitest"; + +import { + compileQuery, + createColumnHelper, + createLocalRowModel, + PretableRowModelError, + type PretableQueryFor, +} from "../index"; +import { + compareRecordRows, + compareWithSortKeys, + fillSortKeysFromPrevious, + sortKeysOf, +} from "../compiled-query"; + +interface Holding { + id: string; + team: string; + score: number; + note: string; + label: string; +} + +const helper = createColumnHelper(); + +/** + * Checks a query literal against a column tuple, exactly as + * `query-delta.test.ts` does. + */ +function queryFor( + value: PretableQueryFor, +): PretableQueryFor { + return value; +} + +/** + * Spied accessors on every column so tests can assert exactly which accessors + * a store fill runs. `label` is inactive in every plan below, so its spy count + * must be 0 throughout. + */ +function createFixture() { + const teamAccessor = vi.fn((row: Holding) => row.team); + const scoreAccessor = vi.fn((row: Holding) => row.score); + const noteAccessor = vi.fn((row: Holding) => row.note); + const labelAccessor = vi.fn((row: Holding) => row.label); + const columns = [ + helper.accessor("team", teamAccessor, { type: "text" }), + helper.accessor("score", scoreAccessor, { type: "number" }), + helper.accessor("note", noteAccessor, { type: "text" }), + helper.accessor("label", labelAccessor, { type: "text" }), + ] as const; + return { columns, teamAccessor, scoreAccessor, noteAccessor, labelAccessor }; +} + +type FixtureColumns = ReturnType["columns"]; + +const SCORE_ASC = queryFor({ + filters: [], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [], +}); + +const NOTE_THEN_SCORE = queryFor({ + filters: [], + sort: [ + { columnId: "note", direction: "asc" }, + { columnId: "score", direction: "asc" }, + ], + rowGroups: [], +}); + +function holding(partial: Partial & { id: string }): Holding { + return { + team: "Alpha", + score: 0, + note: "steady", + label: "unused", + ...partial, + }; +} + +const orderingTable = [ + { + name: "number asc", + columns: [helper.accessor("score", { type: "number" })] as const, + sort: [{ columnId: "score", direction: "asc" }], + left: holding({ id: "a", score: 5 }), + right: holding({ id: "b", score: 9 }), + expected: -1, + }, + { + name: "number desc", + columns: [helper.accessor("score", { type: "number" })] as const, + sort: [{ columnId: "score", direction: "desc" }], + left: holding({ id: "a", score: 5 }), + right: holding({ id: "b", score: 9 }), + expected: 1, + }, + { + name: "text collation (numeric-aware)", + columns: [helper.accessor("note", { type: "text" })] as const, + sort: [{ columnId: "note", direction: "asc" }], + left: holding({ id: "a", note: "item2" }), + right: holding({ id: "b", note: "item10" }), + expected: -1, + }, + { + name: "nulls first", + columns: [helper.accessor("note", { type: "text" })] as const, + sort: [{ columnId: "note", direction: "asc", nulls: "first" }], + left: holding({ id: "a", note: null as unknown as string }), + right: holding({ id: "b", note: "steady" }), + expected: -1, + }, + { + name: "nulls last", + columns: [helper.accessor("note", { type: "text" })] as const, + sort: [{ columnId: "note", direction: "asc", nulls: "last" }], + left: holding({ id: "a", note: null as unknown as string }), + right: holding({ id: "b", note: "steady" }), + expected: 1, + }, + { + name: "custom comparator", + columns: [ + helper.accessor("note", { + type: "text", + compare: (left: string, right: string) => left.length - right.length, + }), + ] as const, + sort: [{ columnId: "note", direction: "asc" }], + left: holding({ id: "a", note: "bbb" }), + right: holding({ id: "b", note: "a" }), + expected: 1, + }, + { + name: "sort-key tie resolves by sourceOrder", + columns: [helper.accessor("score", { type: "number" })] as const, + sort: [{ columnId: "score", direction: "asc" }], + left: holding({ id: "a", score: 5 }), + right: holding({ id: "b", score: 5 }), + expected: -1, + }, +]; + +describe("compareRecordRows", () => { + test("evaluate populates the store: comparison runs no accessors", () => { + const fixture = createFixture(); + const plan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const a = { + rowId: "a", + row: holding({ id: "a", score: 5 }), + sourceOrder: 0, + }; + const b = { + rowId: "b", + row: holding({ id: "b", score: 9 }), + sourceOrder: 1, + }; + plan.evaluate(a); + plan.evaluate(b); + + fixture.scoreAccessor.mockClear(); + fixture.teamAccessor.mockClear(); + fixture.noteAccessor.mockClear(); + + expect(compareRecordRows(plan, a, b)).toBeLessThan(0); + expect(compareRecordRows(plan, b, a)).toBeGreaterThan(0); + expect(fixture.scoreAccessor).not.toHaveBeenCalled(); + expect(fixture.teamAccessor).not.toHaveBeenCalled(); + expect(fixture.noteAccessor).not.toHaveBeenCalled(); + expect(fixture.labelAccessor).not.toHaveBeenCalled(); + }); + + test("the store holds one frozen array per evaluated row", () => { + const fixture = createFixture(); + const plan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const input = { + rowId: "a", + row: holding({ id: "a", score: 5 }), + sourceOrder: 0, + }; + plan.evaluate(input); + + fixture.scoreAccessor.mockClear(); + // Idempotent fill against an already-populated plan surfaces the stored + // entry — the exact array `evaluate` wrote, not a copy. + const stored = fillSortKeysFromPrevious(plan, plan, input); + + expect(stored).toBe(sortKeysOf(plan, input)); + expect(Object.isFrozen(stored)).toBe(true); + expect(stored).toEqual([{ columnId: "score", value: 5 }]); + expect(fixture.scoreAccessor).not.toHaveBeenCalled(); + }); + + test.each(orderingTable)( + "sign-equals compareRecordRows: $name", + ({ columns, sort, left, right, expected }) => { + const plan = compileQuery({ + derivations: columns, + query: { + filters: [], + sort, + rowGroups: [], + } as unknown as PretableQueryFor, + }); + const leftInput = { rowId: left.id, row: left, sourceOrder: 0 }; + const rightInput = { rowId: right.id, row: right, sourceOrder: 1 }; + plan.evaluate(leftInput); + plan.evaluate(rightInput); + + // The expectations were pinned against the metadata comparator before + // its deletion; antisymmetry is asserted alongside the sign. + expect(Math.sign(compareRecordRows(plan, leftInput, rightInput))).toBe( + expected, + ); + expect(Math.sign(compareRecordRows(plan, rightInput, leftInput))).toBe( + -expected, + ); + }, + ); + + test("fails loud on a row the plan never evaluated", () => { + const fixture = createFixture(); + const plan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const known = { + rowId: "a", + row: holding({ id: "a", score: 5 }), + sourceOrder: 0, + }; + const stranger = { + rowId: "b", + row: holding({ id: "b", score: 9 }), + sourceOrder: 1, + }; + plan.evaluate(known); + + expect(() => compareRecordRows(plan, known, stranger)).toThrowError( + /has no sort keys under this plan/, + ); + expect(() => compareRecordRows(plan, stranger, known)).toThrowError( + /has no sort keys under this plan/, + ); + }); + + test("TypeError for a foreign plan object", () => { + const fixture = createFixture(); + const plan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const input = { + rowId: "a", + row: holding({ id: "a", score: 5 }), + sourceOrder: 0, + }; + plan.evaluate(input); + const foreign = { query: SCORE_ASC, derivations: fixture.columns }; + + expect(() => + compareRecordRows(foreign as never, input, input), + ).toThrowError( + new TypeError("Record comparison requires a compiled query plan."), + ); + }); +}); + +describe("compareWithSortKeys", () => { + test.each(orderingTable)( + "sign-equals compareRecordRows over pre-resolved keys: $name", + ({ columns, sort, left, right, expected }) => { + const plan = compileQuery({ + derivations: columns, + query: { + filters: [], + sort, + rowGroups: [], + } as unknown as PretableQueryFor, + }); + const leftInput = { rowId: left.id, row: left, sourceOrder: 0 }; + const rightInput = { rowId: right.id, row: right, sourceOrder: 1 }; + plan.evaluate(leftInput); + plan.evaluate(rightInput); + const leftKeys = sortKeysOf(plan, leftInput); + const rightKeys = sortKeysOf(plan, rightInput); + + const decorated = compareWithSortKeys( + plan, + leftInput, + leftKeys, + rightInput, + rightKeys, + ); + expect(Math.sign(decorated)).toBe(expected); + expect(Math.sign(decorated)).toBe( + Math.sign(compareRecordRows(plan, leftInput, rightInput)), + ); + expect( + Math.sign( + compareWithSortKeys(plan, rightInput, rightKeys, leftInput, leftKeys), + ), + ).toBe(-expected); + }, + ); + + test("honors the PASSED keys and never falls back to the store", () => { + const fixture = createFixture(); + const plan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const a = { + rowId: "a", + row: holding({ id: "a", score: 5 }), + sourceOrder: 0, + }; + const b = { + rowId: "b", + row: holding({ id: "b", score: 9 }), + sourceOrder: 1, + }; + plan.evaluate(a); + plan.evaluate(b); + // The store says a < b. Deliberately wrong keys for `a` invert that: if + // the comparator resolved from the store instead of the arguments, the + // sign would stay negative and this assertion would fail. + const wrongKeysForA = Object.freeze([ + Object.freeze({ columnId: "score" as const, value: 100 }), + ]); + + expect( + compareWithSortKeys(plan, a, wrongKeysForA, b, sortKeysOf(plan, b)), + ).toBeGreaterThan(0); + expect( + compareWithSortKeys(plan, a, sortKeysOf(plan, a), b, sortKeysOf(plan, b)), + ).toBeLessThan(0); + }); + + test("TypeError for a foreign plan object", () => { + const fixture = createFixture(); + const plan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const input = { + rowId: "a", + row: holding({ id: "a", score: 5 }), + sourceOrder: 0, + }; + plan.evaluate(input); + const keys = sortKeysOf(plan, input); + const foreign = { query: SCORE_ASC, derivations: fixture.columns }; + + expect(() => + compareWithSortKeys( + foreign as never, + input, + keys as never, + input, + keys as never, + ), + ).toThrowError( + new TypeError("Key comparison requires a compiled query plan."), + ); + }); +}); + +describe("store-backed grouped pipeline (reroute pin)", () => { + /** + * Regression tripwire for the A2 comparator reroutes: a full cooperative + * setQuery over groups + aggregates + multi-column sort + a filter, with + * the final visible order and an aggregate value HARDCODED from the + * fixture by hand. Written green BEFORE the reroutes; any drift after a + * reroute is a real behavior change, not a fixture artifact. + */ + test("cooperative sort+filter change keeps hand-computed order and aggregates", async () => { + const columns = [ + helper.accessor("team", { type: "text" }), + helper.accessor("score", { type: "number", aggregate: "sum" }), + helper.accessor("note", { type: "text" }), + helper.accessor("label", { type: "text" }), + ] as const; + const rows: Holding[] = [ + holding({ id: "r1", team: "A", score: 5, note: "x" }), + holding({ id: "r2", team: "A", score: 3, note: "y" }), + holding({ id: "r3", team: "B", score: 8, note: "x" }), + holding({ id: "r4", team: "B", score: 1, note: "y" }), + holding({ id: "r5", team: "A", score: 7, note: "x" }), + holding({ id: "r6", team: "B", score: 9, note: "z" }), + ]; + const model = createLocalRowModel({ + rows, + columns, + // Default aggregation population: the rows the filter keeps. + initialExpansion: { kind: "expanded" }, + query: { + filters: [], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [{ columnId: "team", direction: "asc" }], + }, + }); + + const transition = model.setQuery({ + filters: [{ columnId: "score", operator: "gte", value: 3 }], + sort: [ + { columnId: "note", direction: "asc" }, + { columnId: "score", direction: "desc" }, + ], + rowGroups: [{ columnId: "team", direction: "asc" }], + }); + await transition.finished; + + /* + * Hand derivation from the fixture: + * - filter score >= 3 keeps r1(5) r2(3) r3(8) r5(7) r6(9); drops r4(1). + * - groups by team asc: A = {r1, r2, r5}, B = {r3, r6}. + * - within-group sort, note asc then score desc: + * A: note "x" -> r5(7) then r1(5) (desc), then note "y" -> r2(3). + * B: note "x" -> r3, then note "z" -> r6. + * - aggregate sum(score) over the filtered population (the default): + * A = 5 + 3 + 7 = 15; B = 8 + 9 = 17. + */ + const snapshot = model.getState().snapshot; + const shape = snapshot.range(0, 100).map((row) => { + if (row.kind === "group") { + return `group:${String(row.value)}:sum=${String( + (row.aggregates as { score: unknown }).score, + )}`; + } + return row.rowId; + }); + expect(shape).toEqual([ + "group:A:sum=15", + "r5", + "r1", + "r2", + "group:B:sum=17", + "r3", + "r6", + ]); + expect(model.getState().status).toEqual({ kind: "ready" }); + }); + + /** + * The same-reference-mutation recompile is a plan swap: the fresh plan's + * store must be seeded for carried rows, and the visible index must be + * rebuilt under the fresh plan (the retired plan's store still holds + * pre-mutation keys for mutated row objects). Found by probing during the + * A2 reroutes — without both fixes the mutated row keeps its stale rank + * and a later update of a carried row throws the fail-loud store miss. + */ + test("same-reference mutation recompile re-ranks and keeps carried rows comparable", () => { + interface Simple { + id: number; + value: number; + label: string; + } + const simpleHelper = createColumnHelper(); + const columns = [ + simpleHelper.accessor("value", { type: "number" }), + ] as const; + const mutable = Object.preventExtensions({ id: 1, value: 1, label: "a" }); + const b = { id: 2, value: 2, label: "b" }; + const c = { id: 3, value: 3, label: "c" }; + const model = createLocalRowModel({ + rows: [mutable, b, c], + columns, + query: { + filters: [], + sort: [{ columnId: "value", direction: "asc" }], + rowGroups: [], + }, + }); + + // Mutate in place, then reorder the untouched carried rows so one of + // them flows through the cached-metadata path under the fresh plan. + mutable.value = 10; + model.setRows([c, b, mutable]); + expect(model.getState().snapshot.range(0, 3)).toMatchObject([ + { rowId: 2 }, + { rowId: 3 }, + { rowId: 1 }, + ]); + + // Follow-up update of a carried row: its previous record must resolve + // from the committed root's (recompiled) plan. + model.setRows([c, { ...b, value: 5 }, mutable]); + expect(model.getState().snapshot.range(0, 3)).toMatchObject([ + { rowId: 3 }, + { rowId: 2 }, + { rowId: 1 }, + ]); + }); +}); + +describe("sortKeysOf", () => { + test("returns the store's array by identity, stable across calls", () => { + const fixture = createFixture(); + const plan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const input = { + rowId: "a", + row: holding({ id: "a", score: 5 }), + sourceOrder: 0, + }; + plan.evaluate(input); + + fixture.scoreAccessor.mockClear(); + const first = sortKeysOf(plan, input); + const second = sortKeysOf(plan, input); + + expect(first).toEqual([{ columnId: "score", value: 5 }]); + // Identity, not a per-call copy: consumers may compare arrays by + // reference, and resolution must never re-run accessors. + expect(second).toBe(first); + expect(Object.isFrozen(first)).toBe(true); + expect(fixture.scoreAccessor).not.toHaveBeenCalled(); + }); + + test("fails loud on a row the plan never evaluated", () => { + const fixture = createFixture(); + const plan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const stranger = { + rowId: "ghost", + row: holding({ id: "ghost", score: 9 }), + sourceOrder: 0, + }; + + expect(() => sortKeysOf(plan, stranger)).toThrowError( + /has no sort keys under this plan/, + ); + }); + + test("TypeError for a foreign plan object", () => { + const fixture = createFixture(); + const plan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const input = { + rowId: "a", + row: holding({ id: "a", score: 5 }), + sourceOrder: 0, + }; + plan.evaluate(input); + const foreign = { query: SCORE_ASC, derivations: fixture.columns }; + + expect(() => sortKeysOf(foreign as never, input)).toThrowError( + new TypeError("Sort-key resolution requires a compiled query plan."), + ); + }); +}); + +describe("fillSortKeysFromPrevious", () => { + test("carries overlapping sort columns and evaluates newly-active ones once", () => { + const fixture = createFixture(); + const previousPlan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const nextPlan = compileQuery({ + derivations: fixture.columns, + query: NOTE_THEN_SCORE, + }); + const input = { + rowId: "a", + row: holding({ id: "a", score: 5, note: "steady" }), + sourceOrder: 0, + }; + previousPlan.evaluate(input); + + fixture.scoreAccessor.mockClear(); + fixture.noteAccessor.mockClear(); + const keys = fillSortKeysFromPrevious(nextPlan, previousPlan, input); + + expect(keys).toEqual([ + { columnId: "note", value: "steady" }, + { columnId: "score", value: 5 }, + ]); + // Overlapping column carried from the previous plan's store, not re-run. + expect(fixture.scoreAccessor).not.toHaveBeenCalled(); + // Newly-active column evaluated exactly once. + expect(fixture.noteAccessor).toHaveBeenCalledTimes(1); + expect(Object.isFrozen(keys)).toBe(true); + // The fill makes the next plan's record comparator usable. + const other = { + rowId: "b", + row: holding({ id: "b", score: 9, note: "zzz" }), + sourceOrder: 1, + }; + fillSortKeysFromPrevious(nextPlan, previousPlan, other); + expect(compareRecordRows(nextPlan, input, other)).toBeLessThan(0); + }); + + test("idempotent: a second fill returns the same array with zero accessor runs", () => { + const fixture = createFixture(); + const previousPlan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const nextPlan = compileQuery({ + derivations: fixture.columns, + query: NOTE_THEN_SCORE, + }); + const input = { + rowId: "a", + row: holding({ id: "a", score: 5 }), + sourceOrder: 0, + }; + previousPlan.evaluate(input); + const first = fillSortKeysFromPrevious(nextPlan, previousPlan, input); + + fixture.teamAccessor.mockClear(); + fixture.scoreAccessor.mockClear(); + fixture.noteAccessor.mockClear(); + const second = fillSortKeysFromPrevious(nextPlan, previousPlan, input); + + expect(second).toBe(first); + expect(fixture.teamAccessor).not.toHaveBeenCalled(); + expect(fixture.scoreAccessor).not.toHaveBeenCalled(); + expect(fixture.noteAccessor).not.toHaveBeenCalled(); + }); + + test("a keys-only fill upgrades cleanly when evaluate later sees the row", () => { + const fixture = createFixture(); + const previousPlan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const nextPlan = compileQuery({ + derivations: fixture.columns, + query: NOTE_THEN_SCORE, + }); + const input = { + rowId: "a", + row: holding({ id: "a", score: 5, note: "steady" }), + sourceOrder: 0, + }; + previousPlan.evaluate(input); + // Keys-only state under nextPlan: filled, never evaluated. + const filled = fillSortKeysFromPrevious(nextPlan, previousPlan, input); + expect(sortKeysOf(nextPlan, input)).toBe(filled); + + // Evaluate must NOT treat the keys-only state as a metadata cache hit — + // it produces coherent metadata and refreshes the stored keys. + const metadata = nextPlan.evaluate(input); + expect(metadata.filterPasses).toBe(true); + expect(metadata.rowId).toBe("a"); + const afterEvaluate = sortKeysOf(nextPlan, input); + expect(afterEvaluate).toEqual([ + { columnId: "note", value: "steady" }, + { columnId: "score", value: 5 }, + ]); + // A second evaluate is a cache hit; a later fill surfaces the stored + // array by identity with zero accessor runs. + expect(nextPlan.evaluate(input)).toBe(metadata); + fixture.scoreAccessor.mockClear(); + fixture.noteAccessor.mockClear(); + expect(fillSortKeysFromPrevious(nextPlan, previousPlan, input)).toBe( + afterEvaluate, + ); + expect(fixture.scoreAccessor).not.toHaveBeenCalled(); + expect(fixture.noteAccessor).not.toHaveBeenCalled(); + }); + + test("runs the accessor even when the previous plan never saw the row", () => { + const fixture = createFixture(); + const previousPlan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const nextPlan = compileQuery({ + derivations: fixture.columns, + query: NOTE_THEN_SCORE, + }); + const input = { + rowId: "a", + row: holding({ id: "a", score: 5, note: "steady" }), + sourceOrder: 0, + }; + // No previousPlan.evaluate: nothing to carry, every column re-runs. + const keys = fillSortKeysFromPrevious(nextPlan, previousPlan, input); + + expect(keys).toEqual([ + { columnId: "note", value: "steady" }, + { columnId: "score", value: 5 }, + ]); + expect(fixture.scoreAccessor).toHaveBeenCalledTimes(1); + expect(fixture.noteAccessor).toHaveBeenCalledTimes(1); + }); + + test("accessor failure surfaces evaluate's error shape", () => { + const boom = new Error("boom"); + const columns = [ + helper.accessor("score", { type: "number" }), + helper.accessor( + "note", + // Annotated so the throwing accessor still types as a text column; + // an inferred `never` value type breaks the tuple under typecheck. + (): string => { + throw boom; + }, + { type: "text" }, + ), + ] as const; + const previousPlan = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [], + sort: [{ columnId: "score", direction: "asc" }], + rowGroups: [], + }), + }); + const nextPlan = compileQuery({ + derivations: columns, + query: queryFor({ + filters: [], + sort: [{ columnId: "note", direction: "asc" }], + rowGroups: [], + }), + }); + const input = { + rowId: "r1", + row: holding({ id: "r1", score: 5 }), + sourceOrder: 0, + }; + previousPlan.evaluate(input); + + let caught: unknown; + try { + fillSortKeysFromPrevious(nextPlan, previousPlan, input); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(PretableRowModelError); + const error = caught as PretableRowModelError; + expect(error.code).toBe("accessor-failed"); + expect(error.operation).toBe("set-query"); + expect(error.rowId).toBe("r1"); + expect(error.columnId).toBe("note"); + expect(error.cause).toBe(boom); + }); + + test("TypeError for foreign plan objects in either position", () => { + const fixture = createFixture(); + const plan = compileQuery({ + derivations: fixture.columns, + query: SCORE_ASC, + }); + const input = { + rowId: "a", + row: holding({ id: "a", score: 5 }), + sourceOrder: 0, + }; + plan.evaluate(input); + const foreign = { query: SCORE_ASC, derivations: fixture.columns }; + + expect(() => + fillSortKeysFromPrevious(foreign as never, plan, input), + ).toThrowError( + new TypeError("Sort-key carryover requires compiled query plans."), + ); + expect(() => + fillSortKeysFromPrevious(plan, foreign as never, input), + ).toThrowError( + new TypeError("Sort-key carryover requires compiled query plans."), + ); + }); +}); diff --git a/packages/row-model/src/__tests__/transitions.test.ts b/packages/row-model/src/__tests__/transitions.test.ts index 6fbc133d7..488e10b72 100644 --- a/packages/row-model/src/__tests__/transitions.test.ts +++ b/packages/row-model/src/__tests__/transitions.test.ts @@ -418,7 +418,9 @@ describe("cooperative query and derivation transitions", () => { budgetMs: 1, }); const transition = model.setQuery({ - filters: [], + // The filter change keeps this off the #457 sort-only fast path; the + // subject is cooperative delta-journal accounting. + filters: [{ columnId: "score", operator: "gte", value: 0 }], sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); @@ -795,8 +797,10 @@ describe("cooperative query and derivation transitions", () => { }); const supersededStaleTask = scheduler.entries.at(-1)?.task; const replacement = model.setQuery({ - filters: [], - sort: [{ columnId: "score", direction: "desc" }], + // A filter change: a sort-only replacement would commit synchronously + // (#457) and this test needs a pending transition to dispose. + filters: [{ columnId: "score", operator: "gte", value: 5 }], + sort: [], rowGroups: [], }); await expect(superseded.finished).rejects.toMatchObject({ @@ -1091,7 +1095,10 @@ describe("cooperative query and derivation transitions", () => { }); evaluations = 0; const transition = model.setQuery({ - filters: [], + // Every row is team "A", so this filter changes no verdict but keeps + // the query off the #457 sort-only fast path; the subject is the + // cooperative catch-up machinery. + filters: [{ columnId: "team", operator: "equals", value: "A" }], sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); @@ -1133,7 +1140,10 @@ describe("cooperative query and derivation transitions", () => { transitionMaxUnitsPerSlice: 1, }); const transition = model.setQuery({ - filters: [], + // Every row is team "A", so this filter changes no verdict but keeps + // the query off the #457 sort-only fast path; the subject is the + // cooperative catch-up machinery. + filters: [{ columnId: "team", operator: "equals", value: "A" }], sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); @@ -1181,7 +1191,10 @@ describe("cooperative query and derivation transitions", () => { transitionMaxUnitsPerSlice: 1, }); const transition = model.setQuery({ - filters: [], + // Every row is team "A", so this filter changes no verdict but keeps + // the query off the #457 sort-only fast path; the subject is the + // cooperative catch-up machinery. + filters: [{ columnId: "team", operator: "equals", value: "A" }], sort: [{ columnId: "score", direction: "desc" }], rowGroups: [], }); diff --git a/packages/row-model/src/__tests__/types.test.ts b/packages/row-model/src/__tests__/types.test.ts index 3fe84b98b..08a2d0f4a 100644 --- a/packages/row-model/src/__tests__/types.test.ts +++ b/packages/row-model/src/__tests__/types.test.ts @@ -585,7 +585,8 @@ function assertOperationalSignatures( const _from: number = sequence.fromRevision; void _from; } else { - const _reason: "unknown-revision" | "journal-evicted" | "bulk-replace" = + const _reason: + "unknown-revision" | "journal-evicted" | "bulk-replace" | "reorder" = sequence.reason; void _reason; } diff --git a/packages/row-model/src/change-journal.ts b/packages/row-model/src/change-journal.ts index dc3a27f22..ea09b17fa 100644 --- a/packages/row-model/src/change-journal.ts +++ b/packages/row-model/src/change-journal.ts @@ -283,6 +283,26 @@ export function createChangeJournal( ); if (start < 0) return reset(currentRevision, "unknown-revision"); const retained = entries.slice(start); + // "reorder" is a PROMISE (order moved, nothing else), so it only + // survives aggregation when every entry in the range is a reorder + // barrier. Any other entry — changes or a plain barrier — voids the + // promise and the whole range degrades to a plain bulk reset. + let allReorder = retained.length > 0; + let reorderExpected = fromRevision; + for (const entry of retained) { + if ( + entry.kind !== "barrier" || + entry.reason !== "reorder" || + entry.previousRevision !== reorderExpected + ) { + allReorder = false; + break; + } + reorderExpected = entry.revision; + } + if (allReorder && reorderExpected === currentRevision) { + return reset(currentRevision, "reorder"); + } let expected = fromRevision; const changeSets: PretableChangeSet[] = []; for (const entry of retained) { @@ -290,7 +310,10 @@ export function createChangeJournal( return reset(currentRevision, "unknown-revision"); } if (entry.kind === "barrier") { - return reset(currentRevision, entry.reason); + return reset( + currentRevision, + entry.reason === "reorder" ? "bulk-replace" : entry.reason, + ); } changeSets.push(entry.changeSet); expected = entry.changeSet.revision; diff --git a/packages/row-model/src/compiled-query.ts b/packages/row-model/src/compiled-query.ts index 9db3c655e..38ede769a 100644 --- a/packages/row-model/src/compiled-query.ts +++ b/packages/row-model/src/compiled-query.ts @@ -30,6 +30,14 @@ export type CompiledSortKey = CompiledValueForDescriptor< ColumnDescriptorOf >; +/** + * The aggregate leaf's per-evaluation payload. It carries the row's sort + * keys so aggregate-tree comparators are property reads — the leaf-side + * variant of `OrderedRowEntry` (wrapping the leaf itself would ripple + * through the aggregation machinery's `value`/`id`/`row` reads). Keys stay + * valid for the containing tree's lifetime: leaves are rebuilt whenever + * their row re-evaluates, and aggregate trees are bound to one plan. + */ export interface CompiledAggregateDependency { readonly sourceOrder: number; readonly sortKeys: readonly CompiledSortKey[]; @@ -86,6 +94,18 @@ export interface CompiledRowInput< readonly sourceOrder: number; } +/** + * Structural slice of `LocalRowModelInstrumentation` consumed by + * `fillSortKeysFromPrevious`. Declared here (not imported from + * `./diagnostics`) so this module stays free of import cycles. + */ +export interface SortKeyFillInstrumentation { + readonly work: { + sortKeyCarries: number; + sortKeyEvaluations: number; + }; +} + export interface CompiledRowMetadata< TRow extends object, TRowId extends PretableRowId, @@ -96,7 +116,6 @@ export interface CompiledRowMetadata< readonly sourceOrder: number; readonly filterPasses: boolean; readonly groupPath: readonly CompiledGroupKey[]; - readonly sortKeys: readonly CompiledSortKey[]; /** `allLeaf` always exists; `filteredLeaf` exists only when filters pass. */ readonly aggregateLeaves: readonly CompiledAggregateLeaf[]; } @@ -108,14 +127,6 @@ export interface CompiledQuery { evaluate( input: CompiledRowInput, TRowId>, ): CompiledRowMetadata, TRowId, TColumns>; - /** - * Custom comparators must return a number other than `NaN`. Positive and - * negative infinity are accepted as explicit positive/negative ordering. - */ - readonly compareRows: ( - left: CompiledRowMetadata, TRowId, TColumns>, - right: CompiledRowMetadata, TRowId, TColumns>, - ) => number; /** * Compares sibling group keys with the same policy as row sorting. Missing * values (`null`, `undefined`, and `NaN`) default to last. An explicit @@ -252,10 +263,25 @@ interface RuntimeQuery { readonly rowGroups: readonly RuntimeOrdering[]; } +/* + * One WeakMap entry per row per plan, written by `evaluate` (full) or + * `fillSortKeysFromPrevious` (keys-only: `metadata` absent). Merged into a + * single map deliberately: a second per-row WeakMap doubles the synchronous + * ephemeron-table rehash V8 performs at the 2/3-capacity threshold inside + * ONE cooperative unit (~87k entries at 100k rows) — the measured worst + * slice of the grouped rebuild. Fields are mutable so an upgrade reuses the + * entry object: at most one `WeakMap.set` per row, one rehash. + * + * `metadata` reads stay guarded by the rowId/sourceOrder identity check. + * `sortKeys` reads are UNGUARDED: keys depend only on the row object's + * values and this plan's sort columns — they embed no rowId/sourceOrder, so + * re-evaluation under a changed sourceOrder overwrites harmlessly. + */ interface CachedEvaluation { - readonly rowId: PretableRowId; - readonly sourceOrder: number; - readonly metadata: object; + rowId: PretableRowId; + sourceOrder: number; + metadata: object | undefined; + sortKeys: readonly { readonly columnId: string; readonly value: unknown }[]; } const internals = Symbol("compiled-query-internals"); @@ -841,18 +867,22 @@ function semanticValueEqual(left: unknown, right: unknown): boolean { return false; } -function queryEqual(left: RuntimeQuery, right: RuntimeQuery): boolean { - const orderingEqual = ( - a: readonly RuntimeOrdering[], - b: readonly RuntimeOrdering[], - ) => +function orderingEqual( + a: readonly RuntimeOrdering[], + b: readonly RuntimeOrdering[], +): boolean { + return ( a.length === b.length && a.every( (entry, index) => entry.columnId === b[index].columnId && (entry.direction ?? "asc") === (b[index].direction ?? "asc") && (entry.nulls ?? "last") === (b[index].nulls ?? "last"), - ); + ) + ); +} + +function queryEqual(left: RuntimeQuery, right: RuntimeQuery): boolean { return ( filtersEqual(left.filters, right.filters) && orderingEqual(left.sort, right.sort) && @@ -1429,6 +1459,7 @@ class CompiledQueryPlan const cached = this.#evaluationCache.get(input.row); if ( cached && + cached.metadata !== undefined && Object.is(cached.rowId, input.rowId) && cached.sourceOrder === input.sourceOrder ) { @@ -1472,11 +1503,35 @@ class CompiledQueryPlan }), ), ) as readonly CompiledGroupKey[]; + return this.#finalizeMetadata({ + rowId: input.rowId, + row: input.row, + sourceOrder: input.sourceOrder, + filterPasses, + groupPath, + valueOf: (columnId) => values.get(columnId), + }); + } + + /* + * Tail of `evaluate`: writes the row's sort keys to the plan's store, + * builds the dependency, aggregate leaves, and the frozen metadata from a + * per-column value source, then seeds the evaluation cache. `valueOf` must + * cover every sorted and aggregated column of THIS plan. + */ + #finalizeMetadata(input: { + readonly rowId: TRowId; + readonly row: RowForColumns; + readonly sourceOrder: number; + readonly filterPasses: boolean; + readonly groupPath: readonly CompiledGroupKey[]; + readonly valueOf: (columnId: string) => unknown; + }): CompiledRowMetadata, TRowId, TColumns> { const sortKeys = Object.freeze( this.#runtimeQuery.sort.map((entry) => Object.freeze({ columnId: entry.columnId, - value: values.get(entry.columnId), + value: input.valueOf(entry.columnId), }), ), ) as readonly CompiledSortKey[]; @@ -1489,14 +1544,14 @@ class CompiledQueryPlan const allLeaf = Object.freeze({ id: input.rowId, row: input.row, - value: values.get(column.id), + value: input.valueOf(column.id), dependency, }); return Object.freeze({ columnId: column.id, aggregate: column.aggregate, allLeaf, - filteredLeaf: filterPasses ? allLeaf : undefined, + filteredLeaf: input.filterPasses ? allLeaf : undefined, }); }), ) as unknown as readonly CompiledAggregateLeaf[]; @@ -1504,30 +1559,48 @@ class CompiledQueryPlan rowId: input.rowId, row: input.row, sourceOrder: input.sourceOrder, - filterPasses, - groupPath, - sortKeys, + filterPasses: input.filterPasses, + groupPath: input.groupPath, aggregateLeaves, }) as CompiledRowMetadata, TRowId, TColumns>; - this.#evaluationCache.set(input.row, { - rowId: input.rowId, - sourceOrder: input.sourceOrder, - metadata, - }); + const existing = this.#evaluationCache.get(input.row); + if (existing === undefined) { + this.#evaluationCache.set(input.row, { + rowId: input.rowId, + sourceOrder: input.sourceOrder, + metadata, + sortKeys, + }); + } else { + // Upgrade a keys-only entry (or refresh a stale full one) in place — + // no second WeakMap.set, so no second rehash risk. + existing.rowId = input.rowId; + existing.sourceOrder = input.sourceOrder; + existing.metadata = metadata; + existing.sortKeys = sortKeys; + } return metadata; } - readonly compareRows = ( - left: CompiledRowMetadata, TRowId, TColumns>, - right: CompiledRowMetadata, TRowId, TColumns>, - ): number => { + /* + * The single comparison loop behind `compareRecordRows`: per-ordering + * `compareValues` over store-resolved keys, then the `sourceOrder` + * tiebreak. Kept separate from key resolution so both sides resolve + * before any comparison runs. + */ + #compareBySortKeys( + left: { readonly rowId: PretableRowId; readonly sourceOrder: number }, + leftKeys: readonly CompiledSortKey[], + right: { readonly rowId: PretableRowId; readonly sourceOrder: number }, + rightKeys: readonly CompiledSortKey[], + ): number { for (let index = 0; index < this.#runtimeQuery.sort.length; index += 1) { const ordering = this.#runtimeQuery.sort[index]; const column = this.#byId.get(ordering.columnId)!; try { const result = compareValues( - left.sortKeys[index]?.value, - right.sortKeys[index]?.value, + leftKeys[index]?.value, + rightKeys[index]?.value, column, ordering, ); @@ -1541,7 +1614,158 @@ class CompiledQueryPlan } } return left.sourceOrder - right.sourceOrder; - }; + } + + #resolveSortKeys(input: { + readonly rowId: PretableRowId; + readonly row: object; + }): readonly CompiledSortKey[] { + const keys = this.#evaluationCache.get(input.row)?.sortKeys as + readonly CompiledSortKey[] | undefined; + if (keys === undefined) { + throw new Error( + `Row ${String(input.rowId)} has no sort keys under this plan.`, + ); + } + return keys; + } + + /** + * Orders two evaluated rows by the plan's own sort-key store. A missing + * store entry is a defect — the fill points (`evaluate` and + * `fillSortKeysFromPrevious`) are exhaustive — so resolution throws rather + * than lazily re-running accessors. + */ + static compareRecordRows( + plan: unknown, + left: CompiledRowInput, TRowId>, + right: CompiledRowInput, TRowId>, + ): number { + if (!(plan instanceof CompiledQueryPlan)) { + throw new TypeError("Record comparison requires a compiled query plan."); + } + const compiled = plan as CompiledQueryPlan; + return compiled.#compareBySortKeys( + left, + compiled.#resolveSortKeys(left), + right, + compiled.#resolveSortKeys(right), + ); + } + + /** + * Orders two rows by keys the CALLER already resolved — no store lookups. + * Exists so O(n log n) sorts resolve keys once per row (decorate) instead + * of once per comparison; `compareRecordRows` remains the general entry. + * The comparison semantics are the same shared loop. + */ + static compareWithSortKeys( + plan: unknown, + left: CompiledRowInput, TRowId>, + leftKeys: readonly CompiledSortKey[], + right: CompiledRowInput, TRowId>, + rightKeys: readonly CompiledSortKey[], + ): number { + if (!(plan instanceof CompiledQueryPlan)) { + throw new TypeError("Key comparison requires a compiled query plan."); + } + return (plan as CompiledQueryPlan).#compareBySortKeys( + left, + leftKeys, + right, + rightKeys, + ); + } + + /** + * Resolves one evaluated row's keys from the plan's own store. Same + * fail-loud contract as `compareRecordRows`: a missing entry is a defect. + */ + static sortKeysOf( + plan: unknown, + input: CompiledRowInput, TRowId>, + ): readonly CompiledSortKey[] { + if (!(plan instanceof CompiledQueryPlan)) { + throw new TypeError( + "Sort-key resolution requires a compiled query plan.", + ); + } + return (plan as CompiledQueryPlan).#resolveSortKeys(input); + } + + /** + * Fills `nextPlan`'s store for one row from `previousPlan`'s: values carry + * by columnId where the sort columns overlap, accessors run only for + * newly-active sort columns. Precondition (caller-owned): + * `isSortOnlyChange(previousPlan, nextPlan)`, so carried values are the + * ones the next plan's accessors would produce. When instrumentation is + * supplied, one counter is bumped per (row, sort column) entry — carry vs + * accessor — and an already-filled row counts nothing. + */ + static fillSortKeysFromPrevious( + nextPlan: unknown, + previousPlan: unknown, + input: CompiledRowInput, TRowId>, + instrumentation?: SortKeyFillInstrumentation, + ): readonly CompiledSortKey[] { + if ( + !(nextPlan instanceof CompiledQueryPlan) || + !(previousPlan instanceof CompiledQueryPlan) + ) { + throw new TypeError("Sort-key carryover requires compiled query plans."); + } + const next = nextPlan as CompiledQueryPlan; + const previous = previousPlan as CompiledQueryPlan; + const existing = next.#evaluationCache.get(input.row); + if (existing !== undefined) { + return existing.sortKeys as readonly CompiledSortKey[]; + } + + const carried = previous.#evaluationCache.get(input.row)?.sortKeys as + readonly CompiledSortKey[] | undefined; + const sortKeys = Object.freeze( + next.#runtimeQuery.sort.map((entry) => { + const previousKey = carried?.find( + (key) => key.columnId === entry.columnId, + ); + if (previousKey !== undefined) { + if (instrumentation !== undefined) + instrumentation.work.sortKeyCarries += 1; + return Object.freeze({ + columnId: entry.columnId, + value: previousKey.value, + }); + } + let value: unknown; + try { + value = next.#byId.get(entry.columnId)!.accessor(input.row as never); + } catch (cause) { + throw new PretableRowModelError( + "accessor-failed", + `Column ${entry.columnId} accessor failed.`, + { + operation: next.#operation, + rowId: input.rowId, + columnId: entry.columnId, + cause, + }, + ); + } + if (instrumentation !== undefined) + instrumentation.work.sortKeyEvaluations += 1; + return Object.freeze({ columnId: entry.columnId, value }); + }), + ) as readonly CompiledSortKey[]; + // Keys-only entry: `metadata` stays absent, so a later `evaluate` for + // this row misses the metadata guard and upgrades the entry in place. + next.#evaluationCache.set(input.row, { + rowId: input.rowId, + sourceOrder: input.sourceOrder, + metadata: undefined, + sortKeys, + }); + return sortKeys; + } compareGroupKeys( level: number, @@ -1566,6 +1790,181 @@ class CompiledQueryPlan ); } } + + /** + * Facet delta between two plans this module compiled. `undefined` means + * "treat as everything changed" — a foreign object cannot be inspected, so + * it never qualifies as a narrow change. Facets are compared on the RUNTIME + * query, not the public one: under external sort authority the runtime + * sort is `[]` on both sides, so a public-only sort change classifies as + * `sortChanged: false` here, same as a true no-op. + */ + static classifyDelta( + previous: unknown, + next: unknown, + ): + | Readonly<{ + derivationsChanged: boolean; + filtersChanged: boolean; + groupsChanged: boolean; + sortChanged: boolean; + authorityChanged: boolean; + }> + | undefined { + if ( + !(previous instanceof CompiledQueryPlan) || + !(next instanceof CompiledQueryPlan) + ) + return undefined; + + // Both directions are required: a column active under only ONE side's + // query (e.g. sorted in `next` but not in `previous`) would escape a + // single-sided comparison, narrowing the conservatism guarantee. + const derivationsChanged = !( + derivationsEqualForPlan( + previous.#runtimeColumns, + next.#runtimeColumns, + previous.#runtimeQuery, + ) && + derivationsEqualForPlan( + previous.#runtimeColumns, + next.#runtimeColumns, + next.#runtimeQuery, + ) + ); + const filtersChanged = !filtersEqual( + previous.#runtimeQuery.filters, + next.#runtimeQuery.filters, + ); + const groupsChanged = !orderingEqual( + previous.#runtimeQuery.rowGroups, + next.#runtimeQuery.rowGroups, + ); + const sortChanged = !orderingEqual( + previous.#runtimeQuery.sort, + next.#runtimeQuery.sort, + ); + const authorityChanged = + previous.#filterAuthority !== next.#filterAuthority || + previous.#sortAuthority !== next.#sortAuthority; + + return Object.freeze({ + derivationsChanged, + filtersChanged, + groupsChanged, + sortChanged, + authorityChanged, + }); + } +} + +/** + * Facet delta between two compiled plans. `undefined` means either argument + * was not a plan this module compiled, and callers must treat that as + * "everything changed." + */ +export type CompiledQueryDelta = NonNullable< + ReturnType +>; + +export function classifyQueryDelta( + previous: CompiledQuery, + next: CompiledQuery, +): CompiledQueryDelta | undefined { + return CompiledQueryPlan.classifyDelta(previous, next); +} + +/** + * True only when the applied sort is the sole difference between the plans. + */ +export function isSortOnlyChange( + previous: CompiledQuery, + next: CompiledQuery, +): boolean { + const delta = classifyQueryDelta(previous, next); + return ( + delta !== undefined && + delta.sortChanged && + !delta.derivationsChanged && + !delta.filtersChanged && + !delta.groupsChanged && + !delta.authorityChanged + ); +} + +/** + * Orders two evaluated row records under `plan` via the plan's own sort-key + * store. Both rows must already be in the store (`evaluate` or + * `fillSortKeysFromPrevious`); a missing entry throws — it is a defect, not a + * lazy-fill opportunity. + */ +export function compareRecordRows( + plan: CompiledQuery, + left: CompiledRowInput, TRowId>, + right: CompiledRowInput, TRowId>, +): number { + return CompiledQueryPlan.compareRecordRows( + plan, + left, + right, + ); +} + +/** + * Orders two rows by keys the caller already resolved (via `sortKeysOf` or + * `fillSortKeysFromPrevious`) — no store lookups. Exists so O(n log n) sorts + * resolve keys once per row instead of once per comparison; + * `compareRecordRows` remains the general entry with identical semantics. + */ +export function compareWithSortKeys( + plan: CompiledQuery, + left: CompiledRowInput, TRowId>, + leftKeys: readonly CompiledSortKey[], + right: CompiledRowInput, TRowId>, + rightKeys: readonly CompiledSortKey[], +): number { + return CompiledQueryPlan.compareWithSortKeys( + plan, + left, + leftKeys, + right, + rightKeys, + ); +} + +/** + * Resolves one evaluated row's sort keys from `plan`'s own store. Both the + * shape and the fail-loud contract match `compareRecordRows`: the row must + * already be in the store, and a missing entry throws. + */ +export function sortKeysOf( + plan: CompiledQuery, + input: CompiledRowInput, TRowId>, +): readonly CompiledSortKey[] { + return CompiledQueryPlan.sortKeysOf(plan, input); +} + +/** + * Fills `nextPlan`'s sort-key store for one row, carrying values from + * `previousPlan`'s store where the sort columns overlap and running accessors + * only for newly-active sort columns. Idempotent per row. Valid ONLY when + * `isSortOnlyChange(previousPlan, nextPlan)` — the caller owns that check. + */ +export function fillSortKeysFromPrevious< + TColumns, + TRowId extends PretableRowId, +>( + nextPlan: CompiledQuery, + previousPlan: CompiledQuery, + input: CompiledRowInput, TRowId>, + instrumentation?: SortKeyFillInstrumentation, +): readonly CompiledSortKey[] { + return CompiledQueryPlan.fillSortKeysFromPrevious( + nextPlan, + previousPlan, + input, + instrumentation, + ); } export function compileQuery( diff --git a/packages/row-model/src/cooperative-transition.ts b/packages/row-model/src/cooperative-transition.ts index 22adec63e..800b1e48b 100644 --- a/packages/row-model/src/cooperative-transition.ts +++ b/packages/row-model/src/cooperative-transition.ts @@ -23,6 +23,7 @@ import { import type { TransientMap } from "./persistent/transient"; import { instrumentOrderStatisticTree } from "./persistent/order-statistic-tree"; import type { PretableGroupId } from "./types"; +import { orderedRowEntry } from "./ordered-row-entry"; import { createFlatVisibleTree } from "./visible-index"; export interface CooperativeTransitionScheduler { @@ -402,12 +403,7 @@ export function createCooperativeTransitionCandidate< sourceOrder: options.captured.sourceOrder, expansion: options.captured.expansion, flatRows: instrumentOrderStatisticTree( - createFlatVisibleTree( - options.queryPlan.compareRows as unknown as ( - left: RowRecord["metadata"], - right: RowRecord["metadata"], - ) => number, - ), + createFlatVisibleTree(options.queryPlan), instrumentation, ), groups: @@ -573,7 +569,9 @@ export function createCooperativeTransitionCandidate< state.groupBuilder.insert(record); } else if (state.groups === undefined) { if (metadata.filterPasses) { - state.flatRows = state.flatRows.insertOrReplace(record); + state.flatRows = state.flatRows.insertOrReplace( + orderedRowEntry(state.queryPlan, record), + ); } } else { state.groups = updateGroupIndex( diff --git a/packages/row-model/src/create-local-row-model.ts b/packages/row-model/src/create-local-row-model.ts index 69f40c9ac..329d43ef0 100644 --- a/packages/row-model/src/create-local-row-model.ts +++ b/packages/row-model/src/create-local-row-model.ts @@ -1,9 +1,12 @@ import { compileQuery, + fillSortKeysFromPrevious, + isSortOnlyChange, type CompiledFilterAuthority, type CompiledQuery, type CompiledSortAuthority, } from "./compiled-query"; +import { rebuildRootForSortOnlyChange } from "./sort-rebuild"; import { createCooperativeTransitionCandidate, createCooperativeTransitionRuntime, @@ -38,6 +41,7 @@ import type { ExpansionRoot, PretableRevisionCause, RevisionRoot, + RowRecord, } from "./internal-types"; import { attachGroupIndex, @@ -691,6 +695,25 @@ export function createLocalRowModel< } } }; + const publishCommittedRoot = ( + committedRoot: RevisionRoot, + previousRevision: number, + revision: number, + // Only the sort-only fast path may pass "reorder": it is the one commit + // that provably changes order and nothing else. Every other publisher + // keeps the plain barrier default. + barrierReason: "bulk-replace" | "reorder" = "bulk-replace", + ): void => { + queryPlan = committedRoot.queryPlan; + query = committedRoot.queryPlan.query; + derivations = committedRoot.queryPlan.derivations; + commit(committedRoot, READY); + // On a sort-only change this is structurally a no-op (distinct-value + // cache keys hash filter/column/population semantics, never sort); kept + // so both paths publish through one identical recipe. + distinctValues.publishTransitionRoot(committedRoot); + changeJournal.appendBarrier(previousRevision, revision, barrierReason); + }; const transitionError = ( error: unknown, operation: "set-query" | "set-derivations", @@ -797,12 +820,7 @@ export function createLocalRowModel< } catch { // Successful publication cannot be rolled back by advisory cleanup. } - queryPlan = committedRoot.queryPlan; - query = committedRoot.queryPlan.query; - derivations = committedRoot.queryPlan.derivations; - commit(committedRoot, READY); - distinctValues.publishTransitionRoot(committedRoot); - changeJournal.appendBarrier(previousRevision, revision); + publishCommittedRoot(committedRoot, previousRevision, revision); transition.candidate.release(); transition.resolve(revision); return true; @@ -973,6 +991,22 @@ export function createLocalRowModel< filterAuthority, sortAuthority, }); + /* + * The recompile is a plan swap, so the fresh plan's sort-key + * store must be filled for every surviving row — rows the draft + * re-evaluates get overwritten with recomputed values right + * after, and untouched rows carry their previous keys. Without + * this, records carried into the committed root would be + * unresolvable under the root's own plan. + */ + for (const [, record] of previousRoot.rows.entries()) { + fillSortKeysFromPrevious( + nextPlan, + queryPlan, + record as never, + instrumentation, + ); + } drafted = replaceFlatRowsDraft({ root: previousRoot, rows: nextRows, @@ -982,6 +1016,31 @@ export function createLocalRowModel< acceptSameReferenceMutation: true, instrumentation, }); + if (drafted.effective) { + /* + * The draft's visible index reuses trees whose comparators + * captured the RETIRED plan, and that plan's store still holds + * the pre-mutation keys for every in-place-mutated row object. + * Rebuild the index under the fresh plan so every entry is + * re-decorated with keys from the store that actually saw the + * mutation. The draft emits no per-row operations, so the + * rebuild is journal-invisible. + */ + const records: RowRecord[] = []; + for (const entry of drafted.sourceOrder.entries()) { + const record = drafted.rows.get(entry.rowId); + if (record !== undefined) records.push(record); + } + drafted = { + ...drafted, + visible: createVisibleIndex( + records, + nextPlan, + aggregateFilteredRows, + previousRoot.expansion.overrides, + ), + }; + } } if (!drafted.effective) { return { @@ -1122,6 +1181,65 @@ export function createLocalRowModel< notify: superseded, }; } + if ( + isSortOnlyChange(queryPlan, nextPlan) && + nextPlan.query.rowGroups.length === 0 + ) { + cancelActiveTransition("superseded"); + const previousRevision = root.revision; + const revision = previousRevision + 1; + let committedRoot: RevisionRoot; + try { + committedRoot = rebuildRootForSortOnlyChange({ + captured: root, + nextPlan, + revision, + now: transitionRuntime.now, + instrumentation, + }); + } catch (error) { + // Mirrors failTransition's observable semantics: an error status + // carrying this transition's id, a rejected `finished`, and the + // committed root left untouched. + const typed = + findPretableReentrantMutationError(error) ?? + transitionError(error, "set-query"); + state = Object.freeze({ + snapshot, + status: Object.freeze({ + kind: "error" as const, + transitionId: id, + error: typed, + }), + }); + const finished = Promise.reject(typed); + void finished.catch(() => undefined); + return { + transition: Object.freeze({ + id, + requestedQuery: nextPlan.query, + finished, + cancel: () => cancelTransitionHandle(id, "set-query"), + }), + notify: true, + }; + } + publishCommittedRoot( + committedRoot, + previousRevision, + revision, + "reorder", + ); + return { + transition: Object.freeze({ + id, + requestedQuery: nextPlan.query, + finished: Promise.resolve(revision), + cancel: () => cancelTransitionHandle(id, "set-query"), + }), + notify: true, + }; + } const active = startTransition(id, "set-query", nextPlan); return { transition: Object.freeze({ diff --git a/packages/row-model/src/diagnostics.ts b/packages/row-model/src/diagnostics.ts index bd79be293..c35f7cbf4 100644 --- a/packages/row-model/src/diagnostics.ts +++ b/packages/row-model/src/diagnostics.ts @@ -22,6 +22,14 @@ export interface LocalRowModelWorkDiagnostics { readonly groupNodesCopied: number; readonly aggregateMerges: number; readonly transitionRows: number; + /** Sort-only rebuilds taken synchronously, bypassing the cooperative path. */ + readonly synchronousRebuilds: number; + /** Total wall time inside synchronous sort-only rebuilds. */ + readonly synchronousRebuildMs: number; + /** Sort-key entries carried from a previous plan's store, per (row, column). */ + readonly sortKeyCarries: number; + /** Sort-key entries produced by running an accessor, per (row, column). */ + readonly sortKeyEvaluations: number; readonly snapshotOutputRowsRead: number; readonly schedulerSliceDurations: readonly number[]; } @@ -87,6 +95,10 @@ function newInstrumentation(): LocalRowModelInstrumentation { groupNodesCopied: 0, aggregateMerges: 0, transitionRows: 0, + synchronousRebuilds: 0, + synchronousRebuildMs: 0, + sortKeyCarries: 0, + sortKeyEvaluations: 0, snapshotOutputRowsRead: 0, schedulerSliceDurations: [], }, @@ -106,6 +118,10 @@ function resetWork(instrumentation: LocalRowModelInstrumentation): void { "groupNodesCopied", "aggregateMerges", "transitionRows", + "synchronousRebuilds", + "synchronousRebuildMs", + "sortKeyCarries", + "sortKeyEvaluations", "snapshotOutputRowsRead", ] as const) { instrumentation.work[counter] = 0; diff --git a/packages/row-model/src/group-index.ts b/packages/row-model/src/group-index.ts index 9dad5a4cf..f2346ae9c 100644 --- a/packages/row-model/src/group-index.ts +++ b/packages/row-model/src/group-index.ts @@ -1,4 +1,8 @@ -import type { CompiledGroupKey, CompiledQuery } from "./compiled-query"; +import { + compareWithSortKeys, + type CompiledGroupKey, + type CompiledQuery, +} from "./compiled-query"; import type { PretableRowId } from "./column-types"; import type { LocalRowModelInstrumentation } from "./diagnostics"; import { @@ -6,7 +10,12 @@ import { PretableRowModelError, type PretableRowModelOperation, } from "./errors"; -import type { RowRecord, VisibleIndexRoot } from "./internal-types"; +import type { + OrderedRowEntry, + RowRecord, + VisibleIndexRoot, +} from "./internal-types"; +import { orderedRowEntry } from "./ordered-row-entry"; import { createAggregateTree, createDeferredMeasureTransientAggregateTree, @@ -164,7 +173,7 @@ export interface GroupNode< readonly children: MeasuredGroupTree; readonly leaves: OrderStatisticTree< TRowId, - RowRecord, + OrderedRowEntry, number >; readonly filteredCount: number; @@ -847,12 +856,21 @@ function createLeafTree< >(queryPlan: CompiledQuery) { return createOrderStatisticTree< TRowId, - RowRecord, + OrderedRowEntry, number >({ - getId: (record) => record.rowId, + getId: (entry) => entry.record.rowId, + // Entries carry their resolved keys, so a comparison is property reads + // only — no store gets on this slice-hot path (the measured grouped-gate + // regression was per-comparison WeakMap resolution). compare: (left, right) => - queryPlan.compareRows(left.metadata as never, right.metadata as never), + compareWithSortKeys( + queryPlan, + left.record as never, + left.keys, + right.record as never, + right.keys, + ), measure: { empty: 0, fromEntry: () => 1, @@ -889,38 +907,36 @@ type RuntimeAggregateLeaf = { AggregateTreeLeaf | undefined; }; +type AggregateLeafDependency = { + readonly sourceOrder: number; + readonly sortKeys: readonly { columnId: string; value: unknown }[]; +}; + function compareAggregateLeaves( queryPlan: CompiledQuery, left: AggregateTreeLeaf, right: AggregateTreeLeaf, ): number { - const leftDependency = left.dependency as { - readonly sourceOrder: number; - readonly sortKeys: readonly unknown[]; - }; - const rightDependency = right.dependency as { - readonly sourceOrder: number; - readonly sortKeys: readonly unknown[]; - }; - return queryPlan.compareRows( + const leftDependency = left.dependency as AggregateLeafDependency; + const rightDependency = right.dependency as AggregateLeafDependency; + // The dependency carries the row's sort keys (resolved at evaluation), so + // a comparison is property reads only — no store gets on this slice-hot + // path (the measured grouped-gate regression was per-comparison WeakMap + // resolution). + return compareWithSortKeys( + queryPlan, { rowId: left.id, row: left.row, sourceOrder: leftDependency.sourceOrder, - filterPasses: true, - groupPath: [], - sortKeys: leftDependency.sortKeys, - aggregateLeaves: [], } as never, + leftDependency.sortKeys as never, { rowId: right.id, row: right.row, sourceOrder: rightDependency.sourceOrder, - filterPasses: true, - groupPath: [], - sortKeys: rightDependency.sortKeys, - aggregateLeaves: [], } as never, + rightDependency.sortKeys as never, ); } @@ -1321,7 +1337,7 @@ function mutatePath< if (leafLevel) { leaves = operation === "insert" && metadata.filterPasses - ? leaves.insertOrReplace(record) + ? leaves.insertOrReplace(orderedRowEntry(context.queryPlan, record)) : leaves.remove(record.rowId); } else { const childKey = pathKeys[depth + 1]!; @@ -1447,7 +1463,7 @@ interface MutableBuildNode< readonly childrenByKey: Map>; readonly leaves: TransientOrderStatisticTree< TRowId, - RowRecord, + OrderedRowEntry, number >; readonly aggregateRoots: MutableAggregateRoots; @@ -1764,7 +1780,9 @@ export function createGroupIndexBuildDraft< parentGroupId = current.groupId; children = current.childrenByKey; } - if (record.metadata.filterPasses) current!.leaves.insertOrReplace(record); + if (record.metadata.filterPasses) { + current!.leaves.insertOrReplace(orderedRowEntry(queryPlan, record)); + } rowParents.set(record.rowId, current!.groupId); }, sealStep() { @@ -2105,8 +2123,10 @@ function visibleAtNode< ? undefined : visibleAtNode(selected.entry, policy, selected.offset); } - const record = node.leaves.entryAt(descendantOffset); - return record === undefined ? undefined : publicData(record, node.depth + 1); + const entry = node.leaves.entryAt(descendantOffset); + return entry === undefined + ? undefined + : publicData(entry.record, node.depth + 1); } export function visibleAt< @@ -2170,8 +2190,8 @@ export function visibleRange< ); return; } - for (const record of node.leaves.range(descendantsStart, descendantsEnd)) { - result.push(publicData(resolveRecord(record), node.depth + 1)); + for (const entry of node.leaves.range(descendantsStart, descendantsEnd)) { + result.push(publicData(resolveRecord(entry.record), node.depth + 1)); } }; if (from < to) { @@ -2256,10 +2276,10 @@ function dataAtNode< ? undefined : dataAtNode(selected.entry, policy, selected.offset, resolveRecord); } - const record = node.leaves.entryAt(index); - return record === undefined + const entry = node.leaves.entryAt(index); + return entry === undefined ? undefined - : publicData(resolveRecord(record), node.depth + 1); + : publicData(resolveRecord(entry.record), node.depth + 1); } export function visibleDataCount< diff --git a/packages/row-model/src/internal-types.ts b/packages/row-model/src/internal-types.ts index 7e9b3510f..d8eec869e 100644 --- a/packages/row-model/src/internal-types.ts +++ b/packages/row-model/src/internal-types.ts @@ -1,4 +1,8 @@ -import type { CompiledQuery, CompiledRowMetadata } from "./compiled-query"; +import type { + CompiledQuery, + CompiledRowMetadata, + CompiledSortKey, +} from "./compiled-query"; import type { PretableRowId } from "./column-types"; import type { OrderStatisticTree } from "./persistent/order-statistic-tree"; import type { PersistentMap } from "./persistent/persistent-map"; @@ -34,6 +38,23 @@ export interface ExpansionRoot { readonly state: PretableExpansionState; } +/** + * A visible-tree entry: the record decorated with its sort keys, resolved + * exactly once at insert. Comparators become property reads — zero WeakMap + * gets on any O(n log n) or per-insert comparison path (the measured grouped + * gate regression). Keys are valid for the tree's lifetime: a tree is bound + * to one plan (the A2 rebuild-or-reseed invariant), and entry replacement on + * row update replaces the keys with the entry. + */ +export interface OrderedRowEntry< + TRow extends object, + TRowId extends PretableRowId, + TColumns, +> { + readonly record: RowRecord; + readonly keys: readonly CompiledSortKey[]; +} + export interface VisibleIndexRoot< TRow extends object, TRowId extends PretableRowId, @@ -41,7 +62,7 @@ export interface VisibleIndexRoot< > { readonly rows: OrderStatisticTree< TRowId, - RowRecord, + OrderedRowEntry, number >; } diff --git a/packages/row-model/src/ordered-row-entry.ts b/packages/row-model/src/ordered-row-entry.ts new file mode 100644 index 000000000..4986886e2 --- /dev/null +++ b/packages/row-model/src/ordered-row-entry.ts @@ -0,0 +1,27 @@ +import type { PretableRowId } from "./column-types"; +import { sortKeysOf, type CompiledQuery } from "./compiled-query"; +import type { OrderedRowEntry, RowRecord } from "./internal-types"; + +/** + * Standalone module because both visible-index and group-index need it and + * visible-index already imports group-index — either home would cycle. + * + * Decorates a record for tree residence: ONE store get, at insert time. The + * record must already be evaluated (or swap-filled) under `queryPlan` — + * `sortKeysOf` throws otherwise, which is the fail-loud contract. Entry keys + * stay valid for the tree's lifetime because a tree is bound to one plan + * (the A2 rebuild-or-reseed invariant) and row updates replace the entry. + */ +export function orderedRowEntry< + TRow extends object, + TRowId extends PretableRowId, + TColumns, +>( + queryPlan: CompiledQuery, + record: RowRecord, +): OrderedRowEntry { + return Object.freeze({ + record, + keys: sortKeysOf(queryPlan, record as never), + }); +} diff --git a/packages/row-model/src/persistent/order-statistic-tree.ts b/packages/row-model/src/persistent/order-statistic-tree.ts index 4c415bcb6..9408d3348 100644 --- a/packages/row-model/src/persistent/order-statistic-tree.ts +++ b/packages/row-model/src/persistent/order-statistic-tree.ts @@ -5,6 +5,7 @@ import type { LocalRowModelInstrumentation } from "../diagnostics"; const attachInstrumentation = Symbol("attachOrderInstrumentation"); const createDeferredMeasureDraft = Symbol("createDeferredMeasureDraft"); +const buildFromSortedEntries = Symbol("buildFromSortedEntries"); export type OrderStatisticTreeId = string | number; @@ -645,6 +646,64 @@ class PersistentOrderStatisticTree< ); } + /** + * The strict-order check is unconditional: a misordered build silently + * corrupts every later rank and lookup, which is strictly worse than the + * O(n) cost of checking. Duplicates compare 0 and are rejected by the same + * check. + */ + [buildFromSortedEntries]( + sorted: readonly TEntry[], + ): OrderStatisticTree { + const context = this.#context; + const entryIds = sorted.map((entry) => context.getId(entry)); + for (let index = 1; index < sorted.length; index += 1) { + const comparison = compareEntries( + sorted[index - 1]!, + entryIds[index - 1]!, + sorted[index]!, + entryIds[index]!, + context, + ); + if (comparison >= 0) { + throw new TypeError( + "Bulk build input must be strictly sorted by the tree's total order.", + ); + } + } + + const byId = createPersistentMap().asTransient(); + for (let index = 0; index < sorted.length; index += 1) { + byId.set(entryIds[index]!, sorted[index]!); + } + + const build = ( + low: number, + high: number, + ): TreeNode | null => { + if (low > high) return null; + const middle = (low + high) >> 1; + const node = createNode( + sorted[middle] as TEntry, + entryIds[middle]!, + context, + null, + ); + node.left = build(low, middle - 1); + node.right = build(middle + 1, high); + if (node.left !== null || node.right !== null) { + refreshNode(node, context); + } + return node; + }; + + return new PersistentOrderStatisticTree( + build(0, sorted.length - 1), + byId.freeze(), + context, + ); + } + [createDeferredMeasureDraft](): DeferredMeasureTransientOrderStatisticTree< TId, TEntry, @@ -931,6 +990,37 @@ export function createDeferredMeasureTransientOrderStatisticTree< return tree[createDeferredMeasureDraft](); } +/** Internal bulk-build primitive; deliberately omitted from the package index. */ +export function compareOrderStatisticTreeIds( + left: OrderStatisticTreeId, + right: OrderStatisticTreeId, +): number { + return compareIds(left, right); +} + +/** + * Internal bulk-build primitive; deliberately omitted from the package index. + * + * Builds a balanced tree in O(n) from `sorted`, which must be strictly + * increasing under `like`'s total order (comparator, ties broken by ID). + * Throws TypeError when adjacent entries compare `>= 0` — misordered input, + * equal-compare entries with misordered IDs, and duplicate IDs alike — or + * when `like` was not created by this module. + */ +export function createOrderStatisticTreeFromSortedEntries< + TId extends OrderStatisticTreeId, + TEntry, + TMeasure, +>( + like: OrderStatisticTree, + sorted: readonly TEntry[], +): OrderStatisticTree { + if (!(like instanceof PersistentOrderStatisticTree)) { + throw new TypeError("Bulk builds require a tree created by this module."); + } + return like[buildFromSortedEntries](sorted); +} + export function instrumentOrderStatisticTree< TId extends OrderStatisticTreeId, TEntry, diff --git a/packages/row-model/src/sort-rebuild.ts b/packages/row-model/src/sort-rebuild.ts new file mode 100644 index 000000000..df95315ed --- /dev/null +++ b/packages/row-model/src/sort-rebuild.ts @@ -0,0 +1,113 @@ +/** + * Synchronous whole-root rebuild for a sort-only plan change on an ungrouped + * query. Runs to completion on the caller's stack — the deliberate trade + * measured in #457: scheduler hops cost frames in the browser, and identity + * carry makes the total work small enough to spend inline. No record is + * rebuilt and no rows transient is opened: the committed root reuses the + * captured `rows` map BY IDENTITY, and only the next plan's sort-key store + * and the visible tree are produced fresh. + */ + +import type { PretableRowId } from "./column-types"; +import { + compareWithSortKeys, + fillSortKeysFromPrevious, + isSortOnlyChange, + type CompiledQuery, + type CompiledSortKey, +} from "./compiled-query"; +import type { LocalRowModelInstrumentation } from "./diagnostics"; +import type { OrderedRowEntry, RevisionRoot } from "./internal-types"; +import { + compareOrderStatisticTreeIds, + createOrderStatisticTreeFromSortedEntries, + instrumentOrderStatisticTree, +} from "./persistent/order-statistic-tree"; +import { createFlatVisibleTree } from "./visible-index"; + +export function rebuildRootForSortOnlyChange< + TRow extends object, + TRowId extends PretableRowId, + TColumns, +>(options: { + readonly captured: RevisionRoot; + readonly nextPlan: CompiledQuery; + readonly revision: number; + readonly now: () => number; + readonly instrumentation?: LocalRowModelInstrumentation; +}): RevisionRoot { + const { captured, nextPlan, revision, now, instrumentation } = options; + if (!isSortOnlyChange(captured.queryPlan, nextPlan)) { + throw new TypeError( + "Synchronous rebuild requires a sort-only plan change.", + ); + } + if (nextPlan.query.rowGroups.length > 0) { + throw new TypeError("Synchronous rebuild requires an ungrouped query."); + } + const startedAt = now(); + // Decorated sort: keys resolve ONCE per row here (the fill already returns + // them) and travel with the record, so the O(n log n) comparison loop does + // no WeakMap lookups — measured at 50k, per-comparison resolution costs + // ~4x the decorated form. The pairs ARE the tree's entry type, so the + // sorted array feeds the bulk constructor directly. + const visible: OrderedRowEntry[] = []; + for (const source of captured.sourceOrder.entries()) { + const previous = captured.rows.get(source.rowId); + if (previous === undefined) continue; + // Seed the NEXT plan's store for every carried record — the one part of + // a record's derived state that is plan-scoped. Everything else + // (metadata, publicRow, integrity) carries with the record itself. + const keys = fillSortKeysFromPrevious( + nextPlan, + captured.queryPlan, + previous as never, + instrumentation, + ) as readonly CompiledSortKey[]; + if (previous.metadata.filterPasses) { + visible.push(Object.freeze({ record: previous, keys })); + } + } + // The key comparator already totalizes distinct rows via its final + // sourceOrder comparison, so the id clause is unreachable today. It stays + // because the composite mirrors the tree's own order (comparator, then id) + // exactly, so this sort can never diverge from the bulk constructor's + // strict-order verification even if the comparator ever stopped being + // total. Every record's keys were seeded into nextPlan's store by the + // carry loop above, so later insert sites can decorate their entries from + // the store; the tree's own comparator reads only entry-carried keys. + visible.sort( + (left, right) => + compareWithSortKeys( + nextPlan, + left.record as never, + left.keys, + right.record as never, + right.keys, + ) || compareOrderStatisticTreeIds(left.record.rowId, right.record.rowId), + ); + const tree = createOrderStatisticTreeFromSortedEntries( + instrumentOrderStatisticTree( + createFlatVisibleTree(nextPlan), + instrumentation, + ), + visible, + ); + const root: RevisionRoot = Object.freeze({ + revision, + parentRevision: revision - 1, + // Identity — the entire point: records, publicRow, integrity, and the + // rows HAMT all survive a sort-only change untouched. + rows: captured.rows, + sourceOrder: captured.sourceOrder, + visible: Object.freeze({ rows: tree }), + queryPlan: nextPlan, + expansion: captured.expansion, + cause: Object.freeze({ kind: "set-query" as const }), + }); + if (instrumentation !== undefined) { + instrumentation.work.synchronousRebuilds += 1; + instrumentation.work.synchronousRebuildMs += Math.max(0, now() - startedAt); + } + return root; +} diff --git a/packages/row-model/src/transaction-draft.ts b/packages/row-model/src/transaction-draft.ts index 0f8ea27fd..4272d0a83 100644 --- a/packages/row-model/src/transaction-draft.ts +++ b/packages/row-model/src/transaction-draft.ts @@ -1,4 +1,4 @@ -import type { CompiledQuery } from "./compiled-query"; +import { sortKeysOf, type CompiledQuery } from "./compiled-query"; import { attachChangeOperationDiagnosticsForTesting, getChangeOperationDiagnosticsForTesting, @@ -32,6 +32,7 @@ import type { PretableVisibleRowRef, } from "./types"; import type { PretableGroupId } from "./types"; +import { orderedRowEntry } from "./ordered-row-entry"; import { createFlatVisibleTree } from "./visible-index"; interface TransactionDraftInput< @@ -339,13 +340,21 @@ function sameFlatOrder< TRowId extends PretableRowId, TColumns, >( + previousPlan: CompiledQuery, + nextPlan: CompiledQuery, previous: RowRecord, next: RowRecord, ): boolean { + // Each record's keys resolve from the plan that evaluated it: `previous` + // from the committed root's plan, `next` from the drafting plan. Outside + // the same-reference-mutation recompile these are one and the same object. return ( previous.sourceOrder === next.sourceOrder && previous.metadata.filterPasses === next.metadata.filterPasses && - sameKeyValues(previous.metadata.sortKeys, next.metadata.sortKeys) + sameKeyValues( + sortKeysOf(previousPlan, previous as never), + sortKeysOf(nextPlan, next as never), + ) ); } @@ -354,11 +363,13 @@ function sameGroupIndexContribution< TRowId extends PretableRowId, TColumns, >( + previousPlan: CompiledQuery, + nextPlan: CompiledQuery, previous: RowRecord, next: RowRecord, ): boolean { if ( - !sameFlatOrder(previous, next) || + !sameFlatOrder(previousPlan, nextPlan, previous, next) || !sameKeyValues(previous.metadata.groupPath, next.metadata.groupPath) ) { return false; @@ -371,10 +382,6 @@ function sameGroupIndexContribution< readonly value: unknown; readonly dependency: { readonly sourceOrder: number; - readonly sortKeys: readonly { - readonly columnId: string; - readonly value: unknown; - }[]; }; }; readonly filteredLeaf: object | undefined; @@ -403,15 +410,15 @@ function sameGroupIndexContribution< ) ); } + // The dependency's sortKeys are deliberately NOT compared here: + // sort-key changes no longer dirty aggregate leaves BY DESIGN + // (aggregation is order-independent; `sameFlatOrder` above already + // compared keys through the store). return ( (previousLeaf.aggregate === "count" || Object.is(previousLeaf.allLeaf.value, nextLeaf.allLeaf.value)) && previousLeaf.allLeaf.dependency.sourceOrder === - nextLeaf.allLeaf.dependency.sourceOrder && - sameKeyValues( - previousLeaf.allLeaf.dependency.sortKeys, - nextLeaf.allLeaf.dependency.sortKeys, - ) + nextLeaf.allLeaf.dependency.sourceOrder ); }) ); @@ -700,6 +707,8 @@ function rebaseSourceOrder< sourceOrder: number, ): RowRecord["metadata"] { const aggregateLeaves = metadata.aggregateLeaves.map((leaf) => { + // A rebase changes only the source order; the entry-carried sort keys + // ride along unchanged. const dependency = Object.freeze({ ...leaf.allLeaf.dependency, sourceOrder, @@ -972,7 +981,13 @@ export function applyFlatTransactionDraft< return ( (previous?.metadata.filterPasses === true || record.metadata.filterPasses) && - (previous === undefined || !sameFlatOrder(previous, record)) + (previous === undefined || + !sameFlatOrder( + input.root.queryPlan, + input.queryPlan, + previous, + record, + )) ); })); const visibleDraft = visibleNeedsChange @@ -1012,7 +1027,10 @@ export function applyFlatTransactionDraft< }), ); if (previousGroups === undefined) { - if (previous !== undefined && sameFlatOrder(previous, record)) { + if ( + previous !== undefined && + sameFlatOrder(input.root.queryPlan, input.queryPlan, previous, record) + ) { if (record.metadata.filterPasses) { const index = visibleDraft?.rankOf(record.rowId) ?? @@ -1034,7 +1052,11 @@ export function applyFlatTransactionDraft< ? visibleDraft?.rankOf(record.rowId) : undefined; if (previous?.metadata.filterPasses) visibleDraft?.remove(record.rowId); - if (record.metadata.filterPasses) visibleDraft?.insertOrReplace(record); + if (record.metadata.filterPasses) { + visibleDraft?.insertOrReplace( + orderedRowEntry(input.queryPlan, record), + ); + } const index = record.metadata.filterPasses ? visibleDraft?.rankOf(record.rowId) : undefined; @@ -1072,7 +1094,11 @@ export function applyFlatTransactionDraft< } continue; } - if (previous !== undefined && sameFlatOrder(previous, record)) continue; + if ( + previous !== undefined && + sameFlatOrder(input.root.queryPlan, input.queryPlan, previous, record) + ) + continue; } const frozenRows = rowDraft.freeze(); const frozenFlatRows = visibleDraft?.freeze(); @@ -1081,7 +1107,12 @@ export function applyFlatTransactionDraft< ...prepared.flatMap((record) => { const previous = input.root.rows.get(record.rowId); return previous === undefined || - sameGroupIndexContribution(previous, record) + sameGroupIndexContribution( + input.root.queryPlan, + input.queryPlan, + previous, + record, + ) ? [] : [previous]; }), @@ -1089,7 +1120,13 @@ export function applyFlatTransactionDraft< const groupedInsertions = prepared.filter((record) => { const previous = input.root.rows.get(record.rowId); return ( - previous === undefined || !sameGroupIndexContribution(previous, record) + previous === undefined || + !sameGroupIndexContribution( + input.root.queryPlan, + input.queryPlan, + previous, + record, + ) ); }); const grouped = @@ -1352,7 +1389,10 @@ export function replaceFlatRowsDraft< ).asTransient(); const orderChangedRecords = changedRecords.filter((record) => { const previous = input.root.rows.get(record.rowId); - return previous === undefined || !sameFlatOrder(previous, record); + return ( + previous === undefined || + !sameFlatOrder(input.root.queryPlan, input.queryPlan, previous, record) + ); }); const affectedVisibleIds = new Set( orderChangedRecords @@ -1369,8 +1409,8 @@ export function replaceFlatRowsDraft< if (record.metadata.filterPasses) affectedVisibleIds.add(record.rowId); } let hasUnaffectedVisible = false; - for (const record of input.root.visible.rows.entries()) { - if (!affectedVisibleIds.has(record.rowId)) { + for (const entry of input.root.visible.rows.entries()) { + if (!affectedVisibleIds.has(entry.record.rowId)) { hasUnaffectedVisible = true; break; } @@ -1381,12 +1421,7 @@ export function replaceFlatRowsDraft< : instrumentOrderStatisticTree( hasUnaffectedVisible ? input.root.visible.rows - : createFlatVisibleTree( - input.queryPlan.compareRows as unknown as ( - left: RowRecord["metadata"], - right: RowRecord["metadata"], - ) => number, - ), + : createFlatVisibleTree(input.queryPlan), input.instrumentation, ).asTransient(); for (const record of removedRecords) { @@ -1411,7 +1446,9 @@ export function replaceFlatRowsDraft< } } for (const record of orderChangedRecords) { - if (record.metadata.filterPasses) visibleDraft?.insertOrReplace(record); + if (record.metadata.filterPasses) { + visibleDraft?.insertOrReplace(orderedRowEntry(input.queryPlan, record)); + } } const frozenRows = rowDraft.freeze(); const frozenSource = sourceDraft.freeze(); diff --git a/packages/row-model/src/types.ts b/packages/row-model/src/types.ts index 579307077..f48dfd94b 100644 --- a/packages/row-model/src/types.ts +++ b/packages/row-model/src/types.ts @@ -291,7 +291,14 @@ export type PretableChangeSequence = | { readonly kind: "reset"; readonly toRevision: number; - readonly reason: "unknown-revision" | "journal-evicted" | "bulk-replace"; + /** + * `"reorder"` asserts the visible row set and every row's content are + * unchanged — only the order moved (a sort-only commit). Every other + * reason makes no such promise; consumers that do not understand + * `"reorder"` may treat it exactly like `"bulk-replace"`. + */ + readonly reason: + "unknown-revision" | "journal-evicted" | "bulk-replace" | "reorder"; }; /** @public */ diff --git a/packages/row-model/src/visible-index.ts b/packages/row-model/src/visible-index.ts index e614cc42f..6e987eee4 100644 --- a/packages/row-model/src/visible-index.ts +++ b/packages/row-model/src/visible-index.ts @@ -1,6 +1,7 @@ import type { PretableRowId } from "./column-types"; -import type { CompiledQuery } from "./compiled-query"; +import { compareWithSortKeys, type CompiledQuery } from "./compiled-query"; import type { PretableRowModelOperation } from "./errors"; +import { orderedRowEntry } from "./ordered-row-entry"; import { attachGroupIndex, createGroupIndex, @@ -17,6 +18,7 @@ import { type GroupIndexRoot, } from "./group-index"; import type { + OrderedRowEntry, RevisionRoot, RowRecord, VisibleIndexRoot, @@ -35,16 +37,15 @@ export function createFlatVisibleIndex< TColumns, >( records: readonly RowRecord[], - compareRows: ( - left: RowRecord["metadata"], - right: RowRecord["metadata"], - ) => number, + queryPlan: CompiledQuery, ): VisibleIndexRoot { const draft = createFlatVisibleTree( - compareRows, + queryPlan, ).asTransient(); for (const record of records) { - if (record.metadata.filterPasses) draft.insertOrReplace(record); + if (record.metadata.filterPasses) { + draft.insertOrReplace(orderedRowEntry(queryPlan, record)); + } } return Object.freeze({ rows: draft.freeze() }); } @@ -61,15 +62,11 @@ export function createVisibleIndex< operation: PretableRowModelOperation = "set-rows", reusable?: GroupIndexRoot, ): VisibleIndexRoot { - const compareRows = queryPlan.compareRows as unknown as ( - left: RowRecord["metadata"], - right: RowRecord["metadata"], - ) => number; if (queryPlan.query.rowGroups.length === 0) { - return createFlatVisibleIndex(records, compareRows); + return createFlatVisibleIndex(records, queryPlan); } return attachGroupIndex( - createFlatVisibleTree(compareRows), + createFlatVisibleTree(queryPlan), createGroupIndex( records, queryPlan, @@ -85,19 +82,24 @@ export function createFlatVisibleTree< TRow extends object, TRowId extends PretableRowId, TColumns, ->( - compareRows: ( - left: RowRecord["metadata"], - right: RowRecord["metadata"], - ) => number, -) { +>(queryPlan: CompiledQuery) { return createOrderStatisticTree< TRowId, - RowRecord, + OrderedRowEntry, number >({ - getId: (record) => record.rowId, - compare: (left, right) => compareRows(left.metadata, right.metadata), + getId: (entry) => entry.record.rowId, + // Entries carry their resolved keys, so a comparison is property reads + // only — no store gets on this slice-hot path (the measured grouped-gate + // regression was per-comparison WeakMap resolution). + compare: (left, right) => + compareWithSortKeys( + queryPlan, + left.record as never, + left.keys, + right.record as never, + right.keys, + ), measure: { empty: 0, fromEntry: () => 1, @@ -174,7 +176,7 @@ export function createFlatSnapshot< const ordered = visible.entryAt(index); return ordered === undefined ? undefined - : root.rows.get(ordered.rowId)?.publicRow; + : root.rows.get(ordered.record.rowId)?.publicRow; }; const lookupRank = ( ref: PretableVisibleRowRef, @@ -190,7 +192,7 @@ export function createFlatSnapshot< Object.freeze( visible .range(start, end) - .map((record) => root.rows.get(record.rowId)?.publicRow) + .map((entry) => root.rows.get(entry.record.rowId)?.publicRow) .filter((row): row is NonNullable => row !== undefined), ), indexOf: (ref: PretableVisibleRowRef) => lookupRank(ref) ?? -1, diff --git a/scripts/bench-row-model-gate.mjs b/scripts/bench-row-model-gate.mjs index ed2232902..b6ca5f305 100644 --- a/scripts/bench-row-model-gate.mjs +++ b/scripts/bench-row-model-gate.mjs @@ -156,6 +156,9 @@ export function validateRowModelGateSummaries( assertEqual(summary, "long_tasks_count", 0); assertEqual(summary, "scroll_position_drift_px", 0); assertEqual(summary, "visible_row_count_drift", 0); + // The flat sort fast path (#457) is synchronous BY DESIGN and reports under + // work.synchronousRebuildMs, never as a scheduler slice — it is exempt from + // this bound. Grouped and non-sort-only transitions remain governed by it. assertAtMost(summary, "rebuild_slice_max_ms", 8); const rebuild = summary.rowModel.rebuild; if (rebuild?.completed !== true)