FE-1130: Token frame format v2 — packed struct token layout

.changeset/token-frame-format-packed-structs.md

@@ -0,0 +1,6 @@
+---
+"@hashintel/petrinaut-core": patch
+"@hashintel/petrinaut": patch
+---
+
+Packed-struct token frame layout (f64 numbers, u8 booleans, 8-byte strides). `getPlaceTokens(place)` and `buildMetricState(frame, places)` drop unused parameters.

libs/@hashintel/petrinaut-core/src/actual-mode/timeline.ts

@@ -9,9 +9,8 @@ import { parseActualModeTimestampMs } from "./time";
 import type {
   SimulationFrameReader,
   SimulationFrameState,
-  SimulationPlaceTokenValues,
 } from "../simulation/api";
-import type { Color, Place, SDCPN } from "../types/sdcpn";
+import type { Place, SDCPN } from "../types/sdcpn";
 import type {
   ActualModeContextValue,
   ActualModeMarking,
@@ -193,47 +192,10 @@ export const createActualModeTimelineFrameReader = (params: {
     time: point.timeMs / 1_000,
     getPlaceTokenCount: (placeId: string) =>
       getActualModePlaceMarkingTokenCount(marking[placeId]),
-    getPlaceTokenValues: (
-      placeId: string,
-    ): SimulationPlaceTokenValues | null => {
-      const place = definition.places.find(
-        (candidatePlace) => candidatePlace.id === placeId,
-      );
-      const color = place
-        ? definition.types.find((type) => type.id === place.colorId)
-        : null;
-      const placeMarking = marking[placeId];
-
-      if (!place || !color || !isActualModeTokenColourArray(placeMarking)) {
-        return {
-          count: getActualModePlaceMarkingTokenCount(placeMarking),
-          values: new Float64Array(),
-        };
-      }
-
-      const values = new Float64Array(
-        placeMarking.length * color.elements.length,
-      );
-
-      for (const [tokenIndex, token] of placeMarking.entries()) {
-        for (const [elementIndex, element] of color.elements.entries()) {
-          values[tokenIndex * color.elements.length + elementIndex] =
-            token[element.name] ?? token[element.elementId] ?? 0;
-        }
-      }
-
-      return {
-        count: placeMarking.length,
-        values,
-      };
-    },
-    getPlaceTokens: (
-      place: Place,
-      color: Color | null | undefined,
-    ): Record<string, number>[] => {
+    getPlaceTokens: (place: Place): Record<string, number>[] => {
       const placeMarking = marking[place.id];
 
-      if (!color || !isActualModeTokenColourArray(placeMarking)) {
+      if (!place.colorId || !isActualModeTokenColourArray(placeMarking)) {
         return [];
       }
 

libs/@hashintel/petrinaut-core/src/index.ts

@@ -169,7 +169,6 @@ export type {
   SimulationFrameReader,
   SimulationFrameState,
   SimulationFrameSummary,
-  SimulationPlaceTokenValues,
   SimulationState,
   SimulationTransport,
   WorkerFactory,
@@ -334,11 +333,18 @@ export { buildMetricState } from "./simulation/frames/metric-state";
 export {
   coerceTokenAttributeValue,
   coerceTokenRecord,
-  decodeTokenAttributeValue,
-  decodeTokenRecord,
   defaultTokenAttributeValue,
   encodeTokenAttributeValue,
 } from "./simulation/engine/token-values";
+export {
+  computeTokenSlotLayout,
+  createTokenRegionViews,
+  encodeTokenToBytes,
+  readTokenRecord,
+  type PhysicalKind,
+  type TokenLayoutField,
+  type TokenSlotLayout,
+} from "./simulation/engine/token-layout";
 export { compileUserCode } from "./simulation/authoring/user-code/compile-user-code";
 export {
   displayNameSchema,

libs/@hashintel/petrinaut-core/src/simulation/ARCHITECTURE.md

@@ -28,9 +28,15 @@ SDCPN snapshot
 ```
 
 `EngineFrame` is not a public API or stable storage format. It is currently a
-binary `ArrayBuffer` with typed sections for place token counts, place value
-offsets, transition timers/counts/flags, and packed token values. It can only be
-read with the matching SDCPN-derived layout.
+binary `ArrayBuffer` (header-stamped `FRAME_VERSION = 2`) with typed sections
+for place token counts, per-place byte offsets, transition timers/counts/flags,
+and a packed-struct token byte region. Each colour's token layout is computed
+by `engine/token-layout.ts`: `real` and `integer` elements map to f64 (8 B,
+8-aligned; integers rounded by the value codec), `boolean` elements map to u8
+(1 B), fields are ordered by decreasing alignment, and the per-token stride is
+rounded up to 8 bytes so f64 fields stay addressable through a shared
+`Float64Array` view. A frame can only be read with the matching SDCPN-derived
+layout.
 
 The public frame path is:
 

libs/@hashintel/petrinaut-core/src/simulation/api.ts

@@ -2,7 +2,7 @@ import type { AbortSignalLike, WorkerFactoryLike } from "../environment";
 import type { PetrinautExtensionSettings } from "../extensions";
 import type { EventStream } from "../instance";
 import type { ReadableStore } from "../store";
-import type { Color, Place, SDCPN, TokenRecord } from "../types/sdcpn";
+import type { Place, SDCPN, TokenRecord } from "../types/sdcpn";
 
 export type SimulationState =
   | "Initializing"
@@ -92,20 +92,15 @@ export type SimulationFrameState = {
   };
 };
 
-export type SimulationPlaceTokenValues = {
-  values: Float64Array;
-  count: number;
-};
-
 export interface SimulationFrameReader {
   /** Frame index in the simulation history. */
   readonly number: number;
   /** Simulation time for this frame, in seconds. */
   readonly time: number;
 
   getPlaceTokenCount(placeId: string): number;
-  getPlaceTokenValues(placeId: string): SimulationPlaceTokenValues | null;
-  getPlaceTokens(place: Place, color: Color | null | undefined): TokenRecord[];
+  /** Typed token records for a coloured place; `[]` for uncoloured places. */
+  getPlaceTokens(place: Place): TokenRecord[];
   getTransitionState(transitionId: string): {
     /**
      * Time elapsed since this transition last fired, in milliseconds.

libs/@hashintel/petrinaut-core/src/simulation/engine/README.md

@@ -56,24 +56,51 @@ The frame does not contain place or transition IDs. `SimulationInstance` owns an
 `EngineFrameLayout` derived from the SDCPN and passes it to internal readers and
 writers.
 
-## Token Value Storage
-
-Token values are stored in a packed `Float64Array` section inside the frame
-buffer. Place counts and value offsets are stored in separate `Uint32Array`
+## Token Value Storage (packed token structs)
+
+Token values are stored as a byte region of packed structs inside the frame
+buffer (`FRAME_VERSION = 2`). Each colour has a `TokenSlotLayout` computed by
+`engine/token-layout.ts` — the single source of truth for the physical
+mapping:
+
+| Element type | Physical type                       | Size | Alignment |
+| ------------ | ----------------------------------- | ---- | --------- |
+| `real`       | `f64`                               | 8 B  | 8 B       |
+| `integer`    | `f64` (rounded via the value codec) | 8 B  | 8 B       |
+| `boolean`    | `u8` (0/1)                          | 1 B  | 1 B       |
+
+Per colour, fields are ordered by decreasing alignment (stable within equal
+alignment), each field's byte offset is aligned to its physical alignment, and
+the stride (`sizeof(token)`) is rounded up to 8 bytes. Because the token
+region starts at an 8-aligned offset and every stride is a multiple of 8, all
+f64 fields are addressable through a shared `Float64Array` view
+(`index = (placeByteOffset + token * strideBytes + fieldByteOffset) / 8`) and
+u8 fields through a `Uint8Array` view — no `DataView` in hot paths.
+
+Place counts and per-place byte offsets are stored in separate `Uint32Array`
 sections, so a place can be accessed in O(1) when the SDCPN layout is known.
 
 ```text
-Place p1: offset=0, count=2, dimensions=3
-Place p2: offset=6, count=1, dimensions=2
+Colour of p1: { x: real, n: integer, on: boolean } → stride 24 B
+  x @ 0 (f64) · n @ 8 (f64) · on @ 16 (u8) · padding 17..24
+
+Place p1: byteOffset=0,  count=2, strideBytes=24
+Place p2: byteOffset=48, count=1, strideBytes=16
 
-buffer: [v1.x, v1.y, v1.z, v2.x, v2.y, v2.z, v3.a, v3.b]
-         └──────── p1 tokens ────────────┘ └── p2 ──┘
+bytes: [tok0: x n on pad][tok1: x n on pad][tok2: a b]
+        └──────── p1 tokens (48 B) ──────┘ └ p2 (16 B) ┘
 ```
 
 Access:
 
 ```typescript
 const frameView = readEngineFrame(simulation.frameLayout, frame);
-const place = frameView.getPlaceState("p1");
-const values = frameView.getPlaceTokenValues("p1");
+const place = frameView.getPlaceState("p1"); // { byteOffset, count, strideBytes }
+const layout = simulation.frameLayout.placeTokenLayouts[placeIndex];
+const token = readTokenRecord(
+  layout,
+  frameView.tokenF64,
+  frameView.tokenBytes,
+  place.byteOffset + tokenIndex * place.strideBytes,
+);
 ```

libs/@hashintel/petrinaut-core/src/simulation/engine/build-simulation.test.ts

@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";
 import { compileSimulationFrameReader } from "../frames/frame-reader";
 import { materializeEngineFrame } from "../frames/internal-frame";
 import { buildSimulation } from "./build-simulation";
+import { decodePlaceTokens } from "./token-layout.test-helpers";
 
 import type { SimulationInput } from "./types";
 
@@ -60,13 +61,18 @@ describe("buildSimulation", () => {
       engineFrame,
     );
 
-    expect(frame.buffer).toEqual(new Float64Array([1.25, 3, 1]));
+    // Packed struct layout: amount (f64) @ 0, count (f64) @ 8, active (u8)
+    // @ 16, stride rounded up to 24 bytes.
+    expect(frame.buffer).toBeInstanceOf(Uint8Array);
+    expect(frame.buffer.byteLength).toBe(24);
+    expect(
+      new Float64Array(frame.buffer.buffer, frame.buffer.byteOffset, 2),
+    ).toEqual(new Float64Array([1.25, 3]));
+    expect(frame.buffer[16]).toBe(1);
 
     const reader = compileSimulationFrameReader(input.sdcpn)(engineFrame, 0, 0);
 
-    expect(
-      reader.getPlaceTokens(input.sdcpn.places[0]!, input.sdcpn.types[0]),
-    ).toEqual([
+    expect(reader.getPlaceTokens(input.sdcpn.places[0]!)).toEqual([
       {
         amount: 1.25,
         count: 3,
@@ -508,8 +514,8 @@ describe("buildSimulation", () => {
         placeId: "processor-1::port-in",
         placeName: "Processor 1::Port In",
         weight: 1,
-        elementNames: null,
         elements: null,
+        tokenLayout: null,
       },
     ]);
     expect(simulation.compiledTransitions.get("receive")?.inputPlaces).toEqual([
@@ -518,8 +524,8 @@ describe("buildSimulation", () => {
         placeName: "Processor 1::Port Out",
         weight: 1,
         arcType: "standard",
-        elementNames: null,
         elements: null,
+        tokenLayout: null,
       },
     ]);
     expect(frame.places["root-input"]?.count).toBe(2);
@@ -599,14 +605,19 @@ describe("buildSimulation", () => {
     const p1State = frame.places.p1;
     expect(p1State).toBeDefined();
     expect(p1State?.count).toBe(2);
-    expect(p1State?.offset).toBe(0);
+    expect(p1State?.byteOffset).toBe(0);
     const p1Type = simulationInstance.places.get("p1")?.colorId;
     expect(p1Type).toBe("type1");
     const typeDefinition = input.sdcpn.types.find((tp) => tp.id === p1Type);
     expect(typeDefinition?.elements.length).toBe(2);
 
     // Verify buffer contains the correct token values
-    expect(frame.buffer).toEqual(new Float64Array([1.0, 2.0, 3.0, 4.0]));
+    expect(
+      decodePlaceTokens(simulationInstance.frameLayout, engineFrame, "p1"),
+    ).toEqual([
+      { x: 1.0, y: 2.0 },
+      { x: 3.0, y: 4.0 },
+    ]);
 
     // Verify compiled functions exist
     expect(simulationInstance.differentialEquationFns.has("p1")).toBe(true);
@@ -717,9 +728,10 @@ describe("buildSimulation", () => {
     };
 
     const simulationInstance = buildSimulation(input);
+    const engineFrame = simulationInstance.frames[0]!;
     const frame = materializeEngineFrame(
       simulationInstance.frameLayout,
-      simulationInstance.frames[0]!,
+      engineFrame,
     );
 
     // Verify simulation instance properties
@@ -732,7 +744,7 @@ describe("buildSimulation", () => {
     // Verify p1 state (places are sorted by ID, so p1 comes first)
     const p1State = frame.places.p1;
     expect(p1State?.count).toBe(3);
-    expect(p1State?.offset).toBe(0);
+    expect(p1State?.byteOffset).toBe(0);
     const p1Type = simulationInstance.places.get("p1")?.colorId;
     expect(p1Type).toBe("type1");
     const p1TypeDef = input.sdcpn.types.find((tp) => tp.id === p1Type);
@@ -741,7 +753,7 @@ describe("buildSimulation", () => {
     // Verify p2 state (comes after p1)
     const p2State = frame.places.p2;
     expect(p2State?.count).toBe(1);
-    expect(p2State?.offset).toBe(3); // After p1's 3 tokens
+    expect(p2State?.byteOffset).toBe(24); // After p1's 3 tokens × 8-byte stride
     const p2Type = simulationInstance.places.get("p2")?.colorId;
     expect(p2Type).toBe("type2");
     const p2TypeDef = input.sdcpn.types.find((tp) => tp.id === p2Type);
@@ -750,16 +762,22 @@ describe("buildSimulation", () => {
     // Verify p3 state (comes after p2, has no tokens)
     const p3State = frame.places.p3;
     expect(p3State?.count).toBe(0);
-    expect(p3State?.offset).toBe(5); // After p1's 3 values + p2's 2 values
+    expect(p3State?.byteOffset).toBe(40); // After p1's 24 bytes + p2's 16 bytes
     const p3Type = simulationInstance.places.get("p3")?.colorId;
     expect(p3Type).toBe("type1");
     const p3TypeDef = input.sdcpn.types.find((tp) => tp.id === p3Type);
     expect(p3TypeDef?.elements.length).toBe(1);
 
-    // Verify buffer layout: [p1: 10, 20, 30 | p2: 1, 2]
-    expect(frame.buffer).toEqual(
-      new Float64Array([10.0, 20.0, 30.0, 1.0, 2.0]),
-    );
+    // Verify buffer layout: [p1: {x:10}, {x:20}, {x:30} | p2: {x:1, y:2}]
+    expect(
+      decodePlaceTokens(simulationInstance.frameLayout, engineFrame, "p1"),
+    ).toEqual([{ x: 10 }, { x: 20 }, { x: 30 }]);
+    expect(
+      decodePlaceTokens(simulationInstance.frameLayout, engineFrame, "p2"),
+    ).toEqual([{ x: 1, y: 2 }]);
+    expect(
+      decodePlaceTokens(simulationInstance.frameLayout, engineFrame, "p3"),
+    ).toEqual([]);
 
     // Verify transitions exist with initial state
     expect(Object.keys(frame.transitions).length).toBe(2);