perf(codegen): packed-clone endgame — receiver caching, poll striding, integer count accumulators, check-free genuine stores

changelog.d/9111-packed-clone-endgame.md

@@ -0,0 +1 @@
+Packed fast clones: poll-scoped receiver/handle caching with a strided armed check, scope-local i32 deferral for count accumulators, and check-free stores of provably-genuine values — every row of the isolated array-operation matrix (count/reduce/store loops across local, module-global, and masked receivers) now runs ahead of Node on the quiet bench host.

crates/perry-codegen/src/codegen/closure.rs

@@ -1193,6 +1193,11 @@ pub(super) fn compile_closure(
         element_shape_loop_facts: Vec::new(),
         i32_counter_slots: HashMap::new(),
         numeric_accumulator_f64_slots: HashMap::new(),
+        packed_receiver_box_slots: HashMap::new(),
+        packed_receiver_refresh: Vec::new(),
+        packed_receiver_handle_slots: HashMap::new(),
+        poll_stride_counter_slot: None,
+        deferred_integer_update_accumulators: HashSet::new(),
         local_slot_reps: HashMap::new(),
         repsel_context_allows_canonical_i32: repsel_allows,
         // #7109 split the FIELD out of `repsel_context_allows_canonical_i32`;

crates/perry-codegen/src/codegen/entry.rs

@@ -881,6 +881,11 @@ pub(super) fn compile_module_entry(
             element_shape_loop_facts: Vec::new(),
             i32_counter_slots: HashMap::new(),
             numeric_accumulator_f64_slots: HashMap::new(),
+            packed_receiver_box_slots: HashMap::new(),
+            packed_receiver_refresh: Vec::new(),
+            packed_receiver_handle_slots: HashMap::new(),
+            poll_stride_counter_slot: None,
+            deferred_integer_update_accumulators: HashSet::new(),
             local_slot_reps: HashMap::new(),
             // #7109: this entry body selects canonical i32/u32/Str on the same
             // per-value rules as a function body. Phase 1 (#6903) excluded it
@@ -1599,6 +1604,11 @@ pub(super) fn compile_module_entry(
             element_shape_loop_facts: Vec::new(),
             i32_counter_slots: HashMap::new(),
             numeric_accumulator_f64_slots: HashMap::new(),
+            packed_receiver_box_slots: HashMap::new(),
+            packed_receiver_refresh: Vec::new(),
+            packed_receiver_handle_slots: HashMap::new(),
+            poll_stride_counter_slot: None,
+            deferred_integer_update_accumulators: HashSet::new(),
             local_slot_reps: HashMap::new(),
             // #7109: this entry body selects canonical i32/u32/Str on the same
             // per-value rules as a function body. Phase 1 (#6903) excluded it

crates/perry-codegen/src/codegen/function.rs

@@ -1149,6 +1149,11 @@ pub(super) fn compile_function(
             .collect(),
         i32_counter_slots: spec_i32_param_slots,
         numeric_accumulator_f64_slots: HashMap::new(),
+        packed_receiver_box_slots: HashMap::new(),
+        packed_receiver_refresh: Vec::new(),
+        packed_receiver_handle_slots: HashMap::new(),
+        poll_stride_counter_slot: None,
+        deferred_integer_update_accumulators: HashSet::new(),
         repsel_context_allows_canonical_i32: repsel_allows,
         // #7109 split the FIELD out of `repsel_context_allows_canonical_i32`;
         // #7128 split the VALUE, which is what the knob actually reads. Until

crates/perry-codegen/src/codegen/method.rs

@@ -551,6 +551,11 @@ pub(super) fn compile_method(
         element_shape_loop_facts: Vec::new(),
         i32_counter_slots: index_i32_param_slots,
         numeric_accumulator_f64_slots: HashMap::new(),
+        packed_receiver_box_slots: HashMap::new(),
+        packed_receiver_refresh: Vec::new(),
+        packed_receiver_handle_slots: HashMap::new(),
+        poll_stride_counter_slot: None,
+        deferred_integer_update_accumulators: HashSet::new(),
         local_slot_reps: HashMap::new(),
         repsel_context_allows_canonical_i32: repsel_allows,
         // #7109 split the FIELD out of `repsel_context_allows_canonical_i32`;
@@ -1718,6 +1723,11 @@ pub(super) fn compile_static_method(
         element_shape_loop_facts: Vec::new(),
         i32_counter_slots: HashMap::new(),
         numeric_accumulator_f64_slots: HashMap::new(),
+        packed_receiver_box_slots: HashMap::new(),
+        packed_receiver_refresh: Vec::new(),
+        packed_receiver_handle_slots: HashMap::new(),
+        poll_stride_counter_slot: None,
+        deferred_integer_update_accumulators: HashSet::new(),
         local_slot_reps: HashMap::new(),
         repsel_context_allows_canonical_i32: repsel_allows,
         // #7109 split the FIELD out of `repsel_context_allows_canonical_i32`;

crates/perry-codegen/src/expr/index_set_packed_loop.rs

@@ -101,7 +101,67 @@ pub(super) fn lower_packed_f64_range_loop_index_set(
     guard_id: &str,
     side_exit_label: &str,
 ) -> Result<String> {
+    // A RHS that provably materializes a genuine double by construction —
+    // a literal, the canonical-i32 counter, an in-window guarded load, or
+    // float arithmetic over those (the masked store's admission predicate)
+    // — needs no runtime value check at all: the nanbox tag test below
+    // (fmov + tag math + compare + branch, five per-element instructions)
+    // exists only for values that could be boxed at runtime.
+    let statically_genuine =
+        crate::expr::masked_window::masked_store_rhs_is_genuine_f64(ctx, value);
     let (val_double, rhs_notes) = lower_packed_f64_loop_store_value(ctx, arr_id, value)?;
+    if statically_genuine {
+        let arr_expr = Expr::LocalGet(arr_id);
+        let arr_box = lower_expr(ctx, &arr_expr)?;
+        let arr_handle = super::packed_receiver_handle_i64(ctx, Some(arr_id), &arr_box);
+        let blk = ctx.block();
+        let idx_i64 = blk.zext(I32, idx_i32, I64);
+        let byte_offset = blk.shl(I64, &idx_i64, "3");
+        let with_header = blk.add(I64, &byte_offset, "8");
+        let element_addr = blk.add(I64, &arr_handle, &with_header);
+        let element_ptr = blk.inttoptr(I64, &element_addr);
+        // GC_STORE_AUDIT(POINTER_FREE): statically-genuine packed store —
+        // the RHS predicate proves an unboxed double, never a heap pointer.
+        blk.store(DOUBLE, &val_double, &element_ptr);
+        let stored = LoweredValue {
+            semantic: SemanticKind::JsNumber,
+            rep: NativeRep::F64,
+            llvm_ty: DOUBLE,
+            value: val_double.clone(),
+        };
+        ctx.record_lowered_value_with_access_mode_and_facts(
+            "PackedF64RangeLoopStore",
+            Some(arr_id),
+            "packed_f64_range_loop_store",
+            &stored,
+            Some(BoundsState::Guarded {
+                guard_id: guard_id.to_string(),
+            }),
+            None,
+            Some(BufferAccessMode::CheckedNative),
+            None,
+            None,
+            None,
+            vec![
+                array_kind_fact(Some(arr_id), "consumed", "packed_f64", None),
+                raw_f64_layout_fact(Some(arr_id), "consumed", guard_id, None),
+            ],
+            Vec::new(),
+            false,
+            false,
+            {
+                let mut notes = vec![
+                    "rhs_numeric_guard=static_genuine_f64_proof".to_string(),
+                    "index_range=range_guarded_i32_window".to_string(),
+                    "storage_layout=raw_f64_or_hole_slots".to_string(),
+                ];
+                notes.extend(rhs_notes);
+                notes
+            },
+        );
+        let _ = side_exit_label;
+        return Ok(val_double);
+    }
 
     let fast_idx = ctx.new_block("packed_f64_range_store.fast");
     let exit_idx = ctx.new_block("packed_f64_range_store.side_exit");
@@ -164,9 +224,8 @@ pub(super) fn lower_packed_f64_range_loop_index_set(
     {
         let arr_expr = Expr::LocalGet(arr_id);
         let arr_box = lower_expr(ctx, &arr_expr)?;
+        let arr_handle = super::packed_receiver_handle_i64(ctx, Some(arr_id), &arr_box);
         let blk = ctx.block();
-        let arr_bits = blk.bitcast_double_to_i64(&arr_box);
-        let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64);
         let idx_i64 = blk.zext(I32, idx_i32, I64);
         let byte_offset = blk.shl(I64, &idx_i64, "3");
         let with_header = blk.add(I64, &byte_offset, "8");

crates/perry-codegen/src/expr/literals_vars.rs

@@ -543,6 +543,12 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<String> {
                 if let Some(f64_slot) = ctx.numeric_accumulator_f64_slots.get(id).cloned() {
                     return Ok(ctx.block().load(DOUBLE, &f64_slot));
                 }
+                // Poll-scoped receiver cache (packed fast clones): the
+                // receiver box lives in a promotable alloca, refreshed on
+                // every armed poll — see `packed_receiver_box_slots`.
+                if let Some(box_slot) = ctx.packed_receiver_box_slots.get(id).cloned() {
+                    return Ok(ctx.block().load(DOUBLE, &box_slot));
+                }
                 if let Some(i32_slot) = ctx.i32_counter_slots.get(id).cloned() {
                     let i = ctx.block().load(I32, &i32_slot);
                     let v = if ctx.unsigned_i32_locals.contains(id) {
@@ -1110,6 +1116,29 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<String> {
                 super::record_int_facts_for_update(ctx, *id, *op);
                 return Ok(if *prefix { new } else { old });
             }
+            // Packed-clone integer accumulator: touch ONLY the i32 slot —
+            // the double slot stays stale until the clone exits re-sync it
+            // (see `deferred_integer_update_accumulators`). The returned
+            // value materializes via `sitofp` for the rare expression
+            // position; in statement position it is DCE'd.
+            if ctx.deferred_integer_update_accumulators.contains(id) {
+                if let Some(i32_slot) = ctx.i32_counter_slots.get(id).cloned() {
+                    let (old_i32, new_i32) = {
+                        let blk = ctx.block();
+                        let old_i32 = blk.load(I32, &i32_slot);
+                        let delta = match op {
+                            UpdateOp::Increment => "1",
+                            UpdateOp::Decrement => "-1",
+                        };
+                        let new_i32 = blk.add(I32, &old_i32, delta);
+                        blk.store(I32, &new_i32, &i32_slot);
+                        (old_i32, new_i32)
+                    };
+                    super::record_int_facts_for_update(ctx, *id, *op);
+                    let ret_i32 = if *prefix { &new_i32 } else { &old_i32 };
+                    return Ok(ctx.block().sitofp(I32, ret_i32, DOUBLE));
+                }
+            }
             let (storage, storage_is_root) = if let Some(slot) = ctx.locals.get(id).cloned() {
                 (slot, false)
             } else if let Some(global_name) = ctx.module_globals.get(id).cloned() {

crates/perry-codegen/src/expr/masked_window.rs

@@ -12,7 +12,6 @@
 use anyhow::Result;
 use perry_hir::Expr;
 
-use crate::nanbox::POINTER_MASK_I64;
 use crate::native_value::{
     BoundsState, BufferAccessMode, LoweredValue, NativeFactUse, NativeRep, SemanticKind,
 };
@@ -45,10 +44,14 @@ pub(crate) fn masked_window_fact_for_index(
 
 /// Emit the raw in-window f64 element load of the plain-array tiers:
 /// `header + 8 + idx * 8` on the pointer-masked array handle.
-fn emit_raw_window_load(ctx: &mut FnCtx<'_>, arr_box: &str, idx_i32: &str) -> String {
+fn emit_raw_window_load(
+    ctx: &mut FnCtx<'_>,
+    arr_id: Option<u32>,
+    arr_box: &str,
+    idx_i32: &str,
+) -> String {
+    let arr_handle = super::packed_receiver_handle_i64(ctx, arr_id, arr_box);
     let blk = ctx.block();
-    let arr_bits = blk.bitcast_double_to_i64(arr_box);
-    let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64);
     let idx_i64 = blk.zext(I32, idx_i32, I64);
     let byte_offset = blk.shl(I64, &idx_i64, "3");
     let with_header = blk.add(I64, &byte_offset, "8");
@@ -86,7 +89,9 @@ fn emit_window_load_f64(
     fact: &MaskedWindowArrayFact,
 ) -> String {
     match &fact.elem {
-        MaskedWindowElem::PlainF64 => emit_raw_window_load(ctx, arr_box, idx_i32),
+        MaskedWindowElem::PlainF64 => {
+            emit_raw_window_load(ctx, Some(fact.array_local_id), arr_box, idx_i32)
+        }
         MaskedWindowElem::TaI32 { data_ptr } => {
             let data_ptr = data_ptr.clone();
             let raw = emit_ta_window_load(ctx, &data_ptr, idx_i32, "2", I32);
@@ -283,9 +288,8 @@ pub(crate) fn lower_masked_window_index_set(
     fact: &MaskedWindowArrayFact,
 ) {
     {
+        let arr_handle = super::packed_receiver_handle_i64(ctx, Some(arr_id), arr_box);
         let blk = ctx.block();
-        let arr_bits = blk.bitcast_double_to_i64(arr_box);
-        let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64);
         let idx_i64 = blk.zext(I32, idx_i32, I64);
         let byte_offset = blk.shl(I64, &idx_i64, "3");
         let with_header = blk.add(I64, &byte_offset, "8");
@@ -369,7 +373,7 @@ pub(crate) fn lower_masked_window_index_get_i32(
     let idx_i32 = lower_expr_as_i32(ctx, index)?;
     let (value, materialization_note) = match &fact.elem {
         MaskedWindowElem::PlainF64 => {
-            let raw_f64 = emit_raw_window_load(ctx, &arr_box, &idx_i32);
+            let raw_f64 = emit_raw_window_load(ctx, Some(fact.array_local_id), &arr_box, &idx_i32);
             (
                 ctx.block().fptosi(DOUBLE, &raw_f64, I32),
                 "integer_materialization=fptosi_guarded_dense_i32",

crates/perry-codegen/src/expr/mod.rs

@@ -1061,6 +1061,49 @@ pub(crate) struct FnCtx<'a> {
     /// so no conversion exists on either edge; the stale number left in the
     /// root slot during the clone is harmless to a GC scan.
     pub numeric_accumulator_f64_slots: std::collections::HashMap<u32, String>,
+    /// Poll-scoped receiver cache, active only while a packed fast clone is
+    /// being lowered: array local id -> frame-rooted F64 alloca holding the
+    /// receiver BOX. Every in-clone `LocalGet` of the receiver reads this
+    /// alloca instead of the source binding's root, so native-root mem2reg
+    /// promotes it and LLVM hoists the handle mask + element base math out of
+    /// the loop — the per-access root re-derive was 3-5 instructions on every
+    /// packed load and store. The cache is itself rewritten by evacuation and
+    /// refreshes from the source root on the ARMED arm of every loop poll
+    /// (`emit_armed_gc_loop_safepoint`), which is the only place a call-free
+    /// clone can collect; entries in `packed_receiver_refresh` drive that
+    /// reload for EVERY active scope, which keeps nested clones' outer caches
+    /// fresh when an inner loop's poll fires. Rooting the cache is required
+    /// even with that reload: it makes liveness across the poll explicit and
+    /// keeps the shadow and native precise-root lowerings structurally sound.
+    pub packed_receiver_box_slots: std::collections::HashMap<u32, String>,
+    /// (alloca, source ref, source-is-module-global) reload recipes for the
+    /// poll-arm refresh of `packed_receiver_box_slots`.
+    pub packed_receiver_refresh: Vec<(String, String)>,
+    /// Masked-handle twin of `packed_receiver_box_slots`: array id -> i64
+    /// alloca holding `box & POINTER_MASK`. The hot packed lanes' address
+    /// math consults it via `packed_receiver_handle_i64`, removing the
+    /// per-element mask that LLVM cannot PRE across the poll's refresh phi.
+    pub packed_receiver_handle_slots: std::collections::HashMap<u32, String>,
+    /// When set, `emit_armed_gc_loop_safepoint` gates its VOLATILE armed
+    /// load on `(counter & 63) == 0`, so the poll's serialization cost (and
+    /// the receiver-cache re-derive it forces on every element) is paid once
+    /// per 64 iterations instead of per iteration. Sound because the clone
+    /// body is call-free — the poll is its only collection point, and a
+    /// 64-iteration drain delay on a sub-nanosecond loop body is far inside
+    /// the poll contract's tolerance. Holds the counter's i32 slot; set and
+    /// cleared by the packed fast-clone lowering.
+    pub poll_stride_counter_slot: Option<String>,
+    /// Update-only INTEGER accumulators (`c++` count loops) inside a packed
+    /// fast clone: while a local id is in this set, `Expr::Update` touches
+    /// ONLY its parallel i32 slot (a promotable alloca — the loop-carried
+    /// value becomes a 1-cycle integer add instead of the double slot's
+    /// load→fadd→store chain, which cost the count loops ~5 cycles/element
+    /// of pure latency). The double slot goes stale for the clone's duration
+    /// and is re-synced (`sitofp` + store) at every clone exit. Admission
+    /// (packed preheader): writes are Update-only, the local is an
+    /// integer-local with an i32 slot, and an entry range test proves
+    /// |value| < 2^30 so `bound <= 16M` iterations cannot wrap the i32.
+    pub deferred_integer_update_accumulators: std::collections::HashSet<u32>,
 
     /// Representation-selection Phase 1 (RFC `docs/representation-selection-
     /// rfc.md`): LocalId → selected slot representation. Absent = `Boxed`
@@ -1965,6 +2008,25 @@ pub(crate) enum MaskedWindowElem {
     TaF64 { data_ptr: String },
 }
 
+/// The pointer-masked receiver handle for a packed-lane access: the cached
+/// pre-masked value when the clone hoisted it, else the inline
+/// bitcast-and-mask. One i64 load vs two ALU ops — the win is that the load
+/// is loop-invariant to LLVM while the mask of a poll-refreshed phi is not.
+pub(crate) fn packed_receiver_handle_i64(
+    ctx: &mut FnCtx<'_>,
+    arr_id: Option<u32>,
+    arr_box: &str,
+) -> String {
+    if let Some(id) = arr_id {
+        if let Some(slot) = ctx.packed_receiver_handle_slots.get(&id).cloned() {
+            return ctx.block().load(crate::types::I64, &slot);
+        }
+    }
+    let blk = ctx.block();
+    let bits = blk.bitcast_double_to_i64(arr_box);
+    blk.and(crate::types::I64, &bits, crate::nanbox::POINTER_MASK_I64)
+}
+
 /// Read-only masked-index window fact for the dense packed-f64 range loop:
 /// the entry guard (`js_typed_feedback_packed_f64_range_loop_guard_dense`
 /// for the plain tiers, `js_typed_feedback_masked_window_ta_kind` for the