diff --git a/changelog.d/9111-packed-clone-endgame.md b/changelog.d/9111-packed-clone-endgame.md new file mode 100644 index 0000000000..fb57a94adc --- /dev/null +++ b/changelog.d/9111-packed-clone-endgame.md @@ -0,0 +1 @@ +Packed fast clones: poll-scoped receiver/handle caching with a strided armed check, scope-local i32 deferral for count accumulators, and check-free stores of provably-genuine values — every row of the isolated array-operation matrix (count/reduce/store loops across local, module-global, and masked receivers) now runs ahead of Node on the quiet bench host. diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index d0c1c5d0c5..edc80ab12c 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -1193,6 +1193,11 @@ pub(super) fn compile_closure( element_shape_loop_facts: Vec::new(), i32_counter_slots: HashMap::new(), numeric_accumulator_f64_slots: HashMap::new(), + packed_receiver_box_slots: HashMap::new(), + packed_receiver_refresh: Vec::new(), + packed_receiver_handle_slots: HashMap::new(), + poll_stride_counter_slot: None, + deferred_integer_update_accumulators: HashSet::new(), local_slot_reps: HashMap::new(), repsel_context_allows_canonical_i32: repsel_allows, // #7109 split the FIELD out of `repsel_context_allows_canonical_i32`; diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index 445ccf6a97..1b2907032f 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -881,6 +881,11 @@ pub(super) fn compile_module_entry( element_shape_loop_facts: Vec::new(), i32_counter_slots: HashMap::new(), numeric_accumulator_f64_slots: HashMap::new(), + packed_receiver_box_slots: HashMap::new(), + packed_receiver_refresh: Vec::new(), + packed_receiver_handle_slots: HashMap::new(), + poll_stride_counter_slot: None, + deferred_integer_update_accumulators: HashSet::new(), local_slot_reps: HashMap::new(), // #7109: this entry body selects canonical i32/u32/Str on the same // per-value rules as a function body. Phase 1 (#6903) excluded it @@ -1599,6 +1604,11 @@ pub(super) fn compile_module_entry( element_shape_loop_facts: Vec::new(), i32_counter_slots: HashMap::new(), numeric_accumulator_f64_slots: HashMap::new(), + packed_receiver_box_slots: HashMap::new(), + packed_receiver_refresh: Vec::new(), + packed_receiver_handle_slots: HashMap::new(), + poll_stride_counter_slot: None, + deferred_integer_update_accumulators: HashSet::new(), local_slot_reps: HashMap::new(), // #7109: this entry body selects canonical i32/u32/Str on the same // per-value rules as a function body. Phase 1 (#6903) excluded it diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index a3e49f2a5d..31079ce05a 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -1149,6 +1149,11 @@ pub(super) fn compile_function( .collect(), i32_counter_slots: spec_i32_param_slots, numeric_accumulator_f64_slots: HashMap::new(), + packed_receiver_box_slots: HashMap::new(), + packed_receiver_refresh: Vec::new(), + packed_receiver_handle_slots: HashMap::new(), + poll_stride_counter_slot: None, + deferred_integer_update_accumulators: HashSet::new(), repsel_context_allows_canonical_i32: repsel_allows, // #7109 split the FIELD out of `repsel_context_allows_canonical_i32`; // #7128 split the VALUE, which is what the knob actually reads. Until diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index 50a0332b6b..58c0d2f814 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -551,6 +551,11 @@ pub(super) fn compile_method( element_shape_loop_facts: Vec::new(), i32_counter_slots: index_i32_param_slots, numeric_accumulator_f64_slots: HashMap::new(), + packed_receiver_box_slots: HashMap::new(), + packed_receiver_refresh: Vec::new(), + packed_receiver_handle_slots: HashMap::new(), + poll_stride_counter_slot: None, + deferred_integer_update_accumulators: HashSet::new(), local_slot_reps: HashMap::new(), repsel_context_allows_canonical_i32: repsel_allows, // #7109 split the FIELD out of `repsel_context_allows_canonical_i32`; @@ -1718,6 +1723,11 @@ pub(super) fn compile_static_method( element_shape_loop_facts: Vec::new(), i32_counter_slots: HashMap::new(), numeric_accumulator_f64_slots: HashMap::new(), + packed_receiver_box_slots: HashMap::new(), + packed_receiver_refresh: Vec::new(), + packed_receiver_handle_slots: HashMap::new(), + poll_stride_counter_slot: None, + deferred_integer_update_accumulators: HashSet::new(), local_slot_reps: HashMap::new(), repsel_context_allows_canonical_i32: repsel_allows, // #7109 split the FIELD out of `repsel_context_allows_canonical_i32`; diff --git a/crates/perry-codegen/src/expr/index_set_packed_loop.rs b/crates/perry-codegen/src/expr/index_set_packed_loop.rs index 584a52f234..a6d337cc17 100644 --- a/crates/perry-codegen/src/expr/index_set_packed_loop.rs +++ b/crates/perry-codegen/src/expr/index_set_packed_loop.rs @@ -101,7 +101,67 @@ pub(super) fn lower_packed_f64_range_loop_index_set( guard_id: &str, side_exit_label: &str, ) -> Result { + // A RHS that provably materializes a genuine double by construction — + // a literal, the canonical-i32 counter, an in-window guarded load, or + // float arithmetic over those (the masked store's admission predicate) + // — needs no runtime value check at all: the nanbox tag test below + // (fmov + tag math + compare + branch, five per-element instructions) + // exists only for values that could be boxed at runtime. + let statically_genuine = + crate::expr::masked_window::masked_store_rhs_is_genuine_f64(ctx, value); let (val_double, rhs_notes) = lower_packed_f64_loop_store_value(ctx, arr_id, value)?; + if statically_genuine { + let arr_expr = Expr::LocalGet(arr_id); + let arr_box = lower_expr(ctx, &arr_expr)?; + let arr_handle = super::packed_receiver_handle_i64(ctx, Some(arr_id), &arr_box); + let blk = ctx.block(); + let idx_i64 = blk.zext(I32, idx_i32, I64); + let byte_offset = blk.shl(I64, &idx_i64, "3"); + let with_header = blk.add(I64, &byte_offset, "8"); + let element_addr = blk.add(I64, &arr_handle, &with_header); + let element_ptr = blk.inttoptr(I64, &element_addr); + // GC_STORE_AUDIT(POINTER_FREE): statically-genuine packed store — + // the RHS predicate proves an unboxed double, never a heap pointer. + blk.store(DOUBLE, &val_double, &element_ptr); + let stored = LoweredValue { + semantic: SemanticKind::JsNumber, + rep: NativeRep::F64, + llvm_ty: DOUBLE, + value: val_double.clone(), + }; + ctx.record_lowered_value_with_access_mode_and_facts( + "PackedF64RangeLoopStore", + Some(arr_id), + "packed_f64_range_loop_store", + &stored, + Some(BoundsState::Guarded { + guard_id: guard_id.to_string(), + }), + None, + Some(BufferAccessMode::CheckedNative), + None, + None, + None, + vec![ + array_kind_fact(Some(arr_id), "consumed", "packed_f64", None), + raw_f64_layout_fact(Some(arr_id), "consumed", guard_id, None), + ], + Vec::new(), + false, + false, + { + let mut notes = vec![ + "rhs_numeric_guard=static_genuine_f64_proof".to_string(), + "index_range=range_guarded_i32_window".to_string(), + "storage_layout=raw_f64_or_hole_slots".to_string(), + ]; + notes.extend(rhs_notes); + notes + }, + ); + let _ = side_exit_label; + return Ok(val_double); + } let fast_idx = ctx.new_block("packed_f64_range_store.fast"); let exit_idx = ctx.new_block("packed_f64_range_store.side_exit"); @@ -164,9 +224,8 @@ pub(super) fn lower_packed_f64_range_loop_index_set( { let arr_expr = Expr::LocalGet(arr_id); let arr_box = lower_expr(ctx, &arr_expr)?; + let arr_handle = super::packed_receiver_handle_i64(ctx, Some(arr_id), &arr_box); let blk = ctx.block(); - let arr_bits = blk.bitcast_double_to_i64(&arr_box); - let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64); let idx_i64 = blk.zext(I32, idx_i32, I64); let byte_offset = blk.shl(I64, &idx_i64, "3"); let with_header = blk.add(I64, &byte_offset, "8"); diff --git a/crates/perry-codegen/src/expr/literals_vars.rs b/crates/perry-codegen/src/expr/literals_vars.rs index 14013aab5a..30ab1ada26 100644 --- a/crates/perry-codegen/src/expr/literals_vars.rs +++ b/crates/perry-codegen/src/expr/literals_vars.rs @@ -543,6 +543,12 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { if let Some(f64_slot) = ctx.numeric_accumulator_f64_slots.get(id).cloned() { return Ok(ctx.block().load(DOUBLE, &f64_slot)); } + // Poll-scoped receiver cache (packed fast clones): the + // receiver box lives in a promotable alloca, refreshed on + // every armed poll — see `packed_receiver_box_slots`. + if let Some(box_slot) = ctx.packed_receiver_box_slots.get(id).cloned() { + return Ok(ctx.block().load(DOUBLE, &box_slot)); + } if let Some(i32_slot) = ctx.i32_counter_slots.get(id).cloned() { let i = ctx.block().load(I32, &i32_slot); let v = if ctx.unsigned_i32_locals.contains(id) { @@ -1110,6 +1116,29 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { super::record_int_facts_for_update(ctx, *id, *op); return Ok(if *prefix { new } else { old }); } + // Packed-clone integer accumulator: touch ONLY the i32 slot — + // the double slot stays stale until the clone exits re-sync it + // (see `deferred_integer_update_accumulators`). The returned + // value materializes via `sitofp` for the rare expression + // position; in statement position it is DCE'd. + if ctx.deferred_integer_update_accumulators.contains(id) { + if let Some(i32_slot) = ctx.i32_counter_slots.get(id).cloned() { + let (old_i32, new_i32) = { + let blk = ctx.block(); + let old_i32 = blk.load(I32, &i32_slot); + let delta = match op { + UpdateOp::Increment => "1", + UpdateOp::Decrement => "-1", + }; + let new_i32 = blk.add(I32, &old_i32, delta); + blk.store(I32, &new_i32, &i32_slot); + (old_i32, new_i32) + }; + super::record_int_facts_for_update(ctx, *id, *op); + let ret_i32 = if *prefix { &new_i32 } else { &old_i32 }; + return Ok(ctx.block().sitofp(I32, ret_i32, DOUBLE)); + } + } let (storage, storage_is_root) = if let Some(slot) = ctx.locals.get(id).cloned() { (slot, false) } else if let Some(global_name) = ctx.module_globals.get(id).cloned() { diff --git a/crates/perry-codegen/src/expr/masked_window.rs b/crates/perry-codegen/src/expr/masked_window.rs index df8f9bca28..f5c51a4f3e 100644 --- a/crates/perry-codegen/src/expr/masked_window.rs +++ b/crates/perry-codegen/src/expr/masked_window.rs @@ -12,7 +12,6 @@ use anyhow::Result; use perry_hir::Expr; -use crate::nanbox::POINTER_MASK_I64; use crate::native_value::{ BoundsState, BufferAccessMode, LoweredValue, NativeFactUse, NativeRep, SemanticKind, }; @@ -45,10 +44,14 @@ pub(crate) fn masked_window_fact_for_index( /// Emit the raw in-window f64 element load of the plain-array tiers: /// `header + 8 + idx * 8` on the pointer-masked array handle. -fn emit_raw_window_load(ctx: &mut FnCtx<'_>, arr_box: &str, idx_i32: &str) -> String { +fn emit_raw_window_load( + ctx: &mut FnCtx<'_>, + arr_id: Option, + arr_box: &str, + idx_i32: &str, +) -> String { + let arr_handle = super::packed_receiver_handle_i64(ctx, arr_id, arr_box); let blk = ctx.block(); - let arr_bits = blk.bitcast_double_to_i64(arr_box); - let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64); let idx_i64 = blk.zext(I32, idx_i32, I64); let byte_offset = blk.shl(I64, &idx_i64, "3"); let with_header = blk.add(I64, &byte_offset, "8"); @@ -86,7 +89,9 @@ fn emit_window_load_f64( fact: &MaskedWindowArrayFact, ) -> String { match &fact.elem { - MaskedWindowElem::PlainF64 => emit_raw_window_load(ctx, arr_box, idx_i32), + MaskedWindowElem::PlainF64 => { + emit_raw_window_load(ctx, Some(fact.array_local_id), arr_box, idx_i32) + } MaskedWindowElem::TaI32 { data_ptr } => { let data_ptr = data_ptr.clone(); let raw = emit_ta_window_load(ctx, &data_ptr, idx_i32, "2", I32); @@ -283,9 +288,8 @@ pub(crate) fn lower_masked_window_index_set( fact: &MaskedWindowArrayFact, ) { { + let arr_handle = super::packed_receiver_handle_i64(ctx, Some(arr_id), arr_box); let blk = ctx.block(); - let arr_bits = blk.bitcast_double_to_i64(arr_box); - let arr_handle = blk.and(I64, &arr_bits, POINTER_MASK_I64); let idx_i64 = blk.zext(I32, idx_i32, I64); let byte_offset = blk.shl(I64, &idx_i64, "3"); let with_header = blk.add(I64, &byte_offset, "8"); @@ -369,7 +373,7 @@ pub(crate) fn lower_masked_window_index_get_i32( let idx_i32 = lower_expr_as_i32(ctx, index)?; let (value, materialization_note) = match &fact.elem { MaskedWindowElem::PlainF64 => { - let raw_f64 = emit_raw_window_load(ctx, &arr_box, &idx_i32); + let raw_f64 = emit_raw_window_load(ctx, Some(fact.array_local_id), &arr_box, &idx_i32); ( ctx.block().fptosi(DOUBLE, &raw_f64, I32), "integer_materialization=fptosi_guarded_dense_i32", diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index e4c6d0da24..69416e64fd 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -1061,6 +1061,49 @@ pub(crate) struct FnCtx<'a> { /// so no conversion exists on either edge; the stale number left in the /// root slot during the clone is harmless to a GC scan. pub numeric_accumulator_f64_slots: std::collections::HashMap, + /// Poll-scoped receiver cache, active only while a packed fast clone is + /// being lowered: array local id -> frame-rooted F64 alloca holding the + /// receiver BOX. Every in-clone `LocalGet` of the receiver reads this + /// alloca instead of the source binding's root, so native-root mem2reg + /// promotes it and LLVM hoists the handle mask + element base math out of + /// the loop — the per-access root re-derive was 3-5 instructions on every + /// packed load and store. The cache is itself rewritten by evacuation and + /// refreshes from the source root on the ARMED arm of every loop poll + /// (`emit_armed_gc_loop_safepoint`), which is the only place a call-free + /// clone can collect; entries in `packed_receiver_refresh` drive that + /// reload for EVERY active scope, which keeps nested clones' outer caches + /// fresh when an inner loop's poll fires. Rooting the cache is required + /// even with that reload: it makes liveness across the poll explicit and + /// keeps the shadow and native precise-root lowerings structurally sound. + pub packed_receiver_box_slots: std::collections::HashMap, + /// (alloca, source ref, source-is-module-global) reload recipes for the + /// poll-arm refresh of `packed_receiver_box_slots`. + pub packed_receiver_refresh: Vec<(String, String)>, + /// Masked-handle twin of `packed_receiver_box_slots`: array id -> i64 + /// alloca holding `box & POINTER_MASK`. The hot packed lanes' address + /// math consults it via `packed_receiver_handle_i64`, removing the + /// per-element mask that LLVM cannot PRE across the poll's refresh phi. + pub packed_receiver_handle_slots: std::collections::HashMap, + /// When set, `emit_armed_gc_loop_safepoint` gates its VOLATILE armed + /// load on `(counter & 63) == 0`, so the poll's serialization cost (and + /// the receiver-cache re-derive it forces on every element) is paid once + /// per 64 iterations instead of per iteration. Sound because the clone + /// body is call-free — the poll is its only collection point, and a + /// 64-iteration drain delay on a sub-nanosecond loop body is far inside + /// the poll contract's tolerance. Holds the counter's i32 slot; set and + /// cleared by the packed fast-clone lowering. + pub poll_stride_counter_slot: Option, + /// Update-only INTEGER accumulators (`c++` count loops) inside a packed + /// fast clone: while a local id is in this set, `Expr::Update` touches + /// ONLY its parallel i32 slot (a promotable alloca — the loop-carried + /// value becomes a 1-cycle integer add instead of the double slot's + /// load→fadd→store chain, which cost the count loops ~5 cycles/element + /// of pure latency). The double slot goes stale for the clone's duration + /// and is re-synced (`sitofp` + store) at every clone exit. Admission + /// (packed preheader): writes are Update-only, the local is an + /// integer-local with an i32 slot, and an entry range test proves + /// |value| < 2^30 so `bound <= 16M` iterations cannot wrap the i32. + pub deferred_integer_update_accumulators: std::collections::HashSet, /// Representation-selection Phase 1 (RFC `docs/representation-selection- /// rfc.md`): LocalId → selected slot representation. Absent = `Boxed` @@ -1965,6 +2008,25 @@ pub(crate) enum MaskedWindowElem { TaF64 { data_ptr: String }, } +/// The pointer-masked receiver handle for a packed-lane access: the cached +/// pre-masked value when the clone hoisted it, else the inline +/// bitcast-and-mask. One i64 load vs two ALU ops — the win is that the load +/// is loop-invariant to LLVM while the mask of a poll-refreshed phi is not. +pub(crate) fn packed_receiver_handle_i64( + ctx: &mut FnCtx<'_>, + arr_id: Option, + arr_box: &str, +) -> String { + if let Some(id) = arr_id { + if let Some(slot) = ctx.packed_receiver_handle_slots.get(&id).cloned() { + return ctx.block().load(crate::types::I64, &slot); + } + } + let blk = ctx.block(); + let bits = blk.bitcast_double_to_i64(arr_box); + blk.and(crate::types::I64, &bits, crate::nanbox::POINTER_MASK_I64) +} + /// Read-only masked-index window fact for the dense packed-f64 range loop: /// the entry guard (`js_typed_feedback_packed_f64_range_loop_guard_dense` /// for the plain tiers, `js_typed_feedback_masked_window_ta_kind` for the diff --git a/crates/perry-codegen/src/stmt/loops.rs b/crates/perry-codegen/src/stmt/loops.rs index 307261e756..54e4ea3678 100644 --- a/crates/perry-codegen/src/stmt/loops.rs +++ b/crates/perry-codegen/src/stmt/loops.rs @@ -564,7 +564,13 @@ fn emit_range_loop_accumulator_admission( struct PackedAccumulatorScope { accumulators: Vec, unboxed: Vec<(u32, String, String)>, + /// Update-only integer accumulators deferred to the i32 slot for the + /// clone: (id, i32 slot, double slot) — exits re-sync the double. + deferred_integer: Vec<(u32, String, String)>, side_exit_override: Option, + /// Receiver ids this scope cached into promotable allocas (see + /// `FnCtx::packed_receiver_box_slots`); cleared by `finish`. + hoisted_receivers: Vec, } impl PackedAccumulatorScope { @@ -572,9 +578,172 @@ impl PackedAccumulatorScope { Self { accumulators: Vec::new(), unboxed: Vec::new(), + deferred_integer: Vec::new(), side_exit_override: None, + hoisted_receivers: Vec::new(), } } +} + +/// Admit Update-only INTEGER accumulators (`c++`) for i32-slot deferral: +/// writes all Updates, an integer local with a live i32 slot (the +/// authoritative in-clone storage — reads already prefer it), not the +/// loop counter, plain-local storage. An entry range test +/// (|value| < 2^30) branches to the slow clone so `bound <= 16M` +/// iterations of `add i32` cannot wrap; the i32 and double slots are in +/// sync at entry by the Update/LocalSet mirror invariant, so the i32 +/// value IS the number. +fn admit_integer_update_accumulators( + ctx: &mut FnCtx<'_>, + body: &[Stmt], + counter_id: u32, + slow_pre_label: &str, + block_prefix: &str, +) -> Vec<(u32, String, String)> { + let mut writes = std::collections::BTreeMap::new(); + super::stable_packed_accumulator::collect_local_writes(body, &mut writes); + let mut in_range: Option = None; + let mut admitted: Vec<(u32, String, String)> = Vec::new(); + for (id, ws) in &writes { + if *id == counter_id + || ws.is_empty() + || !ws.iter().all(|w| w.is_none()) + || !ctx.integer_locals.contains(id) + || ctx.boxed_vars.contains(id) + || ctx.closure_captures.contains_key(id) + || ctx.module_globals.contains_key(id) + { + continue; + } + let Some(dbl_slot) = ctx.locals.get(id).cloned() else { + continue; + }; + // Count accumulators are rarely index-used, so most have no i32 + // slot yet — create a scope-local one here: range-test the (integer + // by `integer_locals` invariant) double value, seed the slot with + // its exact `fptosi`, and REGISTER it so every in-clone read takes + // the i32-first `LocalGet` arm. `finish`/the trampoline unregister + // it and sync the double back. + let created_slot = if ctx.i32_counter_slots.contains_key(id) { + None + } else { + Some(ctx.func.alloca_entry(I32)) + }; + let existing_slot = ctx.i32_counter_slots.get(id).cloned(); + let blk = ctx.block(); + let (value_i32, ok) = if let Some(new_slot) = &created_slot { + let dbl = blk.load(DOUBLE, &dbl_slot); + let below = blk.fcmp("olt", &dbl, "1073741824.0"); + let above = blk.fcmp("ogt", &dbl, "-1073741824.0"); + let ok = blk.and(I1, &below, &above); + let as_i32 = blk.fptosi(DOUBLE, &dbl, I32); + blk.store(I32, &as_i32, new_slot); + (new_slot.clone(), ok) + } else { + let slot = existing_slot.expect("checked contains_key above"); + let value = blk.load(I32, &slot); + let below = blk.icmp_slt(I32, &value, "1073741824"); + let above = blk.icmp_sgt(I32, &value, "-1073741824"); + (slot, blk.and(I1, &below, &above)) + }; + in_range = Some(match in_range { + Some(prev) => ctx.block().and(I1, &prev, &ok), + None => ok, + }); + if let Some(new_slot) = created_slot { + ctx.i32_counter_slots.insert(*id, new_slot); + } + admitted.push((*id, value_i32, dbl_slot)); + } + if admitted.is_empty() { + return admitted; + } + let ok_idx = ctx.new_block(&format!("{block_prefix}.intacc.ok")); + let ok_label = ctx.block_label(ok_idx); + let in_range = in_range.expect("at least one admitted"); + ctx.block().cond_br(&in_range, &ok_label, slow_pre_label); + ctx.current_block = ok_idx; + for (id, _, _) in &admitted { + ctx.deferred_integer_update_accumulators.insert(*id); + } + admitted +} + +impl PackedAccumulatorScope { + /// Cache each receiver's box in a promotable precise-root alloca for this + /// fast clone. Receivers here are matcher-validated plain locals or module + /// globals (never captures/boxes), and the clone body is call-free, so the + /// only collection point is the loop poll — whose armed arm reloads every + /// entry in `packed_receiver_refresh` and re-derives its masked handle. + fn hoist_receivers(&mut self, ctx: &mut FnCtx<'_>, array_ids: &[u32]) { + for arr_id in array_ids { + if ctx.packed_receiver_box_slots.contains_key(arr_id) { + continue; + } + let source_ref = if let Some(slot) = ctx.locals.get(arr_id) { + slot.clone() + } else if let Some(global_name) = ctx.module_globals.get(arr_id) { + format!("@{}", global_name) + } else { + continue; + }; + let current = ctx.block().load(DOUBLE, &source_ref); + let alloca = ctx.func.alloca_entry(DOUBLE); + let handle_alloca = ctx.func.alloca_entry(I64); + // `root_entry_alloca` hoists the bind into entry setup, so seed + // the cache before that bind can make the collector dereference + // it. The later store publishes the live receiver and the bind + // makes evacuation rewrite this cache itself. Under native roots + // the bind becomes an addrspace(1) value that mem2reg can still + // promote, retaining the receiver-cache fast path while making + // its liveness across a strided poll explicit to the checker. + let undef = crate::nanbox::double_literal(f64::from_bits(crate::nanbox::TAG_UNDEFINED)); + ctx.func.entry_allocas_push_store(DOUBLE, &undef, &alloca); + { + let blk = ctx.block(); + blk.store(DOUBLE, ¤t, &alloca); + } + crate::expr::root_entry_alloca(ctx, &alloca); + { + let blk = ctx.block(); + let bits = blk.bitcast_double_to_i64(¤t); + let handle = blk.and(I64, &bits, crate::nanbox::POINTER_MASK_I64); + blk.store(I64, &handle, &handle_alloca); + } + ctx.packed_receiver_box_slots + .insert(*arr_id, alloca.clone()); + ctx.packed_receiver_handle_slots + .insert(*arr_id, handle_alloca); + ctx.packed_receiver_refresh.push((alloca, source_ref)); + self.hoisted_receivers.push(*arr_id); + } + } + + /// Integer-only scopes still need the side-exit write-back trampoline + /// (the double slot is stale mid-clone); build it and set the override. + fn with_deferred_trampoline( + mut self, + ctx: &mut FnCtx<'_>, + slow_pre_label: &str, + block_prefix: &str, + ) -> Self { + if self.deferred_integer.is_empty() { + return self; + } + let tramp_idx = ctx.new_block(&format!("{block_prefix}.intacc.writeback_exit")); + let saved = ctx.current_block; + ctx.current_block = tramp_idx; + for (_, i32_slot, dbl_slot) in &self.deferred_integer { + let blk = ctx.block(); + let value = blk.load(I32, i32_slot); + let as_double = blk.sitofp(I32, &value, DOUBLE); + blk.store(DOUBLE, &as_double, dbl_slot); + } + ctx.block().br(slow_pre_label); + ctx.current_block = saved; + self.side_exit_override = Some(ctx.block_label(tramp_idx)); + self + } /// The label packed facts should carry as their side exit: the /// write-back trampoline when unboxed accumulators exist, else the slow @@ -603,6 +772,27 @@ impl PackedAccumulatorScope { } ctx.numeric_accumulator_f64_slots.remove(id); } + for (id, i32_slot, dbl_slot) in &self.deferred_integer { + if emit_writeback { + let blk = ctx.block(); + let value = blk.load(I32, i32_slot); + let as_double = blk.sitofp(I32, &value, DOUBLE); + blk.store(DOUBLE, &as_double, dbl_slot); + } + ctx.deferred_integer_update_accumulators.remove(id); + // A slot this scope created is scope-local: unregister it so + // post-loop reads go back to the (now re-synced) double slot. + if ctx.i32_counter_slots.get(id) == Some(i32_slot) { + ctx.i32_counter_slots.remove(id); + } + } + for arr_id in &self.hoisted_receivers { + if let Some(alloca) = ctx.packed_receiver_box_slots.remove(arr_id) { + ctx.packed_receiver_refresh + .retain(|(slot, _)| slot != &alloca); + } + ctx.packed_receiver_handle_slots.remove(arr_id); + } } } @@ -617,9 +807,23 @@ fn emit_packed_numeric_accumulator_admission( let accumulators = super::stable_packed_accumulator::collect_numeric_accumulators( ctx, body, array_id, counter_id, ); - if accumulators.is_empty() { + // Integer (`c++`) accumulators admit independently of the float set — + // a pure count loop has no float accumulator at all. + let scope_deferred_integer = + admit_integer_update_accumulators(ctx, body, counter_id, slow_pre_label, block_prefix); + if accumulators.is_empty() && scope_deferred_integer.is_empty() { return PackedAccumulatorScope::empty(); } + if accumulators.is_empty() { + return PackedAccumulatorScope { + accumulators, + unboxed: Vec::new(), + deferred_integer: scope_deferred_integer, + side_exit_override: None, + hoisted_receivers: Vec::new(), + } + .with_deferred_trampoline(ctx, slow_pre_label, block_prefix); + } let mut loaded: Vec<(u32, String, String)> = Vec::new(); let mut all_numbers: Option = None; for id in &accumulators { @@ -674,7 +878,7 @@ fn emit_packed_numeric_accumulator_admission( ctx.numeric_accumulator_f64_slots.insert(id, alloca.clone()); unboxed.push((id, alloca, slot)); } - let side_exit_override = if unboxed.is_empty() { + let side_exit_override = if unboxed.is_empty() && scope_deferred_integer.is_empty() { None } else { // Side-exit trampoline: any mid-iteration exit (a hole-checked load, @@ -688,6 +892,12 @@ fn emit_packed_numeric_accumulator_admission( let value = ctx.block().load(DOUBLE, alloca); ctx.block().store(DOUBLE, &value, real_slot); } + for (_, i32_slot, dbl_slot) in &scope_deferred_integer { + let blk = ctx.block(); + let value = blk.load(I32, i32_slot); + let as_double = blk.sitofp(I32, &value, DOUBLE); + blk.store(DOUBLE, &as_double, dbl_slot); + } ctx.block().br(slow_pre_label); ctx.current_block = saved; Some(ctx.block_label(tramp_idx)) @@ -695,7 +905,9 @@ fn emit_packed_numeric_accumulator_admission( PackedAccumulatorScope { accumulators, unboxed, + deferred_integer: scope_deferred_integer, side_exit_override, + hoisted_receivers: Vec::new(), } } @@ -767,7 +979,7 @@ fn lower_packed_f64_versioned_for( let packed_scope_id = ctx.next_loop_proof_scope_id(); ctx.current_block = fast_pre_idx; - let acc_scope = emit_packed_numeric_accumulator_admission( + let mut acc_scope = emit_packed_numeric_accumulator_admission( ctx, body, matched.array_id, @@ -775,6 +987,7 @@ fn lower_packed_f64_versioned_for( &slow_pre_label, loop_label, ); + acc_scope.hoist_receivers(ctx, &[matched.array_id]); ctx.packed_f64_loop_facts.push(PackedF64LoopFact { index_local_id: matched.counter_id, array_local_id: matched.array_id, @@ -802,6 +1015,8 @@ fn lower_packed_f64_versioned_for( let len_ptr = blk.inttoptr(I64, &arr_handle); blk.load(I32, &len_ptr) }; + let saved_stride = ctx.poll_stride_counter_slot.take(); + ctx.poll_stride_counter_slot = ctx.i32_counter_slots.get(&matched.counter_id).cloned(); lower_for_after_init_with_i32_bound( ctx, init, @@ -811,6 +1026,7 @@ fn lower_packed_f64_versioned_for( &format!("for.{loop_label}_fast"), Some((matched.counter_id, hoisted_len_i32)), )?; + ctx.poll_stride_counter_slot = saved_stride; ctx.packed_f64_loop_facts .retain(|fact| fact.scope_id != packed_scope_id); acc_scope.finish(ctx); @@ -2225,13 +2441,19 @@ fn lower_packed_f64_range_versioned_for( ctx.current_block = fast_i32_pre_idx; let scope_i32 = ctx.next_loop_proof_scope_id(); - let acc_scope = emit_range_loop_accumulator_admission( + let mut acc_scope = emit_range_loop_accumulator_admission( ctx, &matched, body, &slow_pre_label, "packed_f64_range.fast_i32", ); + let range_receiver_ids: Vec = matched + .arrays + .iter() + .map(|access| access.array_id) + .collect(); + acc_scope.hoist_receivers(ctx, &range_receiver_ids); let fact_side_exit = acc_scope.fact_side_exit(&slow_pre_label); push_packed_f64_range_facts( ctx, @@ -2243,6 +2465,8 @@ fn lower_packed_f64_range_versioned_for( false, &acc_scope.accumulators, ); + let saved_stride = ctx.poll_stride_counter_slot.take(); + ctx.poll_stride_counter_slot = ctx.i32_counter_slots.get(&matched.counter_id).cloned(); lower_for_after_init_with_i32_bound( ctx, init, @@ -2252,6 +2476,7 @@ fn lower_packed_f64_range_versioned_for( "for.packed_f64_range_fast_i32", Some((matched.counter_id, bound_i32.clone())), )?; + ctx.poll_stride_counter_slot = saved_stride; ctx.packed_f64_loop_facts .retain(|fact| fact.scope_id != scope_i32); ctx.masked_window_array_facts @@ -2264,13 +2489,19 @@ fn lower_packed_f64_range_versioned_for( ctx.current_block = fast_pre_idx; let scope_f64 = ctx.next_loop_proof_scope_id(); - let acc_scope = emit_range_loop_accumulator_admission( + let mut acc_scope = emit_range_loop_accumulator_admission( ctx, &matched, body, &slow_pre_label, "packed_f64_range.fast", ); + let range_receiver_ids: Vec = matched + .arrays + .iter() + .map(|access| access.array_id) + .collect(); + acc_scope.hoist_receivers(ctx, &range_receiver_ids); let fact_side_exit = acc_scope.fact_side_exit(&slow_pre_label); push_packed_f64_range_facts( ctx, @@ -2282,6 +2513,8 @@ fn lower_packed_f64_range_versioned_for( has_stores, &acc_scope.accumulators, ); + let saved_stride = ctx.poll_stride_counter_slot.take(); + ctx.poll_stride_counter_slot = ctx.i32_counter_slots.get(&matched.counter_id).cloned(); lower_for_after_init_with_i32_bound( ctx, init, @@ -2291,6 +2524,7 @@ fn lower_packed_f64_range_versioned_for( "for.packed_f64_range_fast", Some((matched.counter_id, bound_i32.clone())), )?; + ctx.poll_stride_counter_slot = saved_stride; ctx.packed_f64_loop_facts .retain(|fact| fact.scope_id != scope_f64); ctx.masked_window_array_facts @@ -2313,13 +2547,19 @@ fn lower_packed_f64_range_versioned_for( let packed_scope_id = ctx.next_loop_proof_scope_id(); ctx.current_block = fast_pre_idx; - let acc_scope = emit_range_loop_accumulator_admission( + let mut acc_scope = emit_range_loop_accumulator_admission( ctx, &matched, body, &slow_pre_label, "packed_f64_range.classic", ); + let range_receiver_ids: Vec = matched + .arrays + .iter() + .map(|access| access.array_id) + .collect(); + acc_scope.hoist_receivers(ctx, &range_receiver_ids); let fact_side_exit = acc_scope.fact_side_exit(&slow_pre_label); push_packed_f64_range_facts( ctx, @@ -2331,6 +2571,8 @@ fn lower_packed_f64_range_versioned_for( false, &acc_scope.accumulators, ); + let saved_stride = ctx.poll_stride_counter_slot.take(); + ctx.poll_stride_counter_slot = ctx.i32_counter_slots.get(&matched.counter_id).cloned(); lower_for_after_init_with_i32_bound( ctx, init, @@ -2340,6 +2582,7 @@ fn lower_packed_f64_range_versioned_for( "for.packed_f64_range_fast", Some((matched.counter_id, bound_i32.clone())), )?; + ctx.poll_stride_counter_slot = saved_stride; ctx.packed_f64_loop_facts .retain(|fact| fact.scope_id != packed_scope_id); ctx.masked_window_array_facts @@ -6123,6 +6366,29 @@ fn emit_armed_gc_loop_safepoint(ctx: &mut FnCtx<'_>) { let done_idx = ctx.new_block("gcpoll.done"); let poll_label = ctx.block_label(poll_idx); let done_label = ctx.block_label(done_idx); + // Packed fast clones stride the poll: the VOLATILE armed load + // serializes, and its clobber potential forces the receiver-cache base + // math to be re-derived on EVERY element (disassembly: the `ldr w, [x]` + // + `cbz` pair plus a re-mask/re-add per element were the last fat in + // an otherwise branchless fcsel loop). Gate it on `(i & 63) == 0` — + // plain scalar ops LLVM folds through its unroller — so the volatile + // load runs once per 64 iterations. Sound: the clone body is call-free, + // the poll is its only collection point, and a 64-iteration drain delay + // on a sub-nanosecond body is far inside the poll contract's tolerance + // (the arm/drain handshake has no fixed-latency requirement, only + // eventual progress — see gc/poll_arm.rs). + if let Some(counter_slot) = ctx.poll_stride_counter_slot.clone() { + let check_idx = ctx.new_block("gcpoll.stride_check"); + let check_label = ctx.block_label(check_idx); + { + let blk = ctx.block(); + let i = blk.load(I32, &counter_slot); + let masked = blk.and(I32, &i, "63"); + let due_slot = blk.icmp_eq(I32, &masked, "0"); + blk.cond_br(&due_slot, &check_label, &done_label); + } + ctx.current_block = check_idx; + } { let blk = ctx.block(); let armed = blk.load_volatile(I32, "@PERRY_GC_POLL_ARMED"); @@ -6131,8 +6397,32 @@ fn emit_armed_gc_loop_safepoint(ctx: &mut FnCtx<'_>) { } ctx.current_block = poll_idx; { + let refresh = ctx.packed_receiver_refresh.clone(); + let handle_pairs: Vec<(String, String)> = ctx + .packed_receiver_handle_slots + .iter() + .filter_map(|(id, handle_slot)| { + ctx.packed_receiver_box_slots + .get(id) + .map(|box_slot| (handle_slot.clone(), box_slot.clone())) + }) + .collect(); let blk = ctx.block(); blk.call_void("js_gc_loop_safepoint", &[]); + // A fired poll may have MOVED every cached packed receiver — reload + // each active cache (all scopes: an inner loop's poll must refresh + // outer clones' caches too) from its GC-updated root before any + // cached-base access runs again. + for (alloca, source_ref) in &refresh { + let fresh = blk.load(DOUBLE, source_ref); + blk.store(DOUBLE, &fresh, alloca); + } + for (handle_slot, box_slot) in &handle_pairs { + let fresh = blk.load(DOUBLE, box_slot); + let bits = blk.bitcast_double_to_i64(&fresh); + let handle = blk.and(I64, &bits, crate::nanbox::POINTER_MASK_I64); + blk.store(I64, &handle, handle_slot); + } blk.br(&done_label); } ctx.current_block = done_idx; @@ -6392,10 +6682,20 @@ fn classify_for_length_hoist_impl( _ => return None, }; let receiver_is_eligible = if allow_materialization_hazard { - ctx.locals.contains_key(&arr_id) + // Module globals qualify alongside plain locals: the storage is a + // registered root cell read with one load (the same addressable set + // `packed_loop_array_binding_storage_is_addressable` admits), the + // matched body is call-free so nothing can rebind the global + // mid-loop, and the entry guard revalidates the live array either + // way. Excluding them silently kept every module-global receiver's + // `i < g.length` loop off the versioned clones (probe: 8.1 ns/el + // for a count loop the local-receiver twin runs at 1.6). + let plain_local = ctx.locals.contains_key(&arr_id) && !ctx.boxed_vars.contains(&arr_id) - && !ctx.module_globals.contains_key(&arr_id) - && !ctx.scalar_replaced_arrays.contains_key(&arr_id) + && !ctx.module_globals.contains_key(&arr_id); + let module_global = + !ctx.locals.contains_key(&arr_id) && ctx.module_globals.contains_key(&arr_id); + (plain_local || module_global) && !ctx.scalar_replaced_arrays.contains_key(&arr_id) } else { array_length_receiver_is_loop_local(ctx, arr_id) };