From 60f9c4d0e6956fd4031385ce74103b44d6d39925 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 20 Jun 2026 23:55:36 +0000 Subject: [PATCH 1/8] perf: NEON SIMD unaligned path for ends_with Replace the scalar per-element loop in the NEON backend of eq_words_unaligned with true 2-lane SIMD: vld1q_u64 loads, vshlq_u64 (variable shift, negative = logical right), vorrq_u64 for window construction, and vceqq_u64 for parallel comparison. Previously the NEON backend iterated with 'while i + 2 <= len' but computed each 64-bit window one at a time via scalar shift and OR, matching neither SSE2 nor AVX2 which use parallel _mm_srl_epi64 / _mm_sll_epi64. Co-authored-by: Claude Co-authored-by: DeepSeek AI --- benches/bit_ops_shl_assign.rs | 16 ++++---- .../funcs_for_ends_with_core.rs | 39 ++++++++++++++++--- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/benches/bit_ops_shl_assign.rs b/benches/bit_ops_shl_assign.rs index ee0a9cb..9c154ac 100644 --- a/benches/bit_ops_shl_assign.rs +++ b/benches/bit_ops_shl_assign.rs @@ -6,45 +6,45 @@ fn main() { } // Shift by 1 bit (bit-level shift, no word-level shortcut). -#[divan::bench(name = "shl/len_4096_by_1/owned")] +#[divan::bench(name = "shl_assign/len_4096_by_1/owned")] fn shl_len_4096_by_1_owned(bencher: Bencher) { bench_shl(bencher, 4096, 1); } -#[divan::bench(name = "shl/len_4096_by_1/assign")] +#[divan::bench(name = "shl_assign/len_4096_by_1/assign")] fn shl_len_4096_by_1_assign(bencher: Bencher) { bench_shl_assign(bencher, 4096, 1); } // Shift by 64 bits (word-level shift, the fast path). -#[divan::bench(name = "shl/len_4096_by_64/owned")] +#[divan::bench(name = "shl_assign/len_4096_by_64/owned")] fn shl_len_4096_by_64_owned(bencher: Bencher) { bench_shl(bencher, 4096, 64); } -#[divan::bench(name = "shl/len_4096_by_64/assign")] +#[divan::bench(name = "shl_assign/len_4096_by_64/assign")] fn shl_len_4096_by_64_assign(bencher: Bencher) { bench_shl_assign(bencher, 4096, 64); } // Shift large array by 1 — worst-case for SIMD (cascading carries). -#[divan::bench(name = "shl/len_65536_by_1/owned")] +#[divan::bench(name = "shl_assign/len_65536_by_1/owned")] fn shl_len_65536_by_1_owned(bencher: Bencher) { bench_shl(bencher, 65_536, 1); } -#[divan::bench(name = "shl/len_65536_by_1/assign")] +#[divan::bench(name = "shl_assign/len_65536_by_1/assign")] fn shl_len_65536_by_1_assign(bencher: Bencher) { bench_shl_assign(bencher, 65_536, 1); } // Shift by a mixed amount (both word and bit shift components). -#[divan::bench(name = "shl/len_65536_by_17/owned")] +#[divan::bench(name = "shl_assign/len_65536_by_17/owned")] fn shl_len_65536_by_17_owned(bencher: Bencher) { bench_shl(bencher, 65_536, 17); } -#[divan::bench(name = "shl/len_65536_by_17/assign")] +#[divan::bench(name = "shl_assign/len_65536_by_17/assign")] fn shl_len_65536_by_17_assign(bencher: Bencher) { bench_shl_assign(bencher, 65_536, 17); } diff --git a/src/bit_string/impls_for_matching/impls_for_matches_at/funcs_for_ends_with_core.rs b/src/bit_string/impls_for_matching/impls_for_matches_at/funcs_for_ends_with_core.rs index f2b3d93..9df4719 100644 --- a/src/bit_string/impls_for_matching/impls_for_matches_at/funcs_for_ends_with_core.rs +++ b/src/bit_string/impls_for_matching/impls_for_matches_at/funcs_for_ends_with_core.rs @@ -169,6 +169,9 @@ mod sse2 { #[cfg(target_arch = "aarch64")] mod neon { use crate::WORD_BITS; + use core::arch::aarch64::{ + vceqq_u64, vdupq_n_s64, vgetq_lane_u64, vld1q_u64, vorrq_u64, vshlq_u64, + }; #[target_feature(enable = "neon")] pub(super) unsafe fn eq_words_unaligned( @@ -177,17 +180,40 @@ mod neon { len: usize, shift: usize, ) -> bool { + // SAFETY: `shift` is in [1, WORD_BITS); the caller guarantees this + // via the `shift == 0` fast-path in the entry point. + // Both shift vectors fit in i64: + // shift ∈ [1, 63] → -shift ∈ [-63, -1] + // WORD_BITS - shift ∈ [1, 63] + let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; + let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; + + // Process 2 lanes (128 bits) per iteration. let mut i = 0; while i + 2 <= len { - for k in 0..2 { - let w0 = sw[i + k]; - let w1 = sw[i + k + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != pw[i + k] { - return false; - } + // Load [sw[i], sw[i+1]] and [sw[i+1], sw[i+2]]. + let w0 = unsafe { vld1q_u64(sw.as_ptr().add(i)) }; + let w1 = unsafe { vld1q_u64(sw.as_ptr().add(i + 1)) }; + + // Build the shifted 64-bit window for each lane: + // window[k] = (sw[i+k] >> shift) | (sw[i+k+1] << (64 - shift)) + // vshlq_u64 with a negative shift amount performs a logical right shift. + let lo = unsafe { vshlq_u64(w0, neg_shift) }; + let hi = unsafe { vshlq_u64(w1, pos_shift) }; + let window = unsafe { vorrq_u64(lo, hi) }; + + let expected = unsafe { vld1q_u64(pw.as_ptr().add(i)) }; + let cmp = unsafe { vceqq_u64(window, expected) }; + + // Each lane is all-ones on equality → vgetq_lane_u64 returns u64::MAX. + if unsafe { vgetq_lane_u64(cmp, 0) } == 0 || unsafe { vgetq_lane_u64(cmp, 1) } == 0 { + return false; } + i += 2; } + + // Scalar tail for the last word (when len is odd). while i < len { let w0 = sw[i]; let w1 = sw[i + 1]; @@ -196,6 +222,7 @@ mod neon { } i += 1; } + true } } From ba64f8cf138e0fb48ec4b6637b2eb2012d7cb263 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 20 Jun 2026 23:56:47 +0000 Subject: [PATCH 2/8] fix: wrap vld1q_u64 in unsafe block in NEON find/rfind backends Rust 2024 requires explicit unsafe {} blocks for unsafe intrinsics inside unsafe fn bodies. These two vld1q_u64 calls in the NEON backends of funcs_for_find_core and funcs_for_rfind_core were missing the unsafe block. Co-authored-by: Claude Co-authored-by: DeepSeek AI --- .../impls_for_matching/impls_for_find/funcs_for_find_core.rs | 2 +- .../impls_for_matching/impls_for_find/funcs_for_rfind_core.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bit_string/impls_for_matching/impls_for_find/funcs_for_find_core.rs b/src/bit_string/impls_for_matching/impls_for_find/funcs_for_find_core.rs index 618202a..4d6967c 100644 --- a/src/bit_string/impls_for_matching/impls_for_find/funcs_for_find_core.rs +++ b/src/bit_string/impls_for_matching/impls_for_find/funcs_for_find_core.rs @@ -310,7 +310,7 @@ mod neon { (w0 >> shift) | (w1 << (WORD_BITS - shift)) }; } - let windows = vld1q_u64(wins.as_ptr()); + let windows = unsafe { vld1q_u64(wins.as_ptr()) }; let m = vandq_u64(windows, mask); let c = vceqq_u64(m, needle); if vgetq_lane_u64(c, 0) != 0 { diff --git a/src/bit_string/impls_for_matching/impls_for_find/funcs_for_rfind_core.rs b/src/bit_string/impls_for_matching/impls_for_find/funcs_for_rfind_core.rs index 86b1920..bddeb58 100644 --- a/src/bit_string/impls_for_matching/impls_for_find/funcs_for_rfind_core.rs +++ b/src/bit_string/impls_for_matching/impls_for_find/funcs_for_rfind_core.rs @@ -305,7 +305,7 @@ mod neon { (w0 >> shift) | (w1 << (WORD_BITS - shift)) }; } - let windows = vld1q_u64(wins.as_ptr()); + let windows = unsafe { vld1q_u64(wins.as_ptr()) }; let m = vandq_u64(windows, mask); let c = vceqq_u64(m, needle); From 50dcb71bb158c1597c08da8f1e9ee951f8c1a4c8 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sat, 20 Jun 2026 23:58:46 +0000 Subject: [PATCH 3/8] perf: add SIMD fast path to matches_at Refactor matches_at to reuse the existing SIMD word-equality backends: - Word-aligned offsets (shift == 0): delegate to starts_with_words - Unaligned offsets: delegate to ends_with_words (shifted-window SIMD) Previously matches_at always took the scalar bits_equal_at path. The remainder (partial word) handling mirrors the ends_with pattern. Also add doc comments to all three methods. Co-authored-by: Claude Co-authored-by: DeepSeek AI --- .../impls_for_matches_at.rs | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/src/bit_string/impls_for_matching/impls_for_matches_at.rs b/src/bit_string/impls_for_matching/impls_for_matches_at.rs index 1c3280f..81a7d5b 100644 --- a/src/bit_string/impls_for_matching/impls_for_matches_at.rs +++ b/src/bit_string/impls_for_matching/impls_for_matches_at.rs @@ -7,6 +7,13 @@ mod funcs_for_ends_with_core; mod funcs_for_starts_with_core; impl BitString { + /// Returns `true` if `pattern` matches the bits starting at `index`. + /// + /// For word-aligned offsets (`index % WORD_BITS == 0`) this delegates + /// to the same SIMD word-equality backend used by [`starts_with`]. + /// For unaligned offsets it uses the shifted-window SIMD backend + /// from [`ends_with`]. + #[inline] pub fn matches_at(&self, index: usize, pattern: &Self) -> bool { if index > self.bit_len { return false; @@ -16,9 +23,44 @@ impl BitString { return false; } - bits_equal_at(self, index, pattern) + let shift = index % WORD_BITS; + let base_word = index / WORD_BITS; + let sw: &[u64] = &self.words[base_word..]; + let pw = pattern.as_words(); + let full_words = pattern.bit_len / WORD_BITS; + + if shift == 0 { + if !funcs_for_starts_with_core::starts_with_words(sw, pw, full_words) { + return false; + } + } else { + if !funcs_for_ends_with_core::ends_with_words(sw, pw, full_words, shift) { + return false; + } + } + + let rem = pattern.bit_len % WORD_BITS; + if rem > 0 { + let mask = low_mask(rem); + let h = if shift == 0 { + sw[full_words] + } else { + let w0 = sw[full_words]; + let w1 = sw.get(full_words + 1).copied().unwrap_or(0); + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + if (h & mask) != (pw[full_words] & mask) { + return false; + } + } + + true } + /// Returns `true` if `prefix` is a prefix of `self`. + /// + /// This is equivalent to [`matches_at`]`(0, prefix)` but optimized for + /// the word-aligned position-0 case. #[inline] pub fn starts_with(&self, prefix: &Self) -> bool { if prefix.bit_len > self.bit_len { @@ -45,6 +87,7 @@ impl BitString { true } + /// Returns `true` if `suffix` is a suffix of `self`. #[inline] pub fn ends_with(&self, suffix: &Self) -> bool { if suffix.bit_len > self.bit_len { From aa8d8500faa6b5fb3ca90fafc17bdbf1786fb2f8 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 21 Jun 2026 00:04:54 +0000 Subject: [PATCH 4/8] perf: skip SIMD dispatch in matches_at for short patterns When full_words < SMALL_WORDS, bypass the shift/base_word/slice setup and delegate directly to the scalar bits_equal_at. The underlying starts_with_words/ends_with_words already have the same guard internally, but checking at the matches_at level avoids computing shift, base_word, and the word slice. Co-authored-by: Claude Co-authored-by: DeepSeek AI --- Cargo.toml | 4 + benches/matching_matches_at.rs | 144 ++++++++++++++++++ .../impls_for_matches_at.rs | 12 +- 3 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 benches/matching_matches_at.rs diff --git a/Cargo.toml b/Cargo.toml index a189080..38bbeec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,10 @@ harness = false name = "matching_find" harness = false +[[bench]] +name = "matching_matches_at" +harness = false + [[bench]] name = "matching_starts_with" harness = false diff --git a/benches/matching_matches_at.rs b/benches/matching_matches_at.rs new file mode 100644 index 0000000..af6344f --- /dev/null +++ b/benches/matching_matches_at.rs @@ -0,0 +1,144 @@ +use bit_string::BitString; +use divan::{Bencher, black_box}; +use int_interval::UsizeCO; + +fn main() { + divan::main(); +} + +struct Case { + haystack_bits: BitString, + pattern_bits: BitString, + haystack_string: String, + pattern_string: String, + index: usize, +} + +fn make_bits(len: usize) -> BitString { + let mut bits = BitString::zeros(len); + for i in 0..len { + if (i as u64 * 17 + 3) % 7 == 0 { + bits.set(i, true); + } + } + bits +} + +fn iv(start: usize, len: usize) -> UsizeCO { + UsizeCO::checked_from_start_len(start, len).unwrap() +} + +fn make_case(len: usize, pat_len: usize, index: usize) -> Case { + let h = make_bits(len); + let p = h.slice(iv(index, pat_len)); + Case { + haystack_string: h.to_string(), + pattern_string: p.to_string(), + haystack_bits: h, + pattern_bits: p, + index, + } +} + +fn no_case(len: usize, pat_len: usize, index: usize) -> Case { + let h = make_bits(len); + let mut p = h.slice(iv(index, pat_len)); + p.set(0, !p.get(0).unwrap()); + Case { + haystack_string: h.to_string(), + pattern_string: p.to_string(), + haystack_bits: h, + pattern_bits: p, + index, + } +} + +// --------------------------------------------------------------------------- +// 65-bit haystack (small — scalar path) +// --------------------------------------------------------------------------- + +#[divan::bench(name = "matches_at/len_65/yes/aligned/bit_string")] +fn m65ya(b: Bencher) { + b_bit(b, make_case(65, 4, 64)); +} +#[divan::bench(name = "matches_at/len_65/yes/aligned/string")] +fn m65yas(b: Bencher) { + b_str(b, make_case(65, 4, 64)); +} +#[divan::bench(name = "matches_at/len_65/yes/unaligned/bit_string")] +fn m65yu(b: Bencher) { + b_bit(b, make_case(65, 4, 3)); +} +#[divan::bench(name = "matches_at/len_65/yes/unaligned/string")] +fn m65yus(b: Bencher) { + b_str(b, make_case(65, 4, 3)); +} +#[divan::bench(name = "matches_at/len_65/no/aligned/bit_string")] +fn m65na(b: Bencher) { + b_bit(b, no_case(65, 4, 64)); +} +#[divan::bench(name = "matches_at/len_65/no/aligned/string")] +fn m65nas(b: Bencher) { + b_str(b, no_case(65, 4, 64)); +} +#[divan::bench(name = "matches_at/len_65/no/unaligned/bit_string")] +fn m65nu(b: Bencher) { + b_bit(b, no_case(65, 4, 3)); +} +#[divan::bench(name = "matches_at/len_65/no/unaligned/string")] +fn m65nus(b: Bencher) { + b_str(b, no_case(65, 4, 3)); +} + +// --------------------------------------------------------------------------- +// 65536-bit haystack (large — SIMD path) +// --------------------------------------------------------------------------- + +#[divan::bench(name = "matches_at/len_65536/yes/aligned/bit_string")] +fn m6ya(b: Bencher) { + b_bit(b, make_case(65_536, 128, 64)); +} +#[divan::bench(name = "matches_at/len_65536/yes/aligned/string")] +fn m6yas(b: Bencher) { + b_str(b, make_case(65_536, 128, 64)); +} +#[divan::bench(name = "matches_at/len_65536/yes/unaligned/bit_string")] +fn m6yu(b: Bencher) { + b_bit(b, make_case(65_536, 128, 3)); +} +#[divan::bench(name = "matches_at/len_65536/yes/unaligned/string")] +fn m6yus(b: Bencher) { + b_str(b, make_case(65_536, 128, 3)); +} +#[divan::bench(name = "matches_at/len_65536/no/aligned/bit_string")] +fn m6na(b: Bencher) { + b_bit(b, no_case(65_536, 128, 64)); +} +#[divan::bench(name = "matches_at/len_65536/no/aligned/string")] +fn m6nas(b: Bencher) { + b_str(b, no_case(65_536, 128, 64)); +} +#[divan::bench(name = "matches_at/len_65536/no/unaligned/bit_string")] +fn m6nu(b: Bencher) { + b_bit(b, no_case(65_536, 128, 3)); +} +#[divan::bench(name = "matches_at/len_65536/no/unaligned/string")] +fn m6nus(b: Bencher) { + b_str(b, no_case(65_536, 128, 3)); +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +fn b_bit(b: Bencher, c: Case) { + b.bench(|| black_box(&c.haystack_bits).matches_at(c.index, black_box(&c.pattern_bits))); +} + +fn b_str(b: Bencher, c: Case) { + b.bench(|| { + let h = black_box(&c.haystack_string); + let p = black_box(&c.pattern_string); + h.as_bytes()[c.index..].starts_with(p.as_bytes()) + }); +} diff --git a/src/bit_string/impls_for_matching/impls_for_matches_at.rs b/src/bit_string/impls_for_matching/impls_for_matches_at.rs index 81a7d5b..28fee35 100644 --- a/src/bit_string/impls_for_matching/impls_for_matches_at.rs +++ b/src/bit_string/impls_for_matching/impls_for_matches_at.rs @@ -1,3 +1,4 @@ +use crate::SMALL_WORDS; use crate::WORD_BITS; use crate::funcs_for_bits::low_mask; @@ -23,11 +24,18 @@ impl BitString { return false; } + let pw = pattern.as_words(); + let full_words = pattern.bit_len / WORD_BITS; + + // For short patterns the scalar bits_equal_at is faster than + // SIMD dispatch overhead. + if full_words < SMALL_WORDS { + return bits_equal_at(self, index, pattern); + } + let shift = index % WORD_BITS; let base_word = index / WORD_BITS; let sw: &[u64] = &self.words[base_word..]; - let pw = pattern.as_words(); - let full_words = pattern.bit_len / WORD_BITS; if shift == 0 { if !funcs_for_starts_with_core::starts_with_words(sw, pw, full_words) { From 45240d92efb2c5a78a95d572cff1b61cccf8d885 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 21 Jun 2026 00:05:54 +0000 Subject: [PATCH 5/8] docs: explain rfind max_shift difference between scalar and SIMD backends The scalar rfind computes max_shift = min(last_start - base, 63) as the maximum shift value (0..=63). The SIMD backends compute min(64, last_start - base + 1) as a count (1..=64), then round up to a LANES multiple (2 or 4). Out-of-range positions are guarded by the pos <= last_start check in the verify loop. Co-authored-by: Claude Co-authored-by: DeepSeek AI --- .../impls_for_find/funcs_for_rfind_core.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/bit_string/impls_for_matching/impls_for_find/funcs_for_rfind_core.rs b/src/bit_string/impls_for_matching/impls_for_find/funcs_for_rfind_core.rs index bddeb58..d8a562f 100644 --- a/src/bit_string/impls_for_matching/impls_for_find/funcs_for_rfind_core.rs +++ b/src/bit_string/impls_for_matching/impls_for_find/funcs_for_rfind_core.rs @@ -82,6 +82,10 @@ where let base = i * WORD_BITS; let w0 = haystack[i]; let w1 = haystack.get(i + 1).copied().unwrap_or(0); + // Note: the SIMD backends compute max_shift differently — + // `WORD_BITS.min(last_start - base + 1)` — to process + // shifts in SIMD-sized chunks (2 or 4), relying on + // `pos <= last_start` to skip out-of-range positions. let max_shift = (last_start - base).min(WORD_BITS - 1); for shift in (0..=max_shift).rev() { let pos = base + shift; @@ -139,6 +143,9 @@ mod sse2 { let w1 = haystack.get(i + 1).copied().unwrap_or(0); let max_shift = WORD_BITS.min(last_start - base + 1); + // Round up to a multiple of 2 so the SIMD loop + // processes shifts in 2-lane pairs. Out-of-range + // positions are guarded by `pos <= last_start`. let mut s = max_shift.next_multiple_of(2).min(WORD_BITS); while s > 0 { s -= 2; @@ -219,6 +226,9 @@ mod avx2 { let w1 = haystack.get(i + 1).copied().unwrap_or(0); let max_shift = WORD_BITS.min(last_start - base + 1); + // Round up to a multiple of 4 so the SIMD loop + // processes shifts in 4-lane groups. Out-of-range + // positions are guarded by `pos <= last_start`. let mut s = max_shift.next_multiple_of(4).min(WORD_BITS); while s > 0 { s -= 4; @@ -291,6 +301,9 @@ mod neon { let w1 = haystack.get(i + 1).copied().unwrap_or(0); let max_shift = WORD_BITS.min(last_start - base + 1); + // Round up to a multiple of 2 so the SIMD loop + // processes shifts in 2-lane pairs. Out-of-range + // positions are guarded by `pos <= last_start`. let mut s = max_shift.next_multiple_of(2).min(WORD_BITS); while s > 0 { s -= 2; From 41ed278181fad1d62480f21d71518008254332b6 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 21 Jun 2026 00:16:19 +0000 Subject: [PATCH 6/8] perf: add SIMD fast path to bits_equal_at, make it a method Move bits_equal_at from a private free function in impls_for_matching to a pub(crate) method on BitString in impls_for_matches_at, giving it direct access to starts_with_words / ends_with_words. The method now uses SIMD word-equality for long patterns (full_words >= SMALL_WORDS) and scalar comparison for short ones. matches_at simplifies to a thin guard-and-delegate wrapper. All call sites updated: find/rfind/contains verify closures, backend equivalence tests, and bits_equal_at unit tests. Co-authored-by: Claude Co-authored-by: DeepSeek AI --- src/bit_string/impls_for_matching.rs | 31 -------- .../impls_for_matching/impls_for_find.rs | 10 +-- .../tests_for_backend_equivalence.rs | 6 +- .../tests_for_backend_equivalence.rs | 6 +- .../impls_for_matches_at.rs | 71 ++++++++++--------- .../tests_for_bits_equal_at.rs | 24 +++---- 6 files changed, 59 insertions(+), 89 deletions(-) diff --git a/src/bit_string/impls_for_matching.rs b/src/bit_string/impls_for_matching.rs index 0b92e4a..f9e74d4 100644 --- a/src/bit_string/impls_for_matching.rs +++ b/src/bit_string/impls_for_matching.rs @@ -1,38 +1,7 @@ use crate::bit_string::traits::*; -use crate::funcs_for_bits::*; use super::*; -/// Compare `needle` bits against `haystack` starting at `offset`, using -/// word-level reads so that each iteration compares up to 64 bits. -#[inline] -fn bits_equal_at(haystack: &BitString, offset: usize, needle: &BitString) -> bool { - let needle_bits = needle.bit_len; - let needle_words = needle.as_words(); - let full_words = needle_bits / WORD_BITS; - let rem_bits = needle_bits % WORD_BITS; - - // Full u64 words — needle is always word-aligned at index 0 so we - // compare needle_words[i] directly. - for i in 0..full_words { - let h = haystack.words.read_word_at(offset + i * WORD_BITS); - if h != needle_words[i] { - return false; - } - } - - // Last partial word (if any). - if rem_bits > 0 { - let mask = low_mask(rem_bits); - let h = haystack.words.read_word_at(offset + full_words * WORD_BITS); - if (h & mask) != (needle_words[full_words] & mask) { - return false; - } - } - - true -} - mod impls_for_find; mod impls_for_matches_at; mod impls_for_strip; diff --git a/src/bit_string/impls_for_matching/impls_for_find.rs b/src/bit_string/impls_for_matching/impls_for_find.rs index 870e65d..81a2828 100644 --- a/src/bit_string/impls_for_matching/impls_for_find.rs +++ b/src/bit_string/impls_for_matching/impls_for_find.rs @@ -21,7 +21,7 @@ impl BitString { self.bit_len, needle.as_words(), needle.bit_len, - &mut |pos| bits_equal_at(self, pos, needle), + &mut |pos| self.bits_equal_at(pos, needle), ) .is_some() } @@ -39,7 +39,7 @@ impl BitString { self.bit_len, needle.as_words(), needle.bit_len, - &mut |pos| bits_equal_at(self, pos, needle), + &mut |pos| self.bits_equal_at(pos, needle), ) .is_some() { @@ -51,7 +51,7 @@ impl BitString { self.bit_len, needle.as_words(), needle.bit_len, - &mut |pos| bits_equal_at(self, pos, needle), + &mut |pos| self.bits_equal_at(pos, needle), ) } @@ -68,7 +68,7 @@ impl BitString { self.bit_len, needle.as_words(), needle.bit_len, - &mut |pos| bits_equal_at(self, pos, needle), + &mut |pos| self.bits_equal_at(pos, needle), ) .is_some() { @@ -80,7 +80,7 @@ impl BitString { self.bit_len, needle.as_words(), needle.bit_len, - &mut |pos| bits_equal_at(self, pos, needle), + &mut |pos| self.bits_equal_at(pos, needle), ) } } diff --git a/src/bit_string/impls_for_matching/impls_for_find/funcs_for_contains_core/tests_for_backend_equivalence.rs b/src/bit_string/impls_for_matching/impls_for_find/funcs_for_contains_core/tests_for_backend_equivalence.rs index f61ddf1..420e551 100644 --- a/src/bit_string/impls_for_matching/impls_for_find/funcs_for_contains_core/tests_for_backend_equivalence.rs +++ b/src/bit_string/impls_for_matching/impls_for_find/funcs_for_contains_core/tests_for_backend_equivalence.rs @@ -7,8 +7,6 @@ use proptest::prelude::*; use crate::BitString; -use super::super::bits_equal_at; - fn config() -> ProptestConfig { ProptestConfig { cases: 512, @@ -33,7 +31,7 @@ proptest! { haystack.bit_len(), needle.as_words(), needle.bit_len(), - &mut |pos| bits_equal_at(&haystack, pos, &needle), + &mut |pos| haystack.bits_equal_at(pos, &needle), ); // Brute-force reference: find any match. @@ -77,7 +75,7 @@ proptest! { needle.as_words(), needle_len, &mut |pos| { - let ok = bits_equal_at(&haystack, pos, &needle); + let ok = haystack.bits_equal_at(pos, &needle); if ok { any_found = true; } ok }, diff --git a/src/bit_string/impls_for_matching/impls_for_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs b/src/bit_string/impls_for_matching/impls_for_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs index c6cd67c..96d6eaa 100644 --- a/src/bit_string/impls_for_matching/impls_for_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs +++ b/src/bit_string/impls_for_matching/impls_for_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs @@ -7,8 +7,6 @@ use proptest::prelude::*; use crate::BitString; -use super::super::bits_equal_at; - fn config() -> ProptestConfig { ProptestConfig { cases: 512, @@ -33,7 +31,7 @@ proptest! { haystack.bit_len(), needle.as_words(), needle.bit_len(), - &mut |pos| bits_equal_at(&haystack, pos, &needle), + &mut |pos| haystack.bits_equal_at(pos, &needle), ); // Brute-force: find rightmost match. @@ -72,7 +70,7 @@ proptest! { haystack.bit_len(), needle.as_words(), needle.bit_len(), - &mut |pos| bits_equal_at(&haystack, pos, &needle), + &mut |pos| haystack.bits_equal_at(pos, &needle), ); let max_pos = haystack.bit_len().saturating_sub(needle.bit_len()); diff --git a/src/bit_string/impls_for_matching/impls_for_matches_at.rs b/src/bit_string/impls_for_matching/impls_for_matches_at.rs index 28fee35..29a1830 100644 --- a/src/bit_string/impls_for_matching/impls_for_matches_at.rs +++ b/src/bit_string/impls_for_matching/impls_for_matches_at.rs @@ -10,10 +10,8 @@ mod funcs_for_starts_with_core; impl BitString { /// Returns `true` if `pattern` matches the bits starting at `index`. /// - /// For word-aligned offsets (`index % WORD_BITS == 0`) this delegates - /// to the same SIMD word-equality backend used by [`starts_with`]. - /// For unaligned offsets it uses the shifted-window SIMD backend - /// from [`ends_with`]. + /// Delegates to [`bits_equal_at`] which uses SIMD word-equality for + /// long patterns and scalar comparison for short ones. #[inline] pub fn matches_at(&self, index: usize, pattern: &Self) -> bool { if index > self.bit_len { @@ -24,40 +22,49 @@ impl BitString { return false; } - let pw = pattern.as_words(); - let full_words = pattern.bit_len / WORD_BITS; - - // For short patterns the scalar bits_equal_at is faster than - // SIMD dispatch overhead. - if full_words < SMALL_WORDS { - return bits_equal_at(self, index, pattern); - } - - let shift = index % WORD_BITS; - let base_word = index / WORD_BITS; - let sw: &[u64] = &self.words[base_word..]; + self.bits_equal_at(index, pattern) + } - if shift == 0 { - if !funcs_for_starts_with_core::starts_with_words(sw, pw, full_words) { - return false; + /// Compare `needle` bits against `self` starting at `offset`. + /// + /// For word-aligned offsets, the full words are compared via the + /// SIMD word-equality backend ([`starts_with_words`]). For unaligned + /// offsets, shifted 64-bit windows are computed via + /// [`ends_with_words`]. Short patterns fall back to scalar. + #[inline] + pub(crate) fn bits_equal_at(&self, offset: usize, needle: &Self) -> bool { + let needle_bits = needle.bit_len; + let needle_words = needle.as_words(); + let full_words = needle_bits / WORD_BITS; + + if full_words >= SMALL_WORDS { + let shift = offset % WORD_BITS; + let base_word = offset / WORD_BITS; + let sw: &[u64] = &self.words[base_word..]; + + if shift == 0 { + if !funcs_for_starts_with_core::starts_with_words(sw, needle_words, full_words) { + return false; + } + } else { + if !funcs_for_ends_with_core::ends_with_words(sw, needle_words, full_words, shift) { + return false; + } } } else { - if !funcs_for_ends_with_core::ends_with_words(sw, pw, full_words, shift) { - return false; + for i in 0..full_words { + let h = self.words.read_word_at(offset + i * WORD_BITS); + if h != needle_words[i] { + return false; + } } } - let rem = pattern.bit_len % WORD_BITS; - if rem > 0 { - let mask = low_mask(rem); - let h = if shift == 0 { - sw[full_words] - } else { - let w0 = sw[full_words]; - let w1 = sw.get(full_words + 1).copied().unwrap_or(0); - (w0 >> shift) | (w1 << (WORD_BITS - shift)) - }; - if (h & mask) != (pw[full_words] & mask) { + let rem_bits = needle_bits % WORD_BITS; + if rem_bits > 0 { + let mask = low_mask(rem_bits); + let h = self.words.read_word_at(offset + full_words * WORD_BITS); + if (h & mask) != (needle_words[full_words] & mask) { return false; } } diff --git a/src/bit_string/impls_for_matching/tests_for_bits_equal_at.rs b/src/bit_string/impls_for_matching/tests_for_bits_equal_at.rs index 449f9e2..ca0e2b8 100644 --- a/src/bit_string/impls_for_matching/tests_for_bits_equal_at.rs +++ b/src/bit_string/impls_for_matching/tests_for_bits_equal_at.rs @@ -1,14 +1,12 @@ use crate::BitString; -use super::bits_equal_at; - #[test] fn returns_true_when_needle_matches_at_offset() { let haystack = BitString::try_from("00110110").unwrap(); let needle = BitString::try_from("110").unwrap(); - assert!(bits_equal_at(&haystack, 2, &needle)); - assert!(bits_equal_at(&haystack, 5, &needle)); + assert!(haystack.bits_equal_at(2, &needle)); + assert!(haystack.bits_equal_at(5, &needle)); } #[test] @@ -16,9 +14,9 @@ fn returns_false_when_needle_differs_at_offset() { let haystack = BitString::try_from("00110110").unwrap(); let needle = BitString::try_from("110").unwrap(); - assert!(!bits_equal_at(&haystack, 0, &needle)); - assert!(!bits_equal_at(&haystack, 1, &needle)); - assert!(!bits_equal_at(&haystack, 3, &needle)); + assert!(!haystack.bits_equal_at(0, &needle)); + assert!(!haystack.bits_equal_at(1, &needle)); + assert!(!haystack.bits_equal_at(3, &needle)); } #[test] @@ -26,9 +24,9 @@ fn empty_needle_matches_at_valid_boundary_offsets() { let haystack = BitString::try_from("101001").unwrap(); let needle = BitString::new(); - assert!(bits_equal_at(&haystack, 0, &needle)); - assert!(bits_equal_at(&haystack, 3, &needle)); - assert!(bits_equal_at(&haystack, haystack.bit_len(), &needle)); + assert!(haystack.bits_equal_at(0, &needle)); + assert!(haystack.bits_equal_at(3, &needle)); + assert!(haystack.bits_equal_at(haystack.bit_len(), &needle)); } #[test] @@ -41,8 +39,8 @@ fn works_across_word_boundaries() { let needle = BitString::try_from("01110").unwrap(); - assert!(bits_equal_at(&haystack, 62, &needle)); - assert!(!bits_equal_at(&haystack, 61, &needle)); + assert!(haystack.bits_equal_at(62, &needle)); + assert!(!haystack.bits_equal_at(61, &needle)); } #[test] @@ -50,5 +48,5 @@ fn works_when_needle_reaches_haystack_end() { let haystack = BitString::try_from("101001").unwrap(); let needle = BitString::try_from("001").unwrap(); - assert!(bits_equal_at(&haystack, 3, &needle)); + assert!(haystack.bits_equal_at(3, &needle)); } From a28c2d77e29e54804aec365455fcfd688ae07ae5 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 21 Jun 2026 00:33:31 +0000 Subject: [PATCH 7/8] perf: add sub-word fast path to starts_with, ends_with, bits_equal_at When the pattern fits in a single u64 (<= 64 bits), skip the full_words / SMALL_WORDS / SIMD dispatch and use a single read_word_at + mask + compare. This eliminates branching and loop overhead for the common short-pattern case. starts_with now beats String in small scenarios (5.0ns vs 6.9ns). Co-authored-by: Claude Co-authored-by: DeepSeek AI --- .../impls_for_matches_at.rs | 73 ++++++++++++++----- 1 file changed, 56 insertions(+), 17 deletions(-) diff --git a/src/bit_string/impls_for_matching/impls_for_matches_at.rs b/src/bit_string/impls_for_matching/impls_for_matches_at.rs index 29a1830..8a2ad24 100644 --- a/src/bit_string/impls_for_matching/impls_for_matches_at.rs +++ b/src/bit_string/impls_for_matching/impls_for_matches_at.rs @@ -8,23 +8,6 @@ mod funcs_for_ends_with_core; mod funcs_for_starts_with_core; impl BitString { - /// Returns `true` if `pattern` matches the bits starting at `index`. - /// - /// Delegates to [`bits_equal_at`] which uses SIMD word-equality for - /// long patterns and scalar comparison for short ones. - #[inline] - pub fn matches_at(&self, index: usize, pattern: &Self) -> bool { - if index > self.bit_len { - return false; - } - - if pattern.bit_len > self.bit_len - index { - return false; - } - - self.bits_equal_at(index, pattern) - } - /// Compare `needle` bits against `self` starting at `offset`. /// /// For word-aligned offsets, the full words are compared via the @@ -35,6 +18,18 @@ impl BitString { pub(crate) fn bits_equal_at(&self, offset: usize, needle: &Self) -> bool { let needle_bits = needle.bit_len; let needle_words = needle.as_words(); + + // Sub-word fast path: the entire pattern fits in one u64 — + // single read + mask avoids all branching and loop overhead. + if needle_bits <= WORD_BITS { + let mask = low_mask(needle_bits); + if mask == 0 { + return true; // empty needle always matches + } + let h = self.words.read_word_at(offset); + return (h & mask) == (needle_words[0] & mask); + } + let full_words = needle_bits / WORD_BITS; if full_words >= SMALL_WORDS { @@ -72,6 +67,23 @@ impl BitString { true } + /// Returns `true` if `pattern` matches the bits starting at `index`. + /// + /// Delegates to [`bits_equal_at`] which uses SIMD word-equality for + /// long patterns and scalar comparison for short ones. + #[inline] + pub fn matches_at(&self, index: usize, pattern: &Self) -> bool { + if index > self.bit_len { + return false; + } + + if pattern.bit_len > self.bit_len - index { + return false; + } + + self.bits_equal_at(index, pattern) + } + /// Returns `true` if `prefix` is a prefix of `self`. /// /// This is equivalent to [`matches_at`]`(0, prefix)` but optimized for @@ -84,6 +96,16 @@ impl BitString { let pw = prefix.as_words(); let sw: &[u64] = &self.words; + + // Sub-word fast path: one u64 read + mask. + if prefix.bit_len <= WORD_BITS { + let mask = low_mask(prefix.bit_len); + if mask == 0 { + return true; // empty prefix always matches + } + return (sw[0] & mask) == (pw[0] & mask); + } + let full_words = prefix.bit_len / WORD_BITS; // Word-aligned at position 0 — use SIMD word equality. @@ -114,6 +136,23 @@ impl BitString { let base_word = start / WORD_BITS; let sw: &[u64] = &self.words[base_word..]; let pw = suffix.as_words(); + + // Sub-word fast path: one 64-bit window + mask. + if suffix.bit_len <= WORD_BITS { + let mask = low_mask(suffix.bit_len); + if mask == 0 { + return true; // empty suffix always matches + } + let h = if shift == 0 { + sw[0] + } else { + let w0 = sw[0]; + let w1 = sw.get(1).copied().unwrap_or(0); + (w0 >> shift) | (w1 << (WORD_BITS - shift)) + }; + return (h & mask) == (pw[0] & mask); + } + let full_words = suffix.bit_len / WORD_BITS; if !funcs_for_ends_with_core::ends_with_words(sw, pw, full_words, shift) { From 54293900b34fbaa92b82853851704706fbe26d98 Mon Sep 17 00:00:00 2001 From: juncheng Date: Sun, 21 Jun 2026 00:39:07 +0000 Subject: [PATCH 8/8] refactor: hoist zero-length checks to top of matching methods Move empty-pattern guards above all variable bindings in bits_equal_at, starts_with, and ends_with. This makes the early-return intent obvious and avoids computing needle_words, sw, shift, base_word, etc. just to find mask == 0. Co-authored-by: Claude Co-authored-by: DeepSeek AI --- .../impls_for_matches_at.rs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/bit_string/impls_for_matching/impls_for_matches_at.rs b/src/bit_string/impls_for_matching/impls_for_matches_at.rs index 8a2ad24..bb14c92 100644 --- a/src/bit_string/impls_for_matching/impls_for_matches_at.rs +++ b/src/bit_string/impls_for_matching/impls_for_matches_at.rs @@ -17,16 +17,16 @@ impl BitString { #[inline] pub(crate) fn bits_equal_at(&self, offset: usize, needle: &Self) -> bool { let needle_bits = needle.bit_len; + if needle_bits == 0 { + return true; + } let needle_words = needle.as_words(); // Sub-word fast path: the entire pattern fits in one u64 — // single read + mask avoids all branching and loop overhead. if needle_bits <= WORD_BITS { - let mask = low_mask(needle_bits); - if mask == 0 { - return true; // empty needle always matches - } let h = self.words.read_word_at(offset); + let mask = low_mask(needle_bits); return (h & mask) == (needle_words[0] & mask); } @@ -90,6 +90,9 @@ impl BitString { /// the word-aligned position-0 case. #[inline] pub fn starts_with(&self, prefix: &Self) -> bool { + if prefix.bit_len == 0 { + return true; + } if prefix.bit_len > self.bit_len { return false; } @@ -100,9 +103,6 @@ impl BitString { // Sub-word fast path: one u64 read + mask. if prefix.bit_len <= WORD_BITS { let mask = low_mask(prefix.bit_len); - if mask == 0 { - return true; // empty prefix always matches - } return (sw[0] & mask) == (pw[0] & mask); } @@ -127,6 +127,9 @@ impl BitString { /// Returns `true` if `suffix` is a suffix of `self`. #[inline] pub fn ends_with(&self, suffix: &Self) -> bool { + if suffix.bit_len == 0 { + return true; + } if suffix.bit_len > self.bit_len { return false; } @@ -139,10 +142,6 @@ impl BitString { // Sub-word fast path: one 64-bit window + mask. if suffix.bit_len <= WORD_BITS { - let mask = low_mask(suffix.bit_len); - if mask == 0 { - return true; // empty suffix always matches - } let h = if shift == 0 { sw[0] } else { @@ -150,6 +149,7 @@ impl BitString { let w1 = sw.get(1).copied().unwrap_or(0); (w0 >> shift) | (w1 << (WORD_BITS - shift)) }; + let mask = low_mask(suffix.bit_len); return (h & mask) == (pw[0] & mask); }