Skip to content

perf: hoist tile/clamp out of highp sampler inner loop #177

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bhark wants to merge 1 commit into
base: main
Choose a base branch
from
Open

perf: hoist tile/clamp out of highp sampler inner loop #177

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 30 additions & 61 deletions src/pipeline/highp.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -747,7 +747,7 @@ fn bilinear(p: &mut Pipeline) {
let wx = [one - fx, fx];
let wy = [one - fy, fy];

sampler_2x2(p.pixmap_src, &p.ctx.sampler, x, y, &wx, &wy, &mut p.r, &mut p.g, &mut p.b, &mut p.a);
sampler::<2>(p.pixmap_src, &p.ctx.sampler, x, y, &wx, &wy, &mut p.r, &mut p.g, &mut p.b, &mut p.a);

p.next_stage();
}
Expand All @@ -761,7 +761,7 @@ fn bicubic(p: &mut Pipeline) {
let wx = [bicubic_far(one - fx), bicubic_near(one - fx), bicubic_near(fx), bicubic_far(fx)];
let wy = [bicubic_far(one - fy), bicubic_near(one - fy), bicubic_near(fy), bicubic_far(fy)];

sampler_4x4(p.pixmap_src, &p.ctx.sampler, x, y, &wx, &wy, &mut p.r, &mut p.g, &mut p.b, &mut p.a);
sampler::<4>(p.pixmap_src, &p.ctx.sampler, x, y, &wx, &wy, &mut p.r, &mut p.g, &mut p.b, &mut p.a);

p.next_stage();
}
Expand Down Expand Up @@ -795,11 +795,11 @@ fn bicubic_far(t: f32x8) -> f32x8 {
}

#[inline(always)]
fn sampler_2x2(
fn sampler<const N: usize>(
pixmap: PixmapRef,
ctx: &super::SamplerCtx,
cx: f32x8, cy: f32x8,
wx: &[f32x8; 2], wy: &[f32x8; 2],
wx: &[f32x8; N], wy: &[f32x8; N],
r: &mut f32x8, g: &mut f32x8, b: &mut f32x8, a: &mut f32x8,
) {
*r = f32x8::default();
Expand All @@ -808,80 +808,49 @@ fn sampler_2x2(
*a = f32x8::default();

let one = f32x8::splat(1.0);
let start = -0.5;
let mut y = cy + f32x8::splat(start);
for j in 0..2 {
let mut x = cx + f32x8::splat(start);
for i in 0..2 {
let mut rr = f32x8::default();
let mut gg = f32x8::default();
let mut bb = f32x8::default();
let mut aa = f32x8::default();
sample(pixmap, ctx, x,y, &mut rr, &mut gg, &mut bb, &mut aa);

let w = wx[i] * wy[j];
*r = mad(w, rr, *r);
*g = mad(w, gg, *g);
*b = mad(w, bb, *b);
*a = mad(w, aa, *a);

x += one;
}

let start = f32x8::splat(-((N - 1) as f32) / 2.0);
let w = pixmap.width() as f32;
let h = pixmap.height() as f32;
let w_lim = f32x8::splat(ulp_sub(w));
let h_lim = f32x8::splat(ulp_sub(h));
let zero = f32x8::default();
let stride = i32x8::splat(pixmap.width() as i32);

// per-axis hoist: N tile_x + N tile_y instead of N*N + N*N. tile() is opaque
// to llvm across SpreadMode, so it never gets cse'd from the inner loop.
let mut tx_int = [i32x8::default(); N];
let mut ty_off = [i32x8::default(); N];
let mut x = cx + start;
let mut y = cy + start;
for i in 0..N {
let t = tile(x, ctx.spread_mode, w, ctx.inv_width);
tx_int[i] = t.max(zero).min(w_lim).trunc_int();
x += one;
}
for j in 0..N {
let t = tile(y, ctx.spread_mode, h, ctx.inv_height);
ty_off[j] = t.max(zero).min(h_lim).trunc_int() * stride;
y += one;
}
}

#[inline(always)]
fn sampler_4x4(
pixmap: PixmapRef,
ctx: &super::SamplerCtx,
cx: f32x8, cy: f32x8,
wx: &[f32x8; 4], wy: &[f32x8; 4],
r: &mut f32x8, g: &mut f32x8, b: &mut f32x8, a: &mut f32x8,
) {
*r = f32x8::default();
*g = f32x8::default();
*b = f32x8::default();
*a = f32x8::default();

let one = f32x8::splat(1.0);
let start = -1.5;
let mut y = cy + f32x8::splat(start);
for j in 0..4 {
let mut x = cx + f32x8::splat(start);
for i in 0..4 {
for j in 0..N {
for i in 0..N {
let ix = (ty_off[j] + tx_int[i]).to_u32x8_bitcast();
let mut rr = f32x8::default();
let mut gg = f32x8::default();
let mut bb = f32x8::default();
let mut aa = f32x8::default();
sample(pixmap, ctx, x,y, &mut rr, &mut gg, &mut bb, &mut aa);
load_8888(&pixmap.gather(ix), &mut rr, &mut gg, &mut bb, &mut aa);

let w = wx[i] * wy[j];
*r = mad(w, rr, *r);
*g = mad(w, gg, *g);
*b = mad(w, bb, *b);
*a = mad(w, aa, *a);

x += one;
}

y += one;
}
}

#[inline(always)]
fn sample(
pixmap: PixmapRef, ctx: &super::SamplerCtx, mut x: f32x8, mut y: f32x8,
r: &mut f32x8, g: &mut f32x8, b: &mut f32x8, a: &mut f32x8,
) {
x = tile(x, ctx.spread_mode, pixmap.width() as f32, ctx.inv_width);
y = tile(y, ctx.spread_mode, pixmap.height() as f32, ctx.inv_height);

let ix = gather_ix(pixmap, x, y);
load_8888(&pixmap.gather(ix), r, g, b, a);
}

#[inline(always)]
fn tile(v: f32x8, mode: SpreadMode, limit: f32, inv_limit: f32) -> f32x8 {
match mode {
Expand Down
Loading