Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

perf(kernel): keep 16-byte kv loads in dsv4 sparse attention and unroll gated pool (#633)

Xiaoze Fan1c3aa0aa4cd8
diff --git a/python/freetoken/kernel/triton/dsv4/compress.py b/python/freetoken/kernel/triton/dsv4/compress.py
index 77bacd42..3ab89ff0 100644
--- a/python/freetoken/kernel/triton/dsv4/compress.py
+++ b/python/freetoken/kernel/triton/dsv4/compress.py
@@ -31,12 +31,13 @@ def _gated_pool_kernel(
kv_base = kv_ptr + b * stride_kb + offs * stride_kd
sc_base = score_ptr + b * stride_sb + offs * stride_sd
maxv = tl.full((BLOCK_D,), float("-inf"), tl.float32)
- for r in range(R):
+ # Explicit unroll: ptxas does not unroll the plain loop Triton 3.8 emits, which serializes these latency-bound loads.
+ for r in tl.range(R, loop_unroll_factor=2):
s = tl.load(sc_base + r * stride_sr, mask=mask, other=float("-inf")).to(tl.float32)
maxv = tl.maximum(maxv, s)
denom = tl.zeros((BLOCK_D,), tl.float32)
acc = tl.zeros((BLOCK_D,), tl.float32)
- for r in range(R):
+ for r in tl.range(R, loop_unroll_factor=2):
s = tl.load(sc_base + r * stride_sr, mask=mask, other=float("-inf")).to(tl.float32)
w = tl.exp(s - maxv)
denom += w
diff --git a/python/freetoken/kernel/triton/dsv4/sparse_attn.py b/python/freetoken/kernel/triton/dsv4/sparse_attn.py
index c22f891e..639c3e5e 100644
--- a/python/freetoken/kernel/triton/dsv4/sparse_attn.py
+++ b/python/freetoken/kernel/triton/dsv4/sparse_attn.py
@@ -99,7 +99,8 @@ def _sparse_attn_paged_kernel(
# multiple KV tiles in shared memory. Result is bit-identical to the two-pool form -- each
# column still reads from the same pool/slot.
is_win = offs_t < N_WINDOW
- base = tl.where(is_win, win_ptr, cmp_ptr) # [BLOCK_T] per-column pool base pointer
+ # The select drops the pools' 16-byte alignment (asserted in the wrapper); without this hint Triton 3.8 gathers in 2-byte loads.
+ base = tl.multiple_of(tl.where(is_win, win_ptr, cmp_ptr), 16) # [BLOCK_T] per-column pool base pointer
kv_ptrs = base[:, None] + idxs[:, None] * stride_wn + offs_d[None, :] * stride_wd
kv = tl.load(kv_ptrs, mask=valid[:, None], other=0.0).to(tl.float32) # [BLOCK_T, D]
@@ -182,7 +183,7 @@ def _sparse_attn_paged_splitk_kernel(
idxs = tl.load(idx_base + offs_t * stride_it, mask=t_mask, other=-1)
valid = idxs >= 0
is_win = offs_t < N_WINDOW
- base = tl.where(is_win, win_ptr, cmp_ptr)
+ base = tl.multiple_of(tl.where(is_win, win_ptr, cmp_ptr), 16)
kv_ptrs = base[:, None] + idxs[:, None] * stride_wn + offs_d[None, :] * stride_wd
kv = tl.load(kv_ptrs, mask=valid[:, None], other=0.0).to(tl.float32)
@@ -314,6 +315,7 @@ def sparse_attn_paged(
# The kernel selects one pool base per column and gathers with a single stride pair, so both
# pools must share strides. Guaranteed here by .contiguous() on their [*, D] shape.
assert window_pool.stride() == cmp_pool.stride(), (window_pool.stride(), cmp_pool.stride())
+ assert window_pool.data_ptr() % 16 == 0 and cmp_pool.data_ptr() % 16 == 0, (window_pool.data_ptr(), cmp_pool.data_ptr())
has_counts = cmp_counts is not None
if has_counts: