FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
perf(kernel): keep 16-byte kv loads in dsv4 sparse attention and unroll gated pool (#633)
Xiaoze Fan1c3aa0aa4cd8
diff --git a/python/freetoken/kernel/triton/dsv4/compress.py b/python/freetoken/kernel/triton/dsv4/compress.pyindex 77bacd42..3ab89ff0 100644--- a/python/freetoken/kernel/triton/dsv4/compress.py+++ b/python/freetoken/kernel/triton/dsv4/compress.py@@ -31,12 +31,13 @@ def _gated_pool_kernel(kv_base = kv_ptr + b * stride_kb + offs * stride_kdsc_base = score_ptr + b * stride_sb + offs * stride_sdmaxv = tl.full((BLOCK_D,), float("-inf"), tl.float32)- for r in range(R):+ # Explicit unroll: ptxas does not unroll the plain loop Triton 3.8 emits, which serializes these latency-bound loads.+ for r in tl.range(R, loop_unroll_factor=2):s = tl.load(sc_base + r * stride_sr, mask=mask, other=float("-inf")).to(tl.float32)maxv = tl.maximum(maxv, s)denom = tl.zeros((BLOCK_D,), tl.float32)acc = tl.zeros((BLOCK_D,), tl.float32)- for r in range(R):+ for r in tl.range(R, loop_unroll_factor=2):s = tl.load(sc_base + r * stride_sr, mask=mask, other=float("-inf")).to(tl.float32)w = tl.exp(s - maxv)denom += wdiff --git a/python/freetoken/kernel/triton/dsv4/sparse_attn.py b/python/freetoken/kernel/triton/dsv4/sparse_attn.pyindex c22f891e..639c3e5e 100644--- a/python/freetoken/kernel/triton/dsv4/sparse_attn.py+++ b/python/freetoken/kernel/triton/dsv4/sparse_attn.py@@ -99,7 +99,8 @@ def _sparse_attn_paged_kernel(# multiple KV tiles in shared memory. Result is bit-identical to the two-pool form -- each# column still reads from the same pool/slot.is_win = offs_t < N_WINDOW- base = tl.where(is_win, win_ptr, cmp_ptr) # [BLOCK_T] per-column pool base pointer+ # The select drops the pools' 16-byte alignment (asserted in the wrapper); without this hint Triton 3.8 gathers in 2-byte loads.+ base = tl.multiple_of(tl.where(is_win, win_ptr, cmp_ptr), 16) # [BLOCK_T] per-column pool base pointerkv_ptrs = base[:, None] + idxs[:, None] * stride_wn + offs_d[None, :] * stride_wdkv = tl.load(kv_ptrs, mask=valid[:, None], other=0.0).to(tl.float32) # [BLOCK_T, D]@@ -182,7 +183,7 @@ def _sparse_attn_paged_splitk_kernel(idxs = tl.load(idx_base + offs_t * stride_it, mask=t_mask, other=-1)valid = idxs >= 0is_win = offs_t < N_WINDOW- base = tl.where(is_win, win_ptr, cmp_ptr)+ base = tl.multiple_of(tl.where(is_win, win_ptr, cmp_ptr), 16)kv_ptrs = base[:, None] + idxs[:, None] * stride_wn + offs_d[None, :] * stride_wdkv = tl.load(kv_ptrs, mask=valid[:, None], other=0.0).to(tl.float32)@@ -314,6 +315,7 @@ def sparse_attn_paged(# The kernel selects one pool base per column and gathers with a single stride pair, so both# pools must share strides. Guaranteed here by .contiguous() on their [*, D] shape.assert window_pool.stride() == cmp_pool.stride(), (window_pool.stride(), cmp_pool.stride())+ assert window_pool.data_ptr() % 16 == 0 and cmp_pool.data_ptr() % 16 == 0, (window_pool.data_ptr(), cmp_pool.data_ptr())has_counts = cmp_counts is not Noneif has_counts: