FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(kernel): avoid nan in triton sampling softmax when a chunk starts with -inf (#654)
Chen Chengee9789ff6cdc
diff --git a/python/freetoken/kernel/triton/sampling.py b/python/freetoken/kernel/triton/sampling.pyindex a9707144..38bc2533 100644--- a/python/freetoken/kernel/triton/sampling.py+++ b/python/freetoken/kernel/triton/sampling.py@@ -92,7 +92,9 @@ def _sm_partial(x = tl.load(logits_ptr + base + offs, mask=mask, other=-float("inf")).to(tl.float32) * inv_tblk_max = tl.max(x, 0)new_m = tl.maximum(m, blk_max)- d = d * tl.exp(m - new_m) + tl.sum(tl.exp(x - new_m), 0)+ # While every logit so far is -inf, shift by 0: exp(-inf - -inf) would be NaN.+ shift = tl.where(new_m == -float("inf"), 0.0, new_m)+ d = d * tl.exp(m - shift) + tl.sum(tl.exp(x - shift), 0)m = new_mtl.store(pm_ptr + pid, m)tl.store(pl_ptr + pid, d)diff --git a/tests/kernels/test_triton_fallbacks.py b/tests/kernels/test_triton_fallbacks.pyindex 45234e66..2306a400 100644--- a/tests/kernels/test_triton_fallbacks.py+++ b/tests/kernels/test_triton_fallbacks.py@@ -55,3 +55,22 @@ def test_norm_fallback_matches_torch(dtype, kind):rtol, atol = (1e-5, 1e-6) if dtype == torch.float32 else (1e-2, 1e-3)torch.testing.assert_close(norm.forward(x), expected.to(dtype), rtol=rtol, atol=atol)++[email protected]("mask", ["prefix", "allowed_ids"])+def test_sampling_softmax_fallback_with_masked_logits(mask):+ from freetoken.kernel.triton.sampling import softmax++ torch.manual_seed(0)+ logits = torch.randn((3, 151936), device="cuda") * 4+ if mask == "prefix":+ # Long enough that some column chunk starts with an all -inf block.+ logits[:2, :16384] = float("-inf")+ else:+ allowed = torch.tensor([5, 70000, 151000], device="cuda")+ logits[:2] = torch.full_like(logits[:2], float("-inf")).index_copy(+ 1, allowed, logits[:2, allowed])+ temperature = torch.tensor([1.0, 0.7, 1.3], device="cuda")+ expected = torch.softmax(logits / temperature[:, None], dim=-1)++ torch.testing.assert_close(softmax(logits, temperature), expected, rtol=1e-5, atol=1e-7)