Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(kernel): avoid nan in triton sampling softmax when a chunk starts with -inf (#654)

Chen Chengee9789ff6cdc
diff --git a/python/freetoken/kernel/triton/sampling.py b/python/freetoken/kernel/triton/sampling.py
index a9707144..38bc2533 100644
--- a/python/freetoken/kernel/triton/sampling.py
+++ b/python/freetoken/kernel/triton/sampling.py
@@ -92,7 +92,9 @@ def _sm_partial(
x = tl.load(logits_ptr + base + offs, mask=mask, other=-float("inf")).to(tl.float32) * inv_t
blk_max = tl.max(x, 0)
new_m = tl.maximum(m, blk_max)
- d = d * tl.exp(m - new_m) + tl.sum(tl.exp(x - new_m), 0)
+ # While every logit so far is -inf, shift by 0: exp(-inf - -inf) would be NaN.
+ shift = tl.where(new_m == -float("inf"), 0.0, new_m)
+ d = d * tl.exp(m - shift) + tl.sum(tl.exp(x - shift), 0)
m = new_m
tl.store(pm_ptr + pid, m)
tl.store(pl_ptr + pid, d)
diff --git a/tests/kernels/test_triton_fallbacks.py b/tests/kernels/test_triton_fallbacks.py
index 45234e66..2306a400 100644
--- a/tests/kernels/test_triton_fallbacks.py
+++ b/tests/kernels/test_triton_fallbacks.py
@@ -55,3 +55,22 @@ def test_norm_fallback_matches_torch(dtype, kind):
rtol, atol = (1e-5, 1e-6) if dtype == torch.float32 else (1e-2, 1e-3)
torch.testing.assert_close(norm.forward(x), expected.to(dtype), rtol=rtol, atol=atol)
+
+
[email protected]("mask", ["prefix", "allowed_ids"])
+def test_sampling_softmax_fallback_with_masked_logits(mask):
+ from freetoken.kernel.triton.sampling import softmax
+
+ torch.manual_seed(0)
+ logits = torch.randn((3, 151936), device="cuda") * 4
+ if mask == "prefix":
+ # Long enough that some column chunk starts with an all -inf block.
+ logits[:2, :16384] = float("-inf")
+ else:
+ allowed = torch.tensor([5, 70000, 151000], device="cuda")
+ logits[:2] = torch.full_like(logits[:2], float("-inf")).index_copy(
+ 1, allowed, logits[:2, allowed])
+ temperature = torch.tensor([1.0, 0.7, 1.3], device="cuda")
+ expected = torch.softmax(logits / temperature[:, None], dim=-1)
+
+ torch.testing.assert_close(softmax(logits, temperature), expected, rtol=1e-5, atol=1e-7)