FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(kernel): avoid row-wise _scaled_mm stall on sm_89 with torch<2.12 (#243)
Berni McCoy58f4b9ec0e16
diff --git a/python/freetoken/kernel/triton/fp8_pertensor_linear.py b/python/freetoken/kernel/triton/fp8_pertensor_linear.pyindex 28a54c94..d1d4de5b 100644--- a/python/freetoken/kernel/triton/fp8_pertensor_linear.py+++ b/python/freetoken/kernel/triton/fp8_pertensor_linear.py@@ -19,7 +19,9 @@ accurate than the previous ``weight.to(bf16) * scale`` materialization, which itfrom __future__ import annotations+import functoolsimport os+import reimport torchimport triton@@ -42,6 +44,64 @@ _TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float_USE_REF = os.environ.get("FREETOKEN_DEBUG_FP8_REF") == "1"+# Row-wise _scaled_mm on sm_89 with torch < 2.12 launches its CUTLASS stream-K kernel off the+# current stream (pytorch/pytorch#177651, fixed by pytorch/pytorch@252bb4a; #182/#72/#220), and+# some builds (Windows) have no row-wise kernel (#227). Fallback: one tensor-wise GEMM per part.+def _torch_version() -> tuple[int, int]:+ m = re.match(r"(\d+)\.(\d+)", torch.__version__)+ return (int(m.group(1)), int(m.group(2))) if m else (0, 0)+++def rowwise_scaled_mm_ok() -> bool:+ """Whether row-wise ``torch._scaled_mm`` may be issued from a side stream on this GPU.+ Decided once per process, at load (never under graph capture). ``FREETOKEN_FP8_ROWWISE_MM=0/1``+ forces the answer."""+ forced = os.environ.get("FREETOKEN_FP8_ROWWISE_MM")+ if forced in ("0", "1"):+ return forced == "1"+ if not torch.cuda.is_available():+ return True+ from freetoken.gpu_select import assigned_visible_gpu++ idx = assigned_visible_gpu()+ dev = torch.device("cuda", torch.cuda.current_device() if idx is None else idx)+ if torch.cuda.get_device_capability(dev) == (8, 9) and _torch_version() < (2, 12):+ return False+ # Probe on the default stream (safe even where the launch ignores the current stream); a+ # build without the row-wise kernel raises here instead of at the first forward.+ try:+ with torch.cuda.device(dev), torch.cuda.stream(torch.cuda.default_stream(dev)):+ a = torch.zeros(16, 32, dtype=FP8, device=dev)+ b = torch.zeros(32, 32, dtype=FP8, device=dev)+ torch._scaled_mm(+ a, b.t(), scale_a=torch.ones(16, 1, device=dev),+ scale_b=torch.ones(1, 32, device=dev), out_dtype=torch.bfloat16,+ )+ torch.cuda.synchronize(dev)+ except RuntimeError:+ return False+ return True+++def weight_scale_segments(weight_scale: torch.Tensor) -> list[tuple[int, int]]:+ """``[start, end)`` row ranges over which ``weight_scale`` is constant (the fused parts).+ Syncs; call at load."""+ s = weight_scale.detach().reshape(-1).float().cpu()+ change = (torch.nonzero(s[1:] != s[:-1]).flatten() + 1).tolist()+ bounds = [0, *change, s.numel()]+ return list(zip(bounds[:-1], bounds[1:]))+++_MAX_SEGMENTS = 8 # q/k/v = 3, GDN qkv|z = 2; a genuine per-row scale stays W8A16 instead+++def _segments_w8a8_ok(segments: list[tuple[int, int]]) -> bool:+ """cuBLASLt needs 16-row aligned fp8 operands; more parts than a fused projection has+ means a genuine per-row scale."""+ return 0 < len(segments) <= _MAX_SEGMENTS and all((e - s) % 16 == 0 for s, e in segments)++# ======================================================================================# Decode (M==1) split-K GEMV: raw fp8 x bf16 reduction in fp32, per-row scale at reduce.# ======================================================================================@@ -222,6 +282,7 @@ def _static_quant(a: torch.Tensor, input_scale: torch.Tensor) -> torch.Tensor:def _scaled_mm(a: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor,input_scale: torch.Tensor, uniform_scale: bool, out_dtype: torch.dtype,+ scale_segments: list[tuple[int, int]] | None = None,) -> torch.Tensor:"""``a @ (weight_fp8 * weight_scale)^T`` as a W8A8 cuBLASLt GEMM.@@ -233,14 +294,26 @@ def _scaled_mm(tensor-wise path. A fused projection, whose ``weight_scale`` is piecewise-constantbecause each part carries its own scalar, takes the row-wise path -- that keeps everypart's scale exact, where vLLM/SGLang instead requantize the parts onto a shared maximum- and eat the precision loss. Row-wise costs ~4% here (5.56 ms vs 5.39 ms per step)."""+ and eat the precision loss. Row-wise costs ~4% here (5.56 ms vs 5.39 ms per step).++ Where row-wise is unsafe (:func:`rowwise_scaled_mm_ok`) ``scale_segments`` is passed and+ each part runs its own tensor-wise GEMM over ``weight[s:e]`` (still stride-only), outputs+ concatenated: the same W8A8 scheme, not bit-identical (accumulation order differs)."""qa = _static_quant(a, input_scale)wt = weight.t() # [N, K] row-major -> [K, N] column-major, stride-only+ sa = input_scale.reshape(())if uniform_scale:return torch._scaled_mm(- qa, wt, scale_a=input_scale.reshape(()), scale_b=weight_scale[0].reshape(()),- out_dtype=out_dtype,+ qa, wt, scale_a=sa, scale_b=weight_scale[0].reshape(()), out_dtype=out_dtype,)+ if scale_segments is not None:+ return torch.cat([+ torch._scaled_mm(+ qa, weight[s:e].t(), scale_a=sa, scale_b=weight_scale[s].reshape(()),+ out_dtype=out_dtype,+ )+ for s, e in scale_segments+ ], dim=1)return torch._scaled_mm(qa, wt,scale_a=input_scale.reshape(1, 1).expand(a.shape[0], 1).contiguous(),@@ -254,9 +327,11 @@ def fp8_pertensor_linear(bias: torch.Tensor | None = None,input_scale: torch.Tensor | None = None,uniform_scale: bool = False,+ scale_segments: list[tuple[int, int]] | None = None,) -> torch.Tensor:"""``y = x @ (weight_fp8 * weight_scale)^T``. ``weight`` [N, K] fp8-e4m3, ``weight_scale``- [N] fp32 (per output row).+ [N] fp32 (per output row). ``scale_segments``: the fused parts' row ranges, precomputed at+ load by the layer; derived here (with a sync) when omitted and needed.Whether the activation is quantized is a property of the *deployment*, never of the batch:with ``input_scale`` on sm_89+ every M runs W8A8, otherwise every M runs W8A16 (split-K@@ -266,12 +341,19 @@ def fp8_pertensor_linear(SGLang likewise run one scheme across all M on any GPU with FP8 tensor cores."""*lead, K = x.shapeN = weight.shape[0]+ w8a8 = input_scale is not None and e4m3_native()+ segments = None+ if w8a8 and not uniform_scale and not rowwise_scaled_mm_ok():+ segments = scale_segments if scale_segments is not None else weight_scale_segments(weight_scale)+ if not _segments_w8a8_ok(segments):+ w8a8 = False # W8A16 below is exact for any per-row scale and never calls _scaled_mmif _USE_REF: # numeric-reference fallback (debug / A-B)w = weight.to(x.dtype) * weight_scale.to(x.dtype)[:, None]out = (x.reshape(-1, K) @ w.t()).reshape(*lead, N)- elif input_scale is not None and e4m3_native():+ elif w8a8:out = _scaled_mm(x.reshape(-1, K), weight, weight_scale, input_scale, uniform_scale, x.dtype,+ scale_segments=segments,).reshape(*lead, N)elif x.numel() // K == 1:out = _gemv(x.reshape(K), e4m3_kernel_view(weight), weight_scale, x.dtype).reshape(*lead, N)@@ -306,6 +388,7 @@ class Fp8PerTensorLinear(BaseOP):# reflective state_dict/load_state_dict skip it entirely on checkpoints without one.self.input_scale: torch.Tensor | None = Noneself._uniform_scale = False+ self._scale_segments: list[tuple[int, int]] | None = Nonedef load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None:# Taken out before BaseOP's reflective pass (so it is not an "unexpected key") and@@ -318,11 +401,15 @@ class Fp8PerTensorLinear(BaseOP):# only piecewise-constant, so decide once here rather than syncing on every forward.scale = self.weight_scaleself._uniform_scale = bool((scale == scale[0]).all().item())+ # Segments for the per-part path; decide row-wise safety now, not under graph capture.+ self._scale_segments = None if self._uniform_scale else weight_scale_segments(scale)+ if self.input_scale is not None and not self._uniform_scale:+ rowwise_scaled_mm_ok()def forward(self, x: torch.Tensor) -> torch.Tensor:return fp8_pertensor_linear(x, self.weight, self.weight_scale, self.bias,- self.input_scale, self._uniform_scale,+ self.input_scale, self._uniform_scale, scale_segments=self._scale_segments,)@@ -341,4 +428,6 @@ __all__ = ["Fp8PerTensorLinear","Fp8PerTensorColMerged","fp8_pertensor_linear",+ "rowwise_scaled_mm_ok",+ "weight_scale_segments",]diff --git a/tests/kernels/test_fp8_pertensor_linear.py b/tests/kernels/test_fp8_pertensor_linear.pyindex 6b158f52..441da408 100644--- a/tests/kernels/test_fp8_pertensor_linear.py+++ b/tests/kernels/test_fp8_pertensor_linear.py@@ -9,6 +9,10 @@ to fp8 and is held to a reference that applies the same quantization.from __future__ import annotations+import subprocess+import sys+import textwrap+import pytestimport torch@@ -125,3 +129,71 @@ def test_layer_load_marks_uniform_scale_and_optional_input_scale():# a reload must not trip over the input_scale it kept from the first loadsingle.load_state_dict({"weight": w8, "weight_scale": flat})assert single.input_scale is None++[email protected](not e4m3_native(), reason="torch._scaled_mm needs sm_89+")[email protected]("M", [1, 4, 64, 300])+def test_per_part_path_matches_rowwise(M: int, monkeypatch):+ """Where row-wise ``_scaled_mm`` is unsafe a fused projection runs one tensor-wise GEMM per+ part instead. Same scheme, so the two paths agree up to accumulation order (~7e-4)."""+ import freetoken.kernel.triton.fp8_pertensor_linear as mod++ K, part_rows = 2048, [1024, 256, 256]+ w8, scale = _quant_parts(part_rows, K, seed=M)+ x = torch.randn(M, K, device=DEV, dtype=torch.bfloat16)+ input_scale = (x.abs().max().float() / 448.0).reshape(())++ monkeypatch.setattr(mod, "rowwise_scaled_mm_ok", lambda: True)+ y_row = mod.fp8_pertensor_linear(x, w8, scale, None, input_scale, False)+ monkeypatch.setattr(mod, "rowwise_scaled_mm_ok", lambda: False)+ y_part = mod.fp8_pertensor_linear(x, w8, scale, None, input_scale, False)+ rel = ((y_part.float() - y_row.float()).norm() / y_row.float().norm()).item()+ assert rel < 2e-3, rel++[email protected](not e4m3_native(), reason="torch._scaled_mm needs sm_89+")+def test_fused_layer_forward_on_a_side_stream_completes():+ """Regression for #182 / #72 / #220: on sm_89 with torch < 2.12 a fused FP8 projection's+ row-wise ``_scaled_mm`` issued from a non-default stream stalls the GPU (PyTorch's+ CUTLASS row-wise kernel ignored the current stream; fixed upstream in 2.12). The layer+ must take a path that completes on every supported build. Runs in a subprocess so a+ stall fails the test instead of hanging the session."""+ script = textwrap.dedent("""+ import os, time, torch+ from freetoken.kernel.triton.fp8_pertensor_linear import FP8, Fp8PerTensorColMerged++ torch.manual_seed(0)+ K, parts = 2048, [8192, 512, 512] # a prefill-sized fused qkv+ w8 = (torch.randn(sum(parts), K, device="cuda") * 8).clamp(-448, 448).to(FP8)+ scale = torch.cat([torch.full((p,), 0.01 * (i + 1), device="cuda")+ for i, p in enumerate(parts)])+ layer = Fp8PerTensorColMerged(K, parts)+ layer.load_state_dict({"weight": w8, "weight_scale": scale,+ "input_scale": torch.tensor(0.02, device="cuda")})+ x = torch.randn(2010, K, device="cuda", dtype=torch.bfloat16) # #182 shape+ torch.cuda.synchronize()++ stream = torch.cuda.Stream()+ events = []+ with torch.cuda.stream(stream):+ for _ in range(128):+ layer.forward(x)+ ev = torch.cuda.Event()+ ev.record(stream)+ events.append(ev)+ deadline = time.monotonic() + 30+ done = 0+ while time.monotonic() < deadline:+ while done < len(events) and events[done].query():+ done += 1+ if done == len(events):+ print("completed", done, flush=True)+ os._exit(0)+ time.sleep(0.05)+ print("stalled at", done, "of", len(events), flush=True)+ os._exit(124) # a normal exit would wait on the stuck kernel+ """)+ proc = subprocess.run(+ [sys.executable, "-c", script], capture_output=True, text=True, timeout=300,+ )+ assert proc.returncode == 0, f"rc={proc.returncode}\n{proc.stdout}\n{proc.stderr[-2000:]}"