Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 TMA block GEMM: cut per-call descriptor host cost (#15372)
Daniel Hana9bcb6cb15f7
diff --git a/tests/test_fp8_tma_eager_host_cost.py b/tests/test_fp8_tma_eager_host_cost.pynew file mode 100644index 00000000000..7c333a003cf--- /dev/null+++ b/tests/test_fp8_tma_eager_host_cost.py@@ -0,0 +1,199 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""TMA block FP8 GEMM host cost: descriptors are built per call from the live tensors without+re-validation, and routing is unchanged in eager and under graph capture (no GPU needed)."""++import pytest+import torch++from unsloth.kernels import fp8 as F8++if F8._TensorDescriptor is None:+ pytest.skip("this Triton has no TensorDescriptor", allow_module_level = True)+++class _Fake:+ def __init__(+ self,+ shape,+ dtype,+ ptr = 1 << 20,+ ):+ self.shape = torch.Size(shape)+ self.dtype = dtype+ self.device = torch.device("cuda", 0)+ self._ptr = ptr++ def data_ptr(self):+ return self._ptr++ def numel(self):+ return self.shape.numel()++ def element_size(self):+ return torch.empty((), dtype = self.dtype).element_size()+++def tma_device(monkeypatch):+ calls = []+ capturing = [False]+ monkeypatch.setattr(F8, "_FP8_TMA_GEMM", True)+ monkeypatch.setattr(F8, "_fp8_tma_gemm_arch_at", lambda index: True)+ monkeypatch.setattr(F8, "_FP8_TMA_SELF_CHECKED", {})+ monkeypatch.setattr(F8, "_fp8_tma_gemm_self_check", lambda index: calls.append(index) or True)+ monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: capturing[0])+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: False)+ return calls, capturing+++def _ok(+ M,+ N = 4096,+ K = 4096,+ dtype = torch.bfloat16,+ batch = None,+):+ lead = (M,) if batch is None else (batch, M // batch)+ A = _Fake(lead + (K,), torch.float8_e4m3fn)+ B = _Fake((N, K), torch.float8_e4m3fn)+ C = _Fake(lead + (N,), dtype)+ return F8._fp8_tma_gemm_ok(A, B, C, 128, 128, 128, False)++[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("M", [128, 512, 1023, 2048])+def test_eager_gemm_takes_tma_at_any_size(tma_device, M, dtype):+ # On B200 the pointer kernel is 1.4-1.8x slower on the GPU at M * N * K below 2**34, so eager+ # GEMMs keep TMA whenever it routes, as on main.+ assert _ok(M, dtype = dtype) is True+ assert _ok(M, dtype = dtype, batch = 1) is True++[email protected]("M", [128, 512, 2048])+def test_graph_capture_takes_tma_at_any_size(tma_device, M):+ calls, capturing = tma_device+ _ok(M) # an eager call first runs the self check+ capturing[0] = True+ assert _ok(M) is True+++def test_capture_before_any_eager_call_keeps_pointer_kernel(tma_device):+ calls, capturing = tma_device+ capturing[0] = True+ assert _ok(4096) is False+ assert calls == []+++def _reference(tensor, rows, cols, block):+ return F8._TensorDescriptor.from_tensor(tensor.view(rows, cols), list(block))+++def _same(desc, ref, tensor):+ assert type(desc) is F8._TensorDescriptor+ assert desc.base.data_ptr() == ref.base.data_ptr() == tensor.data_ptr()+ assert list(desc.shape) == list(ref.shape)+ assert list(desc.strides) == list(ref.strides)+ assert list(desc.block_shape) == list(ref.block_shape)+ fields = {k: v for k, v in vars(ref).items() if k not in ("base", "shape", "strides")}+ assert {k: vars(desc)[k] for k in fields} == fields++[email protected]("dtype", [torch.float8_e4m3fn, torch.bfloat16, torch.float16])+def test_descriptor_matches_from_tensor(dtype):+ t = torch.zeros(2, 96, 256, dtype = dtype)+ _same(F8._fp8_tma_descriptor(t, 192, 256, (128, 128)), _reference(t, 192, 256, (128, 128)), t)+++def test_cached_fields_do_not_keep_the_first_tensor_alive():+ import gc+ import weakref++ F8._FP8_TMA_DESC_FIELDS.clear()+ t = torch.zeros(256, 256, dtype = torch.float16)+ ref = weakref.ref(t)+ F8._fp8_tma_descriptor(t, 256, 256, (128, 128))+ del t+ gc.collect()+ assert ref() is None+ assert all(+ k not in fields+ for fields in F8._FP8_TMA_DESC_FIELDS.values()+ for k in ("base", "shape", "strides")+ )+++def test_sliced_rows_fall_back_to_from_tensor():+ # A weight sliced out of a wider buffer has a row pitch above its width: keep its real strides.+ wide = torch.zeros(256, 384, dtype = torch.float8_e4m3fn)+ t = wide[:, :256]+ desc = F8._fp8_tma_descriptor(t, 256, 256, (128, 128))+ assert list(desc.strides) == [384, 1]+++def _launch(monkeypatch, A, B, C, M, N, K):+ seen = []++ class _Kernel:+ def __getitem__(self, grid):+ return lambda *args, **kwargs: seen.append((grid, args, kwargs))++ monkeypatch.setattr(F8, "_w8a8_block_fp8_matmul_tma", _Kernel())+ As = torch.ones(M, K // 128)+ Bs = torch.ones(N // 128, K // 128)+ F8._launch_fp8_tma_gemm(A, B, C, As, Bs, M, N, K, 128, 128)+ ((grid, args, kwargs),) = seen+ return args[:3]+++def test_launch_skips_per_call_validation(monkeypatch):+ M, N, K = 256, 384, 512+ A = torch.zeros(M, K, dtype = torch.float8_e4m3fn)+ B = torch.zeros(N, K, dtype = torch.float8_e4m3fn)+ C = torch.empty(M, N, dtype = torch.bfloat16)+ _launch(monkeypatch, A, B, C, M, N, K)+ built = []+ real = F8._TensorDescriptor.from_tensor+ monkeypatch.setattr(+ F8._TensorDescriptor,+ "from_tensor",+ staticmethod(lambda *a, **k: built.append(1) or real(*a, **k)),+ )+ for _ in range(5):+ a, b, c = _launch(monkeypatch, A, B, C, M, N, K)+ assert built == []+ _same(a, _reference(A, M, K, (128, 128)), A)+ _same(b, _reference(B, N, K, (128, 128)), B)+ _same(c, _reference(C, M, N, (128, 128)), C)+++def test_replaced_or_edited_weight_is_never_stale(monkeypatch):+ M, N, K = 128, 256, 256+ A = torch.zeros(M, K, dtype = torch.float8_e4m3fn)+ C = torch.empty(M, N, dtype = torch.bfloat16)+ W = torch.nn.Parameter(torch.zeros(N, K, dtype = torch.float8_e4m3fn), requires_grad = False)+ _, b0, _ = _launch(monkeypatch, A, W, C, M, N, K)+ assert b0.base is W++ W.data.copy_(torch.ones(N, K).to(torch.float8_e4m3fn)) # edited in place+ _, b1, _ = _launch(monkeypatch, A, W, C, M, N, K)+ assert b1.base is W and b1.base.data_ptr() == W.data_ptr()+ assert torch.equal(b1.base.float(), torch.ones(N, K))++ W2 = torch.full((N, K), 2.0).to(torch.float8_e4m3fn) # replaced by a new tensor+ _, b2, _ = _launch(monkeypatch, A, W2, C, M, N, K)+ assert b2.base is W2 and b2.base.data_ptr() == W2.data_ptr() != W.data_ptr()++ N2 = 384 # .data swapped for a different shape+ W.data = torch.zeros(N2, K, dtype = torch.float8_e4m3fn)+ C2 = torch.empty(M, N2, dtype = torch.bfloat16)+ _, b3, c3 = _launch(monkeypatch, A, W, C2, M, N2, K)+ _same(b3, _reference(W, N2, K, (128, 128)), W)+ _same(c3, _reference(C2, M, N2, (128, 128)), C2)+++def test_empty_input_still_raises_like_from_tensor():+ t = torch.zeros(0, 256, dtype = torch.float8_e4m3fn)+ with pytest.raises(AssertionError):+ F8._fp8_tma_descriptor(t, 0, 256, (128, 128))diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex f4c4ea24fd4..fd13c7efc3c 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -595,11 +595,45 @@ def _fp8_tma_gemm_ok(A, B, C, BLOCK_SIZE_M, block_n, block_k, long_idx):return checked+# from_tensor re-validates every field per call (~4 us each): validate once per (dtype, block shape).+_FP8_TMA_DESC_FIELDS = {}+++def _fp8_tma_descriptor(tensor, rows, cols, block_shape):+ key = (tensor.dtype, block_shape)+ fields = _FP8_TMA_DESC_FIELDS.get(key)+ if fields is None:+ proto = _TensorDescriptor.from_tensor(tensor.view(rows, cols), list(block_shape))+ # Tensor-independent fields only: caching `base` would keep the first GEMM's tensor alive.+ fields = {+ k: v+ for k, v in (getattr(proto, "__dict__", None) or {}).items()+ if k not in ("base", "shape", "strides")+ }+ _FP8_TMA_DESC_FIELDS[key] = fields+ # A sliced weight's row pitch is not its width: keep from_tensor's real strides.+ if (+ not fields+ or rows <= 0+ or cols <= 0+ or tensor.numel() != rows * cols+ or not tensor.is_contiguous()+ ):+ return _TensorDescriptor.from_tensor(tensor.view(rows, cols), list(block_shape))+ desc = _TensorDescriptor.__new__(_TensorDescriptor)+ desc.__dict__.update(fields)+ desc.base = tensor+ desc.shape = [rows, cols]+ desc.strides = [cols, 1]+ desc.block_shape = list(block_shape)+ return desc++def _launch_fp8_tma_gemm(A, B, C, As, Bs, M, N, K, block_n, block_k):BLOCK_SIZE_M = 128- a_desc = _TensorDescriptor.from_tensor(A.view(M, K), [BLOCK_SIZE_M, block_k])- b_desc = _TensorDescriptor.from_tensor(B, [block_n, block_k])- c_desc = _TensorDescriptor.from_tensor(C.view(M, N), [BLOCK_SIZE_M, block_n])+ a_desc = _fp8_tma_descriptor(A, M, K, (BLOCK_SIZE_M, block_k))+ b_desc = _fp8_tma_descriptor(B, N, K, (block_n, block_k))+ c_desc = _fp8_tma_descriptor(C, M, N, (BLOCK_SIZE_M, block_n))grid = (triton.cdiv(M, BLOCK_SIZE_M) * triton.cdiv(N, block_n),)_w8a8_block_fp8_matmul_tma[grid](a_desc,