Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Skip FP8 LoRA stats increments while compiling (#16645)
Daniel Han7863301bb1cf
diff --git a/tests/test_fp8_lora_compile_stats.py b/tests/test_fp8_lora_compile_stats.pynew file mode 100644index 00000000000..3dddcbfffaa--- /dev/null+++ b/tests/test_fp8_lora_compile_stats.py@@ -0,0 +1,64 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Stats counters must not recompile a compiled base layer on the small-row 16-bit fallback."""++from __future__ import annotations++import pytest+import torch+from torch._dynamo.utils import counters++from unsloth.kernels import fp8_lora as FL++pytestmark = pytest.mark.skipif(+ not hasattr(torch, "float8_e4m3fn"), reason = "needs torch float8 dtypes"+)+++def _frozen_linear(bias):+ torch.manual_seed(0)+ lin = torch.nn.Linear(64, 64, bias = bias, dtype = torch.bfloat16)+ return lin.requires_grad_(False)++[email protected]("bias", [False, True])+def test_small_row_fallback_compiles_once_under_fullgraph(bias):+ torch._dynamo.reset()+ counters.clear()+ FL.reset_fp8_lora_stats()+ lin = _frozen_linear(bias)+ FL.attach_fp8_lora_base(lin, offload = False)+ compiled = torch.compile(lin, backend = "aot_eager", fullgraph = True)+ x = torch.randn(8, 64, dtype = torch.bfloat16, requires_grad = True)+ assert x.shape[0] < FL._MIN_ROWS+ ref = torch.nn.functional.linear(x, lin.weight, lin.bias)+ for _ in range(12):+ out = compiled(x)+ out.float().sum().backward()+ x.grad = None+ torch.testing.assert_close(out, ref)+ assert counters["stats"]["unique_graphs"] == 1, dict(counters["stats"])+ assert not counters["graph_break"], dict(counters["graph_break"])++ FL.reset_fp8_lora_stats()+ for _ in range(3):+ lin(x)+ assert FL.FP8_LORA_STATS["fwd_16bit"] == 3, FL.FP8_LORA_STATS+++def test_state_matmul_small_rows_compiles_once_under_fullgraph():+ torch._dynamo.reset()+ counters.clear()+ lin = _frozen_linear(False)+ state = FL.Fp8LoRABaseState(lin.weight)+ W = lin.weight++ def run(x, dy):+ return state.matmul(x, W), state.dgrad(dy, W)++ compiled = torch.compile(run, backend = "aot_eager", fullgraph = True)+ x, dy = torch.randn(4, 64).bfloat16(), torch.randn(4, 64).bfloat16()+ for _ in range(12):+ compiled(x, dy)+ assert counters["stats"]["unique_graphs"] == 1, dict(counters["stats"])diff --git a/unsloth/kernels/fp8_lora.py b/unsloth/kernels/fp8_lora.pyindex 615044d6038..b48d282c978 100644--- a/unsloth/kernels/fp8_lora.py+++ b/unsloth/kernels/fp8_lora.py@@ -57,6 +57,13 @@ def reset_fp8_lora_stats():FP8_LORA_STATS[key] = 0+def _count(key):+ # Counts eager calls only: a traced increment mutates a guarded global and recompiles every call.+ # The fp8 GEMM ops stay opaque under compile, so their own counts still run.+ if not _compiling():+ FP8_LORA_STATS[key] += 1++_SCALING = os.environ.get("UNSLOTH_FP8_LORA_SCALING", "auto").strip().lower()_SCALING = {"rowwise": "row", "tensorwise": "tensor"}.get(_SCALING, _SCALING)if _SCALING not in ("auto", "row", "tensor"):@@ -419,7 +426,7 @@ def _fwd_fake(x, w8, w_inv):def fp8_lora_fwd(x: torch.Tensor, w8: torch.Tensor, w_inv: torch.Tensor) -> torch.Tensor:"""x [M, K] (16-bit) @ dequant(w8 [N, K]).T -> [M, N] in x.dtype. w_inv: 0-d (tensorwise activations) or[1, N] (rowwise activations: the weight's tensorwise scale per output column)."""- FP8_LORA_STATS["fwd"] += 1+ _count("fwd")out = Noneif w_inv.dim() == 2 and _rowwise_ok(x.dtype, x.device):x8, x_inv = quantize_rowwise(x, E4M3)@@ -443,7 +450,7 @@ def fp8_lora_dgrad("""dy [M, N] (16-bit) @ dequant(W [N, K]) -> dx [M, K] in dy.dtype. w8 is the cached K-major copy [K, N](w_is_t) or the forward copy [N, K], transposed here by the square-tile kernel inside the op, never by anInductor-generated fp8 transpose (those write corrupt bytes for non-square tile configs on sm89)."""- FP8_LORA_STATS["dgrad"] += 1+ _count("dgrad")w8t = w8 if w_is_t else fp8_transpose(w8)out = Noneif w_inv.dim() == 2 and _rowwise_ok(dy.dtype, dy.device):@@ -536,7 +543,7 @@ def fp8_lora_accum(backward: bool,transpose_weight: bool,) -> torch.Tensor:- FP8_LORA_STATS["dgrad" if backward else "fwd"] += 1+ _count("dgrad" if backward else "fwd")if transpose_weight:w8 = fp8_transpose(w8)if w_inv.dim() == 2:@@ -643,7 +650,7 @@ class Fp8LoRABaseState:self.dtype = weight.dtypeself._key = keyself.quantizations += 1- FP8_LORA_STATS["quantize"] += 1+ _count("quantize")return selfdef _dgrad_weight(self):@@ -740,7 +747,7 @@ class Fp8LoRABaseState:shape = X.shapeX2 = X.reshape(-1, shape[-1])if not self._use_fp8(X2):- FP8_LORA_STATS["fwd_16bit"] += 1+ _count("fwd_16bit")result = torch.matmul(X2, self._dense(W).t().to(X2.dtype))elif _is_transposed(W):result = self.dgrad(X2, W.t())@@ -788,7 +795,7 @@ class Fp8LoRABaseState:shape = dY.shapedY2 = dY.reshape(-1, shape[-1])if _DGRAD_DTYPE is None or not self._use_fp8(dY2):- FP8_LORA_STATS["dgrad_16bit"] += 1+ _count("dgrad_16bit")if W is None and not self.offloaded:raise RuntimeError("Unsloth: 16-bit dgrad needs the base weight")result = torch.matmul(dY2, self._dense(W).to(dY2.dtype))@@ -822,7 +829,7 @@ class _Fp8FrozenLinear(torch.autograd.Function):shape = dy.shapedy2 = dy.reshape(-1, shape[-1])if _DGRAD_DTYPE is None:- FP8_LORA_STATS["dgrad_16bit"] += 1+ _count("dgrad_16bit")dx = torch.matmul(dy2, weight.to(dy2.dtype))else:dx = fp8_lora_dgrad(dy2, w8t, w_inv, ctx.w_is_t)@@ -833,7 +840,7 @@ def _fp8_base_forward(self, x):state = self.__dict__.get("_unsloth_fp8_lora")weight = self.weightif state is None or weight.requires_grad or not state._use_fp8(x.reshape(-1, x.shape[-1])):- FP8_LORA_STATS["fwd_16bit"] += 1+ _count("fwd_16bit")if state is not None and state.offloaded:if weight.requires_grad: # unfrozen: train the exact weightstate.restore(self)