Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Skip FP8 LoRA stats increments while compiling (#16645)

Daniel Han7863301bb1cf
diff --git a/tests/test_fp8_lora_compile_stats.py b/tests/test_fp8_lora_compile_stats.py
new file mode 100644
index 00000000000..3dddcbfffaa
--- /dev/null
+++ b/tests/test_fp8_lora_compile_stats.py
@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Stats counters must not recompile a compiled base layer on the small-row 16-bit fallback."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+from torch._dynamo.utils import counters
+
+from unsloth.kernels import fp8_lora as FL
+
+pytestmark = pytest.mark.skipif(
+ not hasattr(torch, "float8_e4m3fn"), reason = "needs torch float8 dtypes"
+)
+
+
+def _frozen_linear(bias):
+ torch.manual_seed(0)
+ lin = torch.nn.Linear(64, 64, bias = bias, dtype = torch.bfloat16)
+ return lin.requires_grad_(False)
+
+
[email protected]("bias", [False, True])
+def test_small_row_fallback_compiles_once_under_fullgraph(bias):
+ torch._dynamo.reset()
+ counters.clear()
+ FL.reset_fp8_lora_stats()
+ lin = _frozen_linear(bias)
+ FL.attach_fp8_lora_base(lin, offload = False)
+ compiled = torch.compile(lin, backend = "aot_eager", fullgraph = True)
+ x = torch.randn(8, 64, dtype = torch.bfloat16, requires_grad = True)
+ assert x.shape[0] < FL._MIN_ROWS
+ ref = torch.nn.functional.linear(x, lin.weight, lin.bias)
+ for _ in range(12):
+ out = compiled(x)
+ out.float().sum().backward()
+ x.grad = None
+ torch.testing.assert_close(out, ref)
+ assert counters["stats"]["unique_graphs"] == 1, dict(counters["stats"])
+ assert not counters["graph_break"], dict(counters["graph_break"])
+
+ FL.reset_fp8_lora_stats()
+ for _ in range(3):
+ lin(x)
+ assert FL.FP8_LORA_STATS["fwd_16bit"] == 3, FL.FP8_LORA_STATS
+
+
+def test_state_matmul_small_rows_compiles_once_under_fullgraph():
+ torch._dynamo.reset()
+ counters.clear()
+ lin = _frozen_linear(False)
+ state = FL.Fp8LoRABaseState(lin.weight)
+ W = lin.weight
+
+ def run(x, dy):
+ return state.matmul(x, W), state.dgrad(dy, W)
+
+ compiled = torch.compile(run, backend = "aot_eager", fullgraph = True)
+ x, dy = torch.randn(4, 64).bfloat16(), torch.randn(4, 64).bfloat16()
+ for _ in range(12):
+ compiled(x, dy)
+ assert counters["stats"]["unique_graphs"] == 1, dict(counters["stats"])
diff --git a/unsloth/kernels/fp8_lora.py b/unsloth/kernels/fp8_lora.py
index 615044d6038..b48d282c978 100644
--- a/unsloth/kernels/fp8_lora.py
+++ b/unsloth/kernels/fp8_lora.py
@@ -57,6 +57,13 @@ def reset_fp8_lora_stats():
FP8_LORA_STATS[key] = 0
+def _count(key):
+ # Counts eager calls only: a traced increment mutates a guarded global and recompiles every call.
+ # The fp8 GEMM ops stay opaque under compile, so their own counts still run.
+ if not _compiling():
+ FP8_LORA_STATS[key] += 1
+
+
_SCALING = os.environ.get("UNSLOTH_FP8_LORA_SCALING", "auto").strip().lower()
_SCALING = {"rowwise": "row", "tensorwise": "tensor"}.get(_SCALING, _SCALING)
if _SCALING not in ("auto", "row", "tensor"):
@@ -419,7 +426,7 @@ def _fwd_fake(x, w8, w_inv):
def fp8_lora_fwd(x: torch.Tensor, w8: torch.Tensor, w_inv: torch.Tensor) -> torch.Tensor:
"""x [M, K] (16-bit) @ dequant(w8 [N, K]).T -> [M, N] in x.dtype. w_inv: 0-d (tensorwise activations) or
[1, N] (rowwise activations: the weight's tensorwise scale per output column)."""
- FP8_LORA_STATS["fwd"] += 1
+ _count("fwd")
out = None
if w_inv.dim() == 2 and _rowwise_ok(x.dtype, x.device):
x8, x_inv = quantize_rowwise(x, E4M3)
@@ -443,7 +450,7 @@ def fp8_lora_dgrad(
"""dy [M, N] (16-bit) @ dequant(W [N, K]) -> dx [M, K] in dy.dtype. w8 is the cached K-major copy [K, N]
(w_is_t) or the forward copy [N, K], transposed here by the square-tile kernel inside the op, never by an
Inductor-generated fp8 transpose (those write corrupt bytes for non-square tile configs on sm89)."""
- FP8_LORA_STATS["dgrad"] += 1
+ _count("dgrad")
w8t = w8 if w_is_t else fp8_transpose(w8)
out = None
if w_inv.dim() == 2 and _rowwise_ok(dy.dtype, dy.device):
@@ -536,7 +543,7 @@ def fp8_lora_accum(
backward: bool,
transpose_weight: bool,
) -> torch.Tensor:
- FP8_LORA_STATS["dgrad" if backward else "fwd"] += 1
+ _count("dgrad" if backward else "fwd")
if transpose_weight:
w8 = fp8_transpose(w8)
if w_inv.dim() == 2:
@@ -643,7 +650,7 @@ class Fp8LoRABaseState:
self.dtype = weight.dtype
self._key = key
self.quantizations += 1
- FP8_LORA_STATS["quantize"] += 1
+ _count("quantize")
return self
def _dgrad_weight(self):
@@ -740,7 +747,7 @@ class Fp8LoRABaseState:
shape = X.shape
X2 = X.reshape(-1, shape[-1])
if not self._use_fp8(X2):
- FP8_LORA_STATS["fwd_16bit"] += 1
+ _count("fwd_16bit")
result = torch.matmul(X2, self._dense(W).t().to(X2.dtype))
elif _is_transposed(W):
result = self.dgrad(X2, W.t())
@@ -788,7 +795,7 @@ class Fp8LoRABaseState:
shape = dY.shape
dY2 = dY.reshape(-1, shape[-1])
if _DGRAD_DTYPE is None or not self._use_fp8(dY2):
- FP8_LORA_STATS["dgrad_16bit"] += 1
+ _count("dgrad_16bit")
if W is None and not self.offloaded:
raise RuntimeError("Unsloth: 16-bit dgrad needs the base weight")
result = torch.matmul(dY2, self._dense(W).to(dY2.dtype))
@@ -822,7 +829,7 @@ class _Fp8FrozenLinear(torch.autograd.Function):
shape = dy.shape
dy2 = dy.reshape(-1, shape[-1])
if _DGRAD_DTYPE is None:
- FP8_LORA_STATS["dgrad_16bit"] += 1
+ _count("dgrad_16bit")
dx = torch.matmul(dy2, weight.to(dy2.dtype))
else:
dx = fp8_lora_dgrad(dy2, w8t, w_inv, ctx.w_is_t)
@@ -833,7 +840,7 @@ def _fp8_base_forward(self, x):
state = self.__dict__.get("_unsloth_fp8_lora")
weight = self.weight
if state is None or weight.requires_grad or not state._use_fp8(x.reshape(-1, x.shape[-1])):
- FP8_LORA_STATS["fwd_16bit"] += 1
+ _count("fwd_16bit")
if state is not None and state.offloaded:
if weight.requires_grad: # unfrozen: train the exact weight
state.restore(self)