Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

QLoRA quant-state fallback: gate per weight, not a process-global flag (#14231)

Daniel Han2ff041b95a62
diff --git a/tests/python/test_get_lora_parameters_fsdp2_quant_state.py b/tests/python/test_get_lora_parameters_fsdp2_quant_state.py
index 030e183ae86..a083a07d5d2 100644
--- a/tests/python/test_get_lora_parameters_fsdp2_quant_state.py
+++ b/tests/python/test_get_lora_parameters_fsdp2_quant_state.py
@@ -121,10 +121,40 @@ def test_dense_weight_never_takes_the_module_getattr_path():
assert "quant_state" not in lookups, lookups
-def test_fallback_only_after_fsdp2_qlora_preparation():
- # Without prepare_fsdp2_qlora no weight can lose its quant_state, so the default LoRA forward keeps
- # the merge base's single getattr and never consults the base layer.
+def test_fallback_reads_no_process_global():
+ # The fallback is gated per weight on the packed (N, 1) shape, never on a module-global flag: a
+ # global read inside a compiled LoRA forward is a dynamo guard, so preparing one FSDP2 QLoRA model
+ # recompiled every compiled dense LoRA forward (#13880 review).
for name in ("get_lora_parameters", "get_lora_parameters_bias"):
- fn = _load_function(name, fsdp2_qlora_prepared = False)
- proj, _ = _fsdp2_unsharded_4bit()
- assert fn(proj)[1] is None
+ fn = _load_function(name)
+ assert "_FSDP2_QLORA_PREPARED" not in fn.__code__.co_names, name
+ proj, state = _fsdp2_unsharded_4bit()
+ assert fn(proj)[1] is state
+
+
+def test_compiled_dense_lookup_does_not_recompile_when_flag_flips():
+ # The review's repro: compile a dense LoRA lookup, flip the old global, call again: one graph.
+ import torch._dynamo
+
+ get_lora_parameters = _load_function("get_lora_parameters", fsdp2_qlora_prepared = False)
+ base_layer = torch.nn.Linear(64, 32, bias = False)
+ proj = _LoRA(base_layer)
+ graphs = []
+
+ def backend(gm, example_inputs):
+ graphs.append(gm)
+ return gm
+
+ torch._dynamo.reset()
+
+ @torch.compile(backend = backend, fullgraph = True)
+ def lookup(x):
+ W, W_quant, A, B, s = get_lora_parameters(proj)
+ return x @ W.t() + (x @ A.t()) @ B.t() * s
+
+ x = torch.randn(2, 64)
+ out = lookup(x)
+ get_lora_parameters.__globals__["_FSDP2_QLORA_PREPARED"] = True
+ assert torch.equal(lookup(x), out)
+ assert len(graphs) == 1, len(graphs)
+ torch._dynamo.reset()
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index fca52dadc9d..f4d995cf80b 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -13,7 +13,6 @@ import os
import sys
import torch
from unsloth_zoo.utils import Version
-from . import utils as _kernel_utils
from .utils import (
_has_active_dora_adapter,
_has_active_lora_bias,
@@ -1056,7 +1055,7 @@ def fast_lora_forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor:
scaling = self.scaling[active_adapter]
W = self.base_layer.weight
W_quant = getattr(W, "quant_state", None)
- if W_quant is None and _kernel_utils._FSDP2_QLORA_PREPARED:
+ if W_quant is None and W.ndim == 2 and W.shape[1] == 1:
W_quant = _base_quant_state(self.base_layer, W)
return _apply(LoRA_W, x, W, W_quant, lora_A, lora_B, scaling)
pass
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index d650ede50a3..625a067fa38 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -327,9 +327,9 @@ def QUANT_STATE(W):
return getattr(W, "quant_state", None)
-# Set by models/fsdp2_qlora.prepare_fsdp2_qlora: only FSDP2-sharded 4-bit weights lose quant_state, so every
-# other LoRA forward keeps the single getattr(W, "quant_state") and never reaches _base_quant_state.
-_FSDP2_QLORA_PREPARED = False
+# Only an FSDP2 all-gathered 4-bit weight loses quant_state, and its packed bytes are (N, 1): the getters
+# gate _base_quant_state on that shape, per weight. Never on a process-global flag: dynamo guards on a
+# global it reads, so preparing one FSDP2 QLoRA model recompiled every compiled dense LoRA forward (#13880).
def _base_quant_state(base_layer, W):
@@ -459,7 +459,7 @@ def get_lora_parameters(proj):
W_quant = getattr(W, "quant_state", None)
if W_quant is None:
- if _FSDP2_QLORA_PREPARED:
+ if getattr(W, "ndim", 0) == 2 and W.shape[1] == 1:
W_quant = _base_quant_state(base_layer, W)
# Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed
# compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.
@@ -520,7 +520,7 @@ def get_lora_parameters_bias(proj):
W_quant = getattr(W, "quant_state", None)
if W_quant is None:
- if _FSDP2_QLORA_PREPARED:
+ if getattr(W, "ndim", 0) == 2 and W.shape[1] == 1:
W_quant = _base_quant_state(base_layer, W)
# Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed
# compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index 00e4678dafd..898659cc275 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -664,6 +664,10 @@ def _wrap(cls, compile_options):
except torch.OutOfMemoryError:
raise
except Exception as e:
+ # Non-reentrant checkpoint ends a recompute by raising through the forward: control
+ # flow, not a compile failure (demoting on it left the stock forward in every block).
+ if type(e).__name__ == "_StopRecomputationError":
+ raise
if os.environ.get("UNSLOTH_BLOCK_COMPILE_DEBUG") == "1":
import traceback
traceback.print_exc()
diff --git a/unsloth/models/fsdp2_qlora.py b/unsloth/models/fsdp2_qlora.py
index d971a7a4f71..a32cd98790a 100644
--- a/unsloth/models/fsdp2_qlora.py
+++ b/unsloth/models/fsdp2_qlora.py
@@ -521,13 +521,6 @@ def prepare_fsdp2_qlora(
info["params4bit"] = len(params4bit)
if not params4bit:
return info
- try:
- from ..kernels import utils as kernel_utils
- except ImportError: # loaded on its own, without the Unsloth package (tests)
- kernel_utils = None
- if kernel_utils is not None:
- # Sharded 4-bit weights can now reach the LoRA kernels without their quant_state.
- kernel_utils._FSDP2_QLORA_PREPARED = True
if dtype is None:
dtype = next(
(