Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
QLoRA quant-state fallback: gate per weight, not a process-global flag (#14231)
Daniel Han2ff041b95a62
diff --git a/tests/python/test_get_lora_parameters_fsdp2_quant_state.py b/tests/python/test_get_lora_parameters_fsdp2_quant_state.pyindex 030e183ae86..a083a07d5d2 100644--- a/tests/python/test_get_lora_parameters_fsdp2_quant_state.py+++ b/tests/python/test_get_lora_parameters_fsdp2_quant_state.py@@ -121,10 +121,40 @@ def test_dense_weight_never_takes_the_module_getattr_path():assert "quant_state" not in lookups, lookups-def test_fallback_only_after_fsdp2_qlora_preparation():- # Without prepare_fsdp2_qlora no weight can lose its quant_state, so the default LoRA forward keeps- # the merge base's single getattr and never consults the base layer.+def test_fallback_reads_no_process_global():+ # The fallback is gated per weight on the packed (N, 1) shape, never on a module-global flag: a+ # global read inside a compiled LoRA forward is a dynamo guard, so preparing one FSDP2 QLoRA model+ # recompiled every compiled dense LoRA forward (#13880 review).for name in ("get_lora_parameters", "get_lora_parameters_bias"):- fn = _load_function(name, fsdp2_qlora_prepared = False)- proj, _ = _fsdp2_unsharded_4bit()- assert fn(proj)[1] is None+ fn = _load_function(name)+ assert "_FSDP2_QLORA_PREPARED" not in fn.__code__.co_names, name+ proj, state = _fsdp2_unsharded_4bit()+ assert fn(proj)[1] is state+++def test_compiled_dense_lookup_does_not_recompile_when_flag_flips():+ # The review's repro: compile a dense LoRA lookup, flip the old global, call again: one graph.+ import torch._dynamo++ get_lora_parameters = _load_function("get_lora_parameters", fsdp2_qlora_prepared = False)+ base_layer = torch.nn.Linear(64, 32, bias = False)+ proj = _LoRA(base_layer)+ graphs = []++ def backend(gm, example_inputs):+ graphs.append(gm)+ return gm++ torch._dynamo.reset()++ @torch.compile(backend = backend, fullgraph = True)+ def lookup(x):+ W, W_quant, A, B, s = get_lora_parameters(proj)+ return x @ W.t() + (x @ A.t()) @ B.t() * s++ x = torch.randn(2, 64)+ out = lookup(x)+ get_lora_parameters.__globals__["_FSDP2_QLORA_PREPARED"] = True+ assert torch.equal(lookup(x), out)+ assert len(graphs) == 1, len(graphs)+ torch._dynamo.reset()diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex fca52dadc9d..f4d995cf80b 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -13,7 +13,6 @@ import osimport sysimport torchfrom unsloth_zoo.utils import Version-from . import utils as _kernel_utilsfrom .utils import (_has_active_dora_adapter,_has_active_lora_bias,@@ -1056,7 +1055,7 @@ def fast_lora_forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor:scaling = self.scaling[active_adapter]W = self.base_layer.weightW_quant = getattr(W, "quant_state", None)- if W_quant is None and _kernel_utils._FSDP2_QLORA_PREPARED:+ if W_quant is None and W.ndim == 2 and W.shape[1] == 1:W_quant = _base_quant_state(self.base_layer, W)return _apply(LoRA_W, x, W, W_quant, lora_A, lora_B, scaling)passdiff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex d650ede50a3..625a067fa38 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -327,9 +327,9 @@ def QUANT_STATE(W):return getattr(W, "quant_state", None)-# Set by models/fsdp2_qlora.prepare_fsdp2_qlora: only FSDP2-sharded 4-bit weights lose quant_state, so every-# other LoRA forward keeps the single getattr(W, "quant_state") and never reaches _base_quant_state.-_FSDP2_QLORA_PREPARED = False+# Only an FSDP2 all-gathered 4-bit weight loses quant_state, and its packed bytes are (N, 1): the getters+# gate _base_quant_state on that shape, per weight. Never on a process-global flag: dynamo guards on a+# global it reads, so preparing one FSDP2 QLoRA model recompiled every compiled dense LoRA forward (#13880).def _base_quant_state(base_layer, W):@@ -459,7 +459,7 @@ def get_lora_parameters(proj):W_quant = getattr(W, "quant_state", None)if W_quant is None:- if _FSDP2_QLORA_PREPARED:+ if getattr(W, "ndim", 0) == 2 and W.shape[1] == 1:W_quant = _base_quant_state(base_layer, W)# Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed# compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.@@ -520,7 +520,7 @@ def get_lora_parameters_bias(proj):W_quant = getattr(W, "quant_state", None)if W_quant is None:- if _FSDP2_QLORA_PREPARED:+ if getattr(W, "ndim", 0) == 2 and W.shape[1] == 1:W_quant = _base_quant_state(base_layer, W)# Only fall back to a weight_scale(_inv) when the weight is still fp8; a bf16 weight (a decompressed# compressed-tensors layer) must not carry a scale as its quant state or fast_gemv crashes.diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex 00e4678dafd..898659cc275 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -664,6 +664,10 @@ def _wrap(cls, compile_options):except torch.OutOfMemoryError:raiseexcept Exception as e:+ # Non-reentrant checkpoint ends a recompute by raising through the forward: control+ # flow, not a compile failure (demoting on it left the stock forward in every block).+ if type(e).__name__ == "_StopRecomputationError":+ raiseif os.environ.get("UNSLOTH_BLOCK_COMPILE_DEBUG") == "1":import tracebacktraceback.print_exc()diff --git a/unsloth/models/fsdp2_qlora.py b/unsloth/models/fsdp2_qlora.pyindex d971a7a4f71..a32cd98790a 100644--- a/unsloth/models/fsdp2_qlora.py+++ b/unsloth/models/fsdp2_qlora.py@@ -521,13 +521,6 @@ def prepare_fsdp2_qlora(info["params4bit"] = len(params4bit)if not params4bit:return info- try:- from ..kernels import utils as kernel_utils- except ImportError: # loaded on its own, without the Unsloth package (tests)- kernel_utils = None- if kernel_utils is not None:- # Sharded 4-bit weights can now reach the LoRA kernels without their quant_state.- kernel_utils._FSDP2_QLORA_PREPARED = Trueif dtype is None:dtype = next((