Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FSDP2 QLoRA: keep packed 4-bit bytes out of the loader dtype pass, fix merged_4bit save (#14165)
Daniel Han37745d5ad1f7
diff --git a/tests/test_fsdp2_qlora_dtype_pass.py b/tests/test_fsdp2_qlora_dtype_pass.pynew file mode 100644index 00000000000..ce24e480f08--- /dev/null+++ b/tests/test_fsdp2_qlora_dtype_pass.py@@ -0,0 +1,112 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FSDP2 packs 4-bit weights in a float dtype. The vision loader's dtype pass (unsloth_zoo's+patch_model_and_tokenizer: `module.to(module._pre_set_compute_dtype)`, set on gpt-oss's 4-bit+down_projs when loading in float16) would then cast the packed bytes as if they were weights.+`packed_4bit_as_uint8` shows them as uint8 for that pass, as without FSDP2."""++import ast+import importlib.util+from pathlib import Path++import pytest+import torch++bnb = pytest.importorskip("bitsandbytes")++_ROOT = Path(__file__).resolve().parents[1]+++def _load_module():+ path = _ROOT / "unsloth" / "models" / "fsdp2_qlora.py"+ spec = importlib.util.spec_from_file_location("_unsloth_fsdp2_qlora_dtype_pass", path)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def _float_packed_linear4bit(storage):+ torch.manual_seed(0)+ lin = bnb.nn.Linear4bit(+ 128, 64, bias = True, compute_dtype = torch.float16, quant_type = "nf4", quant_storage = storage+ )+ lin.weight = bnb.nn.Params4bit(+ torch.randn(64, 128) * 0.05,+ requires_grad = False,+ quant_type = "nf4",+ quant_storage = storage,+ blocksize = 64,+ module = lin,+ )+ lin = lin.to("cpu")+ lin.bias = torch.nn.Parameter(torch.ones(64, dtype = torch.float16))+ assert lin.weight.dtype == storage and lin.weight.bnb_quantized+ return lin+++def _dtype_pass(model):+ # unsloth_zoo.patching_utils.patch_model_and_tokenizer, "Upcast or downcast if explicitly set".+ for module in model.modules():+ if hasattr(module, "_pre_set_compute_dtype"):+ module.to(module._pre_set_compute_dtype)++[email protected]("storage", [torch.float16, torch.bfloat16])+def test_packed_bytes_survive_the_pre_set_compute_dtype_pass(storage):+ fsdp2_qlora = _load_module()+ lin = _float_packed_linear4bit(storage)+ lin._pre_set_compute_dtype = torch.float32+ model = torch.nn.Sequential(lin)+ weight = lin.weight+ before = weight.data.clone()+ with fsdp2_qlora.packed_4bit_as_uint8(model) as n:+ assert n == 1+ _dtype_pass(model)+ assert lin.weight is weight+ assert weight.dtype == storage and weight.quant_storage == storage+ assert torch.equal(weight.data.view(torch.uint8), before.view(torch.uint8))+ # The rest of the module is still cast, as without FSDP2.+ assert lin.bias.dtype == torch.float32+++def test_disabled_and_uint8_are_untouched():+ fsdp2_qlora = _load_module()+ lin = _float_packed_linear4bit(torch.bfloat16)+ with fsdp2_qlora.packed_4bit_as_uint8(lin, enabled = False) as n:+ assert n == 0 and lin.weight.dtype == torch.bfloat16+ lin8 = _float_packed_linear4bit(torch.uint8)+ with fsdp2_qlora.packed_4bit_as_uint8(lin8) as n:+ assert n == 0 and lin8.weight.dtype == torch.uint8+++def test_without_it_the_pass_casts_packed_bytes():+ # The failure the helper prevents: the packed float16 words are converted to float32 values.+ lin = _float_packed_linear4bit(torch.float16)+ lin._pre_set_compute_dtype = torch.float32+ _dtype_pass(torch.nn.Sequential(lin))+ assert lin.weight.dtype == torch.float32+++def test_vision_loader_wraps_patch_model_and_tokenizer():+ tree = ast.parse((_ROOT / "unsloth" / "models" / "vision.py").read_text(encoding = "utf-8"))+ wrapped = []+ for node in ast.walk(tree):+ if not isinstance(node, ast.With):+ continue+ ctx = [+ item.context_expr.func.id+ for item in node.items+ if isinstance(item.context_expr, ast.Call)+ and isinstance(item.context_expr.func, ast.Name)+ ]+ if "packed_4bit_as_uint8" not in ctx:+ continue+ for inner in ast.walk(node):+ if (+ isinstance(inner, ast.Call)+ and isinstance(inner.func, ast.Name)+ and inner.func.id == "patch_model_and_tokenizer"+ ):+ wrapped.append(inner)+ assert len(wrapped) == 1diff --git a/tests/test_fsdp2_save_args.py b/tests/test_fsdp2_save_args.pyindex 18f5ba7e89e..4640c4f18c6 100644--- a/tests/test_fsdp2_save_args.py+++ b/tests/test_fsdp2_save_args.py@@ -96,3 +96,63 @@ def test_peft_saves_gather_the_frozen_tensors_peft_writes():_forwarding, model, ("out", True, None, False), {})assert arguments["save_embedding_layers"] is False+++ "method,offload",+ [+ ("merged_16bit", True),+ ("lora", True),+ ("merged_4bit", False),+ ("merged_4bit_forced", False),+ ],+)+def test_merged_4bit_keeps_gathered_weights_on_their_device(monkeypatch, method, offload):+ # PEFT's 4-bit merge dequantizes on the packed weights' GPU; LoRA offloaded to the CPU next to+ # it fails with "Expected all tensors to be on the same device".+ import contextlib++ seen = {}++ @contextlib.contextmanager+ def _gathered(model, **kwargs):+ seen.update(kwargs)+ yield False++ monkeypatch.setattr(fsdp2_save, "fsdp2_units", lambda model: [object()])+ monkeypatch.setattr(fsdp2_save, "fsdp2_gather_active", lambda: False)+ monkeypatch.setattr(fsdp2_save, "fsdp2_gathered", _gathered)+ save = unsloth_save.unsloth_generic_save_pretrained_merged+ for call in (dict(save_method = method), {}):+ seen.clear()+ positional = () if call else (method,)+ assert save(_Model(), "out", None, *positional, **call) is None+ got = seen.get("offload_to_cpu", True)+ assert (got is True) if offload else callable(got)+++def test_merged_4bit_keeps_only_merge_operands_on_the_gpu():+ # Embeddings and LoRA on unquantized layers still go to the CPU; only the adapters of layers+ # with a packed 4-bit base stay with it.+ class Quant(torch.nn.Parameter):+ pass++ def lora_layer(quantized):+ layer = torch.nn.Module()+ layer.base_layer = torch.nn.Linear(4, 4)+ if quantized:+ layer.base_layer.weight.quant_state = object()+ layer.lora_A = torch.nn.ModuleDict({"default": torch.nn.Linear(4, 2, bias = False)})+ layer.lora_B = torch.nn.ModuleDict({"default": torch.nn.Linear(2, 4, bias = False)})+ return layer++ model = torch.nn.Module()+ model.embed = torch.nn.Embedding(8, 4)+ model.q4 = lora_layer(True)+ model.plain = lora_layer(False)+ offload = fsdp2_save.offload_all_but_4bit_merge_operands(model)+ assert not offload("q4.lora_A.default.weight", model.q4.lora_A["default"].weight)+ assert not offload("q4.lora_B.default.weight", model.q4.lora_B["default"].weight)+ assert offload("embed.weight", model.embed.weight)+ assert offload("plain.lora_A.default.weight", model.plain.lora_A["default"].weight)+ assert offload("plain.base_layer.weight", model.plain.base_layer.weight)diff --git a/unsloth/models/fsdp2_qlora.py b/unsloth/models/fsdp2_qlora.pyindex a32cd98790a..c24ea0a3214 100644--- a/unsloth/models/fsdp2_qlora.py+++ b/unsloth/models/fsdp2_qlora.py@@ -173,6 +173,42 @@ def params4bit_to_uint8_storage(model):return converted, left+def packed_4bit_as_uint8(model, enabled = True):+ """Show float-packed 4-bit weights as the uint8 bitsandbytes stores by default for the duration.+ Loader dtype passes (`module.to(module._pre_set_compute_dtype)`, forced float32) cast every+ floating parameter of a module, so packed bytes held in float16 / bfloat16 would be converted as+ if they were weights; as uint8 they are skipped, as they are without FSDP2. Restored afterwards."""+ packed = []+ if enabled:+ for module in model.modules():+ for name, param in module._parameters.items():+ if (+ param is None+ or not is_params4bit(param)+ or not param.is_floating_point()+ or not getattr(param, "bnb_quantized", False)+ or getattr(param, "quant_state", None) is None+ or not param.is_contiguous()+ or param.device.type == "meta"+ ):+ continue+ packed.append((module, name, param.dtype))+ for module, name, _ in packed:+ param = module._parameters[name]+ param.data = param.data.reshape(-1).view(torch.uint8).reshape(-1, 1)+ param.quant_storage = torch.uint8+ try:+ yield len(packed)+ finally:+ for module, name, dtype in packed:+ param = module._parameters.get(name)+ if param is None or param.dtype != torch.uint8:+ continue+ param.data = param.data.reshape(-1).view(dtype).reshape(-1, 1)+ param.quant_storage = dtype++def _packed_numel(param):if is_params4bit(param) and param.is_floating_point():return param.numel() * param.element_size()diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.pyindex 6b7cfca52de..7229a8803f8 100644--- a/unsloth/models/fsdp2_save.py+++ b/unsloth/models/fsdp2_save.py@@ -163,6 +163,7 @@ def fsdp2_gathered(enter; rank 0 sees whole parameters (and yields True), the other ranks yield False and shouldnot write. `trainable_only` gathers just the trainable parameters (enough for an adapter save),plus those `include(name, param)` selects (frozen ones the save still writes).+ `offload_to_cpu` may be a predicate `(name, param) -> bool`.Packed 4-bit weights stay on their device: the merge dequantizes them there."""if _ACTIVE:yield _ACTIVE[-1]@@ -206,7 +207,8 @@ def fsdp2_gathered(del fullcontinuemodule = owners[id(param)][0][0]- if offload_to_cpu and getattr(param, "quant_state", None) is None:+ offload = offload_to_cpu(name, param) if callable(offload_to_cpu) else offload_to_cpu+ if offload and getattr(param, "quant_state", None) is None:full = full.cpu()replaced[id(param)] = (param, _rebuild(param, full.detach(), module))if writer:@@ -227,6 +229,28 @@ def fsdp2_gathered(_wait_for_writer(dist, writer)+def offload_all_but_4bit_merge_operands(model):+ """`offload_to_cpu` for a merged 4-bit save: PEFT merges a LoRA layer where its packed 4-bit base+ weight stays (its GPU), so only the adapter weights of those layers stay there too. Everything+ else (embeddings, unquantized towers, LoRA on unquantized layers with their base) goes to the CPU."""+ keep = set()+ for module in model.modules() if isinstance(model, torch.nn.Module) else ():+ weight = getattr(getattr(module, "base_layer", None), "weight", None)+ if weight is None or getattr(weight, "quant_state", None) is None:+ continue+ for attr in (+ "lora_A",+ "lora_B",+ "lora_embedding_A",+ "lora_embedding_B",+ "lora_magnitude_vector",+ ):+ sub = getattr(module, attr, None)+ if isinstance(sub, torch.nn.Module):+ keep.update(id(param) for param in sub.parameters())+ return lambda name, param: id(param) not in keep++def fsdp2_gather_active():return bool(_ACTIVE)diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex fe6c0cb4d40..c7bfed88d5e 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -170,7 +170,7 @@ from .loader_utils import (sync_load_when_quantizing,warn_if_bitsandbytes_quantized_nothing,)-from .fsdp2_qlora import prepare_fsdp2_qlora, set_fsdp2_quant_storage+from .fsdp2_qlora import packed_4bit_as_uint8, prepare_fsdp2_qlora, set_fsdp2_quant_storage# `unsloth.save` imports `.models.loader_utils`, so binding a name out of it here at module# scope closes a cycle and a cold `import unsloth.save` fails on the half-built module.# `_gpu_init` hid that by importing `.models` first; the MLX branch never reaches it. So the@@ -4362,14 +4362,16 @@ class FastBaseModel:patch_medias_processor = Noneif patch_medias_processor is not None:patch_medias_processor(tokenizer)- model, tokenizer = patch_model_and_tokenizer(- model,- tokenizer,- downcast_rope = False,- fix_embeddings = False,- do_forced_float32 = do_forced_float32,- correct_dtype = correct_dtype,- )+ # FSDP2 packs 4-bit weights in a float dtype, which the dtype passes in here would cast.+ with packed_4bit_as_uint8(model, enabled = fsdp2_requested()):+ model, tokenizer = patch_model_and_tokenizer(+ model,+ tokenizer,+ downcast_rope = False,+ fix_embeddings = False,+ do_forced_float32 = do_forced_float32,+ correct_dtype = correct_dtype,+ )try:model, tokenizer = patch_tokenizer(model, tokenizer)diff --git a/unsloth/save.py b/unsloth/save.pyindex 6d5b62e8f94..6e8aa5310f3 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -6228,9 +6228,10 @@ def _fsdp2_peft_saved_frozen(model, arguments):return include-def _fsdp2_collective_save(trainable_only):+def _fsdp2_collective_save(trainable_only, offload_to_cpu = None):"""Under FSDP2 every rank calls the save: the shards are gathered, rank 0 writes, the others- return None. `trainable_only(self, args, kwargs)` says whether the adapter alone is enough.+ return None. `trainable_only(self, args, kwargs)` says whether the adapter alone is enough;+ `offload_to_cpu(self, args, kwargs)` (default True) whether gathered weights go to the CPU.A model FSDP2 does not shard, or a call that brings its own `state_dict`, is passed through."""def decorate(fn):@@ -6251,7 +6252,10 @@ def _fsdp2_collective_save(trainable_only):+ ("trainable" if only else "sharded")+ " weights to rank 0, which writes. Every rank must make this call.")- with fsdp2_gathered(self, trainable_only = only, include = include) as writer:+ offload = True if offload_to_cpu is None else offload_to_cpu(self, args, kwargs)+ with fsdp2_gathered(+ self, trainable_only = only, include = include, offload_to_cpu = offload+ ) as writer:if writer is False:return Nonereturn fn(self, *args, **kwargs)@@ -6273,9 +6277,23 @@ def _merged_save_method(args, kwargs):return args[2] if len(args) > 2 else "merged_16bit"+def _fsdp2_merged_offload(model, args, kwargs):+ if _merged_save_method(args, kwargs) not in (+ "merged_4bit",+ "merged_4bit_forced",+ "forced_merged_4bit",+ ):+ return True+ from .models.fsdp2_save import offload_all_but_4bit_merge_operands+ return offload_all_but_4bit_merge_operands(model)++@_fsdp2_collective_save(lambda self, args, kwargs: isinstance(self, PeftModel)- and _is_adapter_save_method(_merged_save_method(args, kwargs))+ and _is_adapter_save_method(_merged_save_method(args, kwargs)),+ # merged_4bit merges with PEFT where the packed 4-bit weights stay (on the GPU), so those layers'+ # LoRA weights stay on that device too; every other gathered weight still goes to the CPU.+ offload_to_cpu = lambda self, args, kwargs: _fsdp2_merged_offload(self, args, kwargs),)def unsloth_generic_save_pretrained_merged(self,