Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Dequantize pre-quantized bitsandbytes checkpoints for full finetuning (#13079)
Daniel Han7a11957e3e29
diff --git a/tests/test_full_finetune_prequantized_bnb.py b/tests/test_full_finetune_prequantized_bnb.pynew file mode 100644index 00000000000..70173ba25ac--- /dev/null+++ b/tests/test_full_finetune_prequantized_bnb.py@@ -0,0 +1,115 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Full finetuning a pre-quantized bitsandbytes checkpoint dequantizes it instead of crashing (#2613)."""++import importlib.util+import subprocess+import sys+import textwrap+from types import SimpleNamespace++import pytest+import torch+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py++from unsloth.models.loader_utils import _dequantize_bitsandbytes_for_full_finetuning+++class _Deserialize:+ def __init__(self, hf_quantizer):+ self.hf_quantizer = hf_quantizer+++class _FakeModel(torch.nn.Module):+ """Dequantizes like transformers 4.x: unguarded deletes of the quantization metadata."""++ def __init__(+ self,+ quant_method,+ conversions,+ pre_quantized = True,+ ):+ super().__init__()+ self.hf_quantizer = SimpleNamespace(+ quantization_config = SimpleNamespace(quant_method = quant_method),+ pre_quantized = pre_quantized,+ )+ self.config = SimpleNamespace() # an extracted text core's config carries none of it+ self.is_loaded_in_4bit = True+ self._weight_conversions = conversions(self.hf_quantizer)+ self.dequantized_with = None++ def dequantize(self, dtype = None):+ self.dequantized_with = dtype+ del self.hf_quantizer+ del self.config.quantization_config+ del self.config._pre_quantization_dtype+ del self.quantization_method+ return self+++def test_bitsandbytes_is_dequantized_and_its_load_state_cleared(capsys):+ kept = SimpleNamespace(operations = [object()])+ model = _FakeModel(+ "bitsandbytes",+ lambda q: [SimpleNamespace(operations = [_Deserialize(q)]), kept],+ )+ assert _dequantize_bitsandbytes_for_full_finetuning(model, torch.bfloat16, "local/dir") is True+ assert model.dequantized_with is torch.bfloat16+ assert model._weight_conversions == [kept]+ assert model.is_loaded_in_4bit is False+ assert "local/dir" in capsys.readouterr().out+++def test_other_quant_methods_and_on_the_fly_configs_are_left_alone():+ for model in (+ _FakeModel("fp8", lambda q: [SimpleNamespace(operations = [_Deserialize(q)])]),+ _FakeModel("bitsandbytes", lambda q: [], pre_quantized = False),+ ):+ assert _dequantize_bitsandbytes_for_full_finetuning(model, torch.bfloat16) is False+ assert model.dequantized_with is None and model.is_loaded_in_4bit is True+ assert _dequantize_bitsandbytes_for_full_finetuning(torch.nn.Linear(2, 2)) is False+++ not has_real_cuda() or importlib.util.find_spec("bitsandbytes") is None,+ reason = "needs CUDA and bitsandbytes",+)+def test_full_finetune_a_local_bnb_4bit_folder(tmp_path):+ script = textwrap.dedent(+ """+ import sys, torch+ from unsloth import FastLanguageModel+ from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig+ src, quantized, saved = sys.argv[1:4]+ AutoModelForCausalLM.from_pretrained(+ src, quantization_config = BitsAndBytesConfig(load_in_4bit = True), device_map = "cuda:0"+ ).save_pretrained(quantized)+ AutoTokenizer.from_pretrained(src).save_pretrained(quantized)+ model, tokenizer = FastLanguageModel.from_pretrained(+ quantized, max_seq_length = 64, load_in_4bit = False, full_finetuning = True+ )+ assert all(p.is_floating_point() and p.requires_grad for p in model.parameters())+ ids = torch.tensor([[1, 2, 3, 4]], device = model.device)+ model(input_ids = ids, labels = ids).loss.backward()+ model.save_pretrained(saved)+ print("FULL_FINETUNE_OK")+ """+ )+ result = subprocess.run(+ [+ sys.executable,+ "-c",+ script,+ "trl-internal-testing/tiny-Qwen3ForCausalLM",+ str(tmp_path / "bnb"),+ str(tmp_path / "saved"),+ ],+ capture_output = True,+ text = True,+ timeout = 900,+ )+ assert "FULL_FINETUNE_OK" in result.stdout, result.stdout[-3000:] + result.stderr[-3000:]+ config = (tmp_path / "saved" / "config.json").read_text(encoding = "utf-8")+ assert "quantization_config" not in configdiff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 456e7e40463..bfa8ae02056 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -2724,6 +2724,54 @@ def _decompress_compressed_tensors_model(model):return True+def _dequantize_bitsandbytes_for_full_finetuning(+ model,+ dtype = None,+ model_name = "",+):+ """Full finetuning marks every weight trainable, which the uint8 / int8 weights of a pre-quantized bitsandbytes checkpoint (a local `-bnb-4bit` folder, which the name mapper cannot redirect) reject (#2613)."""+ quantizer = getattr(model, "hf_quantizer", None)+ method = getattr(getattr(quantizer, "quantization_config", None), "quant_method", None)+ # pre_quantized False = an explicit on-the-fly config; rounding then restoring would train a lossy copy.+ if str(getattr(method, "value", method)).lower() != "bitsandbytes" or not getattr(+ quantizer, "pre_quantized", True+ ):+ return False+ print(+ f"Unsloth: `{model_name}` is a pre-quantized bitsandbytes checkpoint, so full finetuning "+ "dequantizes it to 16bit. For the best accuracy, full finetune the original 16bit model instead."+ )+ # transformers < 5 deletes these unguarded, and a composite's extracted text core lacks some of them.+ for owner, attribute in (+ (model, "quantization_method"),+ (model.config, "quantization_config"),+ (model.config, "_pre_quantization_dtype"),+ ):+ if not hasattr(owner, attribute):+ setattr(owner, attribute, None)+ # transformers < 5 has no dtype argument; prepare_model_for_training casts the weights anyway.+ if "dtype" in inspect.signature(model.dequantize).parameters:+ model.dequantize(dtype = dtype)+ else:+ model.dequantize()+ # Left True by dequantize; the trainer's DataParallel gate and PEFT still read them.+ for attribute in ("is_loaded_in_4bit", "is_loaded_in_8bit"):+ if getattr(model, attribute, False):+ setattr(model, attribute, False)+ # transformers 5 keeps the load's bnb deserialize converter, whose missing reverse op makes save_pretrained raise NotImplementedError.+ conversions = getattr(model, "_weight_conversions", None)+ if isinstance(conversions, list):+ model._weight_conversions = [+ conversion+ for conversion in conversions+ if not any(+ getattr(op, "hf_quantizer", None) is quantizer+ for op in getattr(conversion, "operations", None) or ()+ )+ ]+ return True++def _prepare_compressed_tensors_model(model, full_finetuning = False):# Routed FP8 / NVFP4 weights are frozen, so full finetuning always takes the decompressed bf16 weights.if full_finetuning:diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 793850e2f75..ff783f4bf4c 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -147,6 +147,7 @@ from .loader_utils import (_dequantize_leftover_fp8_params,_restore_dropped_fp8_scales,_prepare_compressed_tensors_model,+ _dequantize_bitsandbytes_for_full_finetuning,planner_class_mismatch_reason,planner_model_class,exclude_no_placement_params,@@ -3614,6 +3615,8 @@ class FastBaseModel:dtype = torch_dtype,)_prepare_compressed_tensors_model(model, full_finetuning = full_finetuning)+ if full_finetuning:+ _dequantize_bitsandbytes_for_full_finetuning(model, torch_dtype, model_name)if load_in_16bit and not load_in_4bit and not load_in_8bit:_dequantize_leftover_fp8_params(model,