Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Load Ling 3.x low-rank queries on native bailing_hybrid and fix its exports (#14136)
Daniel Han32980a424082
diff --git a/tests/test_bailing_hybrid_low_rank_query.py b/tests/test_bailing_hybrid_low_rank_query.pynew file mode 100644index 00000000000..08458359e90--- /dev/null+++ b/tests/test_bailing_hybrid_low_rank_query.py@@ -0,0 +1,134 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Ling 3.x checkpoints with `q_lora_rank` load their low-rank query on transformers' bailing_hybrid (#13948)."""++import json+import os+import shutil++import pytest+import torch++import unsloth # noqa: F401+from unsloth.import_fixes import fix_transformers_bailing_hybrid_low_rank_query+from unsloth.save import _base_config_dir++modeling = pytest.importorskip("transformers.models.bailing_hybrid.modeling_bailing_hybrid")+from transformers import BailingHybridConfig, BailingHybridForCausalLM+++def _config(q_lora_rank):+ return BailingHybridConfig(+ vocab_size = 64,+ pad_token_id = 0,+ eos_token_id = 1,+ hidden_size = 32,+ intermediate_size = 64,+ moe_intermediate_size = 16,+ num_hidden_layers = 4,+ num_attention_heads = 2,+ num_key_value_heads = 2,+ head_dim = 8,+ num_experts = 4,+ num_experts_per_tok = 2,+ n_group = 1,+ topk_group = 1,+ first_k_dense_replace = 1,+ layer_group_size = 4,+ kv_lora_rank = 16,+ qk_nope_head_dim = 8,+ qk_rope_head_dim = 4,+ v_head_dim = 8,+ q_lora_rank = q_lora_rank,+ )+++def _mla(model):+ return model.model.layers[3].self_attn+++def test_low_rank_query_replaces_q_proj():+ fix_transformers_bailing_hybrid_low_rank_query()+ torch.manual_seed(0)+ model = BailingHybridForCausalLM(_config(12)).float().eval()+ keys = set(model.state_dict())+ assert {+ "model.layers.3.self_attn.q_a_proj.weight",+ "model.layers.3.self_attn.q_a_layernorm.weight",+ "model.layers.3.self_attn.q_b_proj.weight",+ } <= keys+ assert "model.layers.3.self_attn.q_proj.weight" not in keys+ attention = _mla(model)+ assert attention.q_b_proj.weight.shape == (2 * 12, 12)+ x = torch.randn(1, 5, 32)+ expected = attention.q_b_proj(attention.q_a_layernorm(attention.q_a_proj(x)))+ torch.testing.assert_close(attention.q_proj(x), expected)+++def test_checkpoint_low_rank_query_loads_without_missing_keys(tmp_path):+ fix_transformers_bailing_hybrid_low_rank_query()+ torch.manual_seed(0)+ model = BailingHybridForCausalLM(_config(12)).float()+ model.save_pretrained(tmp_path)+ reloaded, info = BailingHybridForCausalLM.from_pretrained(tmp_path, output_loading_info = True)+ assert not info["missing_keys"] and not info["unexpected_keys"]+ torch.testing.assert_close(+ _mla(reloaded).q_a_proj.weight.float(), _mla(model).q_a_proj.weight.float()+ )+++def test_full_rank_query_is_untouched():+ fix_transformers_bailing_hybrid_low_rank_query()+ model = BailingHybridForCausalLM(_config(None))+ attention = _mla(model)+ assert isinstance(attention.q_proj, torch.nn.Linear)+ assert not hasattr(attention, "q_a_proj")+++def test_patch_is_idempotent():+ fix_transformers_bailing_hybrid_low_rank_query()+ init = modeling.BailingHybridAttention.__init__+ fix_transformers_bailing_hybrid_low_rank_query()+ assert modeling.BailingHybridAttention.__init__ is init+++def test_base_config_dir_holds_only_the_config(tmp_path, monkeypatch):+ import huggingface_hub++ cached = tmp_path / "config.json"+ cached.write_text('{"kda_lower_bound": -5}')+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", lambda *a, **k: str(cached))+ directory = _base_config_dir("org/model")+ try:+ assert os.listdir(directory) == ["config.json"]+ with open(os.path.join(directory, "config.json")) as f:+ assert json.load(f) == {"kda_lower_bound": -5}+ finally:+ shutil.rmtree(directory)+++def test_base_config_dir_passes_anonymous_through(tmp_path, monkeypatch):+ import huggingface_hub++ cached = tmp_path / "config.json"+ cached.write_text("{}")+ seen = []+ monkeypatch.setattr(+ huggingface_hub,+ "hf_hub_download",+ lambda *a, **k: seen.append(k["token"]) or str(cached),+ )+ for token in (False, None, "hf_explicit"):+ shutil.rmtree(_base_config_dir("org/model", token))+ assert seen == [False, None, "hf_explicit"]+++def test_base_config_dir_unavailable(monkeypatch):+ import huggingface_hub++ def missing(*args, **kwargs):+ raise huggingface_hub.errors.LocalEntryNotFoundError("offline")++ monkeypatch.setattr(huggingface_hub, "hf_hub_download", missing)+ assert _base_config_dir("org/model") is Nonediff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 1bd664b9c29..3d87fab1fdb 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -12604,3 +12604,59 @@ def fix_transformers_longcat_lsa_config():logger.info("Unsloth: LongcatCausalLM configs without a model_type load on transformers' longcat_flash.")+++class _BailingHybridLowRankQuery:+ """`q_b_proj(q_a_layernorm(q_a_proj(x)))` standing in for the attention's `q_proj` call. Not a+ module, so the state dict holds the three checkpoint tensors and no `q_proj`."""++ __slots__ = ("attention",)++ def __init__(self, attention):+ self.attention = attention++ def __call__(self, hidden_states):+ attention = self.attention+ return attention.q_b_proj(attention.q_a_layernorm(attention.q_a_proj(hidden_states)))+++def fix_transformers_bailing_hybrid_low_rank_query():+ """transformers' bailing_hybrid (5.20) builds the full-attention query as one `q_proj`, so a Ling+ 3.x checkpoint with `q_lora_rank` set (Ling-3.0-tiny, Ling-3.1-flash: `q_a_proj`, `q_a_layernorm`,+ `q_b_proj` on disk) loads those layers with a randomly initialised query (#13948). Build the+ low-rank query the checkpoint and its own modeling code carry; `q_lora_rank = None` (Ling-3.0-flash)+ is untouched. Lazy: called from the loader once the config names bailing_hybrid."""+ try:+ from transformers.models.bailing_hybrid import modeling_bailing_hybrid as module+ except Exception:+ return+ cls = getattr(module, "BailingHybridAttention", None)+ norm = getattr(module, "BailingHybridRMSNorm", None)+ if cls is None or norm is None or getattr(cls.__init__, "_unsloth_low_rank_query", False):+ return+ try:+ if "q_a_proj" in inspect.getsource(cls):+ return+ except Exception:+ return+ import torch++ original = cls.__init__++ @functools.wraps(original)+ def __init__(self, config, *args, **kwargs):+ original(self, config, *args, **kwargs)+ rank = getattr(config, "q_lora_rank", None)+ query = self.__dict__.get("_modules", {}).get("q_proj")+ if not rank or not isinstance(query, torch.nn.Linear):+ return+ del self.q_proj+ self.q_a_proj = torch.nn.Linear(+ config.hidden_size, rank, bias = bool(getattr(config, "use_qkv_bias", False))+ )+ self.q_a_layernorm = norm(rank) # the checkpoint's own code keeps the default eps+ self.q_b_proj = torch.nn.Linear(rank, query.out_features, bias = False)+ object.__setattr__(self, "q_proj", _BailingHybridLowRankQuery(self))++ __init__._unsloth_low_rank_query = True+ cls.__init__ = __init__diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex e83042a6bb3..452624ba6e0 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -3836,6 +3836,9 @@ class FastBaseModel:_cfg_val = kwargs.pop("max_position_embeddings", None)if _cfg_val is not None:setattr(model_config, "max_position_embeddings", _cfg_val)+ if "bailing_hybrid" in (model_types or ()):+ from ..import_fixes import fix_transformers_bailing_hybrid_low_rank_query+ fix_transformers_bailing_hybrid_low_rank_query()with _tolerate_dtype_cast_on_quantized_model(bool(trust_remote_code)and (diff --git a/unsloth/save.py b/unsloth/save.pyindex 7d55ac73412..f53d243a980 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -5149,6 +5149,32 @@ def _apply_token_to_child_env(env, token, *, explicit):env["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "0"+def _base_config_dir(repo_id, token = None):+ """Temporary dir with only `repo_id`'s config.json (all convert_lora_to_gguf.py --base reads), else None."""+ import tempfile++ try:+ from huggingface_hub import hf_hub_download++ # The live cache, not huggingface_hub's frozen constant: a runtime redirect would else miss (#6890).+ try:+ from unsloth_zoo.hf_cache import _active_caches+ cache_dir = _active_caches()[1]+ except Exception:+ cache_dir = None+ config = hf_hub_download(+ repo_id,+ "config.json",+ cache_dir = str(cache_dir) if cache_dir is not None else None,+ token = token,+ )+ except Exception:+ return None+ directory = tempfile.mkdtemp(prefix = "unsloth-lora-gguf-base-")+ shutil.copyfile(config, os.path.join(directory, "config.json"))+ return directory++def _lora_base_model_id(model):"""Base model id for a PEFT model: prefer the active adapter's recorded `base_model_name_or_path`, the authoritative source, else the model config."""base = None@@ -5299,6 +5325,7 @@ def _unsloth_save_lora_gguf(passmodel_name = _model_basename(base_model_id)+ base_config_dir = None# Save the adapter: an isolated temp dir for a hub push, else save_directory itself, wrapped so it is always cleaned up.if push_to_hub:lora_dir = tempfile.mkdtemp(prefix = "unsloth-lora-gguf-")@@ -5327,12 +5354,22 @@ def _unsloth_save_lora_gguf(out_gguf = os.path.join(lora_dir, f"{model_name}-lora-{outtype}.gguf")cmd = [sys.executable, converter, lora_dir, "--outfile", out_gguf, "--outtype", outtype]+ # Pass --trust-remote-code only when the model really came from custom code (the approved load decision), not merely because its config carries an auto_map entry.+ loaded_via_remote_code = _loaded_via_remote_code(model)+ # --base-model-id's strict AutoConfig rejects an older config (Ling-3.0-tiny's int kda_lower_bound, #13948); --base falls back to config.json.+ local_base = Noneif os.path.isdir(base_model_id):- cmd += ["--base", base_model_id]+ local_base = base_model_id+ elif not loaded_via_remote_code:+ # False stays anonymous; a token we resolved ourselves is left to huggingface_hub, which honors HF_HUB_DISABLE_IMPLICIT_TOKEN.+ local_base = base_config_dir = _base_config_dir(+ base_model_id, token if token is False or token_is_explicit else None+ )+ if local_base is not None:+ cmd += ["--base", local_base]else:cmd += ["--base-model-id", base_model_id]- # Pass --trust-remote-code only when the model really came from custom code (the approved load decision), not merely because its config carries an auto_map entry.- if _loaded_via_remote_code(model):+ if loaded_via_remote_code:cmd.append("--trust-remote-code")env = os.environ.copy()@@ -5406,6 +5443,8 @@ def _unsloth_save_lora_gguf(finally:if push_to_hub:shutil.rmtree(lora_dir, ignore_errors = True)+ if base_config_dir is not None:+ shutil.rmtree(base_config_dir, ignore_errors = True)def unsloth_convert_lora_to_ggml_and_push_to_hub(