Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Load Ling 3.x low-rank queries on native bailing_hybrid and fix its exports (#14136)

Daniel Han32980a424082
diff --git a/tests/test_bailing_hybrid_low_rank_query.py b/tests/test_bailing_hybrid_low_rank_query.py
new file mode 100644
index 00000000000..08458359e90
--- /dev/null
+++ b/tests/test_bailing_hybrid_low_rank_query.py
@@ -0,0 +1,134 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Ling 3.x checkpoints with `q_lora_rank` load their low-rank query on transformers' bailing_hybrid (#13948)."""
+
+import json
+import os
+import shutil
+
+import pytest
+import torch
+
+import unsloth # noqa: F401
+from unsloth.import_fixes import fix_transformers_bailing_hybrid_low_rank_query
+from unsloth.save import _base_config_dir
+
+modeling = pytest.importorskip("transformers.models.bailing_hybrid.modeling_bailing_hybrid")
+from transformers import BailingHybridConfig, BailingHybridForCausalLM
+
+
+def _config(q_lora_rank):
+ return BailingHybridConfig(
+ vocab_size = 64,
+ pad_token_id = 0,
+ eos_token_id = 1,
+ hidden_size = 32,
+ intermediate_size = 64,
+ moe_intermediate_size = 16,
+ num_hidden_layers = 4,
+ num_attention_heads = 2,
+ num_key_value_heads = 2,
+ head_dim = 8,
+ num_experts = 4,
+ num_experts_per_tok = 2,
+ n_group = 1,
+ topk_group = 1,
+ first_k_dense_replace = 1,
+ layer_group_size = 4,
+ kv_lora_rank = 16,
+ qk_nope_head_dim = 8,
+ qk_rope_head_dim = 4,
+ v_head_dim = 8,
+ q_lora_rank = q_lora_rank,
+ )
+
+
+def _mla(model):
+ return model.model.layers[3].self_attn
+
+
+def test_low_rank_query_replaces_q_proj():
+ fix_transformers_bailing_hybrid_low_rank_query()
+ torch.manual_seed(0)
+ model = BailingHybridForCausalLM(_config(12)).float().eval()
+ keys = set(model.state_dict())
+ assert {
+ "model.layers.3.self_attn.q_a_proj.weight",
+ "model.layers.3.self_attn.q_a_layernorm.weight",
+ "model.layers.3.self_attn.q_b_proj.weight",
+ } <= keys
+ assert "model.layers.3.self_attn.q_proj.weight" not in keys
+ attention = _mla(model)
+ assert attention.q_b_proj.weight.shape == (2 * 12, 12)
+ x = torch.randn(1, 5, 32)
+ expected = attention.q_b_proj(attention.q_a_layernorm(attention.q_a_proj(x)))
+ torch.testing.assert_close(attention.q_proj(x), expected)
+
+
+def test_checkpoint_low_rank_query_loads_without_missing_keys(tmp_path):
+ fix_transformers_bailing_hybrid_low_rank_query()
+ torch.manual_seed(0)
+ model = BailingHybridForCausalLM(_config(12)).float()
+ model.save_pretrained(tmp_path)
+ reloaded, info = BailingHybridForCausalLM.from_pretrained(tmp_path, output_loading_info = True)
+ assert not info["missing_keys"] and not info["unexpected_keys"]
+ torch.testing.assert_close(
+ _mla(reloaded).q_a_proj.weight.float(), _mla(model).q_a_proj.weight.float()
+ )
+
+
+def test_full_rank_query_is_untouched():
+ fix_transformers_bailing_hybrid_low_rank_query()
+ model = BailingHybridForCausalLM(_config(None))
+ attention = _mla(model)
+ assert isinstance(attention.q_proj, torch.nn.Linear)
+ assert not hasattr(attention, "q_a_proj")
+
+
+def test_patch_is_idempotent():
+ fix_transformers_bailing_hybrid_low_rank_query()
+ init = modeling.BailingHybridAttention.__init__
+ fix_transformers_bailing_hybrid_low_rank_query()
+ assert modeling.BailingHybridAttention.__init__ is init
+
+
+def test_base_config_dir_holds_only_the_config(tmp_path, monkeypatch):
+ import huggingface_hub
+
+ cached = tmp_path / "config.json"
+ cached.write_text('{"kda_lower_bound": -5}')
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", lambda *a, **k: str(cached))
+ directory = _base_config_dir("org/model")
+ try:
+ assert os.listdir(directory) == ["config.json"]
+ with open(os.path.join(directory, "config.json")) as f:
+ assert json.load(f) == {"kda_lower_bound": -5}
+ finally:
+ shutil.rmtree(directory)
+
+
+def test_base_config_dir_passes_anonymous_through(tmp_path, monkeypatch):
+ import huggingface_hub
+
+ cached = tmp_path / "config.json"
+ cached.write_text("{}")
+ seen = []
+ monkeypatch.setattr(
+ huggingface_hub,
+ "hf_hub_download",
+ lambda *a, **k: seen.append(k["token"]) or str(cached),
+ )
+ for token in (False, None, "hf_explicit"):
+ shutil.rmtree(_base_config_dir("org/model", token))
+ assert seen == [False, None, "hf_explicit"]
+
+
+def test_base_config_dir_unavailable(monkeypatch):
+ import huggingface_hub
+
+ def missing(*args, **kwargs):
+ raise huggingface_hub.errors.LocalEntryNotFoundError("offline")
+
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", missing)
+ assert _base_config_dir("org/model") is None
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 1bd664b9c29..3d87fab1fdb 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -12604,3 +12604,59 @@ def fix_transformers_longcat_lsa_config():
logger.info(
"Unsloth: LongcatCausalLM configs without a model_type load on transformers' longcat_flash."
)
+
+
+class _BailingHybridLowRankQuery:
+ """`q_b_proj(q_a_layernorm(q_a_proj(x)))` standing in for the attention's `q_proj` call. Not a
+ module, so the state dict holds the three checkpoint tensors and no `q_proj`."""
+
+ __slots__ = ("attention",)
+
+ def __init__(self, attention):
+ self.attention = attention
+
+ def __call__(self, hidden_states):
+ attention = self.attention
+ return attention.q_b_proj(attention.q_a_layernorm(attention.q_a_proj(hidden_states)))
+
+
+def fix_transformers_bailing_hybrid_low_rank_query():
+ """transformers' bailing_hybrid (5.20) builds the full-attention query as one `q_proj`, so a Ling
+ 3.x checkpoint with `q_lora_rank` set (Ling-3.0-tiny, Ling-3.1-flash: `q_a_proj`, `q_a_layernorm`,
+ `q_b_proj` on disk) loads those layers with a randomly initialised query (#13948). Build the
+ low-rank query the checkpoint and its own modeling code carry; `q_lora_rank = None` (Ling-3.0-flash)
+ is untouched. Lazy: called from the loader once the config names bailing_hybrid."""
+ try:
+ from transformers.models.bailing_hybrid import modeling_bailing_hybrid as module
+ except Exception:
+ return
+ cls = getattr(module, "BailingHybridAttention", None)
+ norm = getattr(module, "BailingHybridRMSNorm", None)
+ if cls is None or norm is None or getattr(cls.__init__, "_unsloth_low_rank_query", False):
+ return
+ try:
+ if "q_a_proj" in inspect.getsource(cls):
+ return
+ except Exception:
+ return
+ import torch
+
+ original = cls.__init__
+
+ @functools.wraps(original)
+ def __init__(self, config, *args, **kwargs):
+ original(self, config, *args, **kwargs)
+ rank = getattr(config, "q_lora_rank", None)
+ query = self.__dict__.get("_modules", {}).get("q_proj")
+ if not rank or not isinstance(query, torch.nn.Linear):
+ return
+ del self.q_proj
+ self.q_a_proj = torch.nn.Linear(
+ config.hidden_size, rank, bias = bool(getattr(config, "use_qkv_bias", False))
+ )
+ self.q_a_layernorm = norm(rank) # the checkpoint's own code keeps the default eps
+ self.q_b_proj = torch.nn.Linear(rank, query.out_features, bias = False)
+ object.__setattr__(self, "q_proj", _BailingHybridLowRankQuery(self))
+
+ __init__._unsloth_low_rank_query = True
+ cls.__init__ = __init__
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index e83042a6bb3..452624ba6e0 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -3836,6 +3836,9 @@ class FastBaseModel:
_cfg_val = kwargs.pop("max_position_embeddings", None)
if _cfg_val is not None:
setattr(model_config, "max_position_embeddings", _cfg_val)
+ if "bailing_hybrid" in (model_types or ()):
+ from ..import_fixes import fix_transformers_bailing_hybrid_low_rank_query
+ fix_transformers_bailing_hybrid_low_rank_query()
with _tolerate_dtype_cast_on_quantized_model(
bool(trust_remote_code)
and (
diff --git a/unsloth/save.py b/unsloth/save.py
index 7d55ac73412..f53d243a980 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -5149,6 +5149,32 @@ def _apply_token_to_child_env(env, token, *, explicit):
env["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "0"
+def _base_config_dir(repo_id, token = None):
+ """Temporary dir with only `repo_id`'s config.json (all convert_lora_to_gguf.py --base reads), else None."""
+ import tempfile
+
+ try:
+ from huggingface_hub import hf_hub_download
+
+ # The live cache, not huggingface_hub's frozen constant: a runtime redirect would else miss (#6890).
+ try:
+ from unsloth_zoo.hf_cache import _active_caches
+ cache_dir = _active_caches()[1]
+ except Exception:
+ cache_dir = None
+ config = hf_hub_download(
+ repo_id,
+ "config.json",
+ cache_dir = str(cache_dir) if cache_dir is not None else None,
+ token = token,
+ )
+ except Exception:
+ return None
+ directory = tempfile.mkdtemp(prefix = "unsloth-lora-gguf-base-")
+ shutil.copyfile(config, os.path.join(directory, "config.json"))
+ return directory
+
+
def _lora_base_model_id(model):
"""Base model id for a PEFT model: prefer the active adapter's recorded `base_model_name_or_path`, the authoritative source, else the model config."""
base = None
@@ -5299,6 +5325,7 @@ def _unsloth_save_lora_gguf(
pass
model_name = _model_basename(base_model_id)
+ base_config_dir = None
# Save the adapter: an isolated temp dir for a hub push, else save_directory itself, wrapped so it is always cleaned up.
if push_to_hub:
lora_dir = tempfile.mkdtemp(prefix = "unsloth-lora-gguf-")
@@ -5327,12 +5354,22 @@ def _unsloth_save_lora_gguf(
out_gguf = os.path.join(lora_dir, f"{model_name}-lora-{outtype}.gguf")
cmd = [sys.executable, converter, lora_dir, "--outfile", out_gguf, "--outtype", outtype]
+ # Pass --trust-remote-code only when the model really came from custom code (the approved load decision), not merely because its config carries an auto_map entry.
+ loaded_via_remote_code = _loaded_via_remote_code(model)
+ # --base-model-id's strict AutoConfig rejects an older config (Ling-3.0-tiny's int kda_lower_bound, #13948); --base falls back to config.json.
+ local_base = None
if os.path.isdir(base_model_id):
- cmd += ["--base", base_model_id]
+ local_base = base_model_id
+ elif not loaded_via_remote_code:
+ # False stays anonymous; a token we resolved ourselves is left to huggingface_hub, which honors HF_HUB_DISABLE_IMPLICIT_TOKEN.
+ local_base = base_config_dir = _base_config_dir(
+ base_model_id, token if token is False or token_is_explicit else None
+ )
+ if local_base is not None:
+ cmd += ["--base", local_base]
else:
cmd += ["--base-model-id", base_model_id]
- # Pass --trust-remote-code only when the model really came from custom code (the approved load decision), not merely because its config carries an auto_map entry.
- if _loaded_via_remote_code(model):
+ if loaded_via_remote_code:
cmd.append("--trust-remote-code")
env = os.environ.copy()
@@ -5406,6 +5443,8 @@ def _unsloth_save_lora_gguf(
finally:
if push_to_hub:
shutil.rmtree(lora_dir, ignore_errors = True)
+ if base_config_dir is not None:
+ shutil.rmtree(base_config_dir, ignore_errors = True)
def unsloth_convert_lora_to_ggml_and_push_to_hub(