Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Load and fine-tune LongCat-Flash-Lite-Sparse on transformers' longcat_flash (#11620)

Daniel Hancccba4413e64
diff --git a/tests/test_longcat_lsa.py b/tests/test_longcat_lsa.py
new file mode 100644
index 00000000000..2c57d1a943c
--- /dev/null
+++ b/tests/test_longcat_lsa.py
@@ -0,0 +1,648 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""LongCat-Flash-Lite-Sparse loads via ``unsloth/models/longcat_lsa.py``: load, forward vs an
+SGLang-transcribed reference, cache, and save back to the published layout, on a tiny checkpoint."""
+
+import json
+import math
+import os
+import warnings
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("transformers")
+pytest.importorskip("transformers.models.longcat_flash")
+safetensors_torch = pytest.importorskip("safetensors.torch")
+
+import torch.nn.functional as F # noqa: E402
+from real_accelerator import has_real_cuda # noqa: E402
+from transformers import AutoConfig, AutoModelForCausalLM # noqa: E402
+
+from unsloth.import_fixes import fix_transformers_longcat_lsa_config # noqa: E402
+from unsloth.models.longcat_lsa import ( # noqa: E402
+ LONGCAT_LSA_MODEL_TYPE,
+ is_longcat_lsa_config_dict,
+ register_longcat_lsa,
+)
+
+# The published config.json, verbatim apart from the dimensions shrunk below.
+REAL_CONFIG = {
+ "architectures": ["LongcatCausalLM"],
+ "attention_bias": False,
+ "attention_dropout": 0.0,
+ "vocab_size": 131072,
+ "hidden_size": 3072,
+ "ffn_hidden_size": 6144,
+ "expert_ffn_hidden_size": 1024,
+ "num_layers": 14,
+ "num_attention_heads": 32,
+ "kv_lora_rank": 512,
+ "q_lora_rank": 1536,
+ "qk_rope_head_dim": 64,
+ "v_head_dim": 128,
+ "qk_nope_head_dim": 128,
+ "mla_scale_q_lora": True,
+ "mla_scale_kv_lora": True,
+ "routed_scaling_factor": 6.0,
+ "n_routed_experts": 256,
+ "max_position_embeddings": 983040,
+ "rms_norm_eps": 1e-05,
+ "use_cache": True,
+ "bos_token_id": 1,
+ "eos_token_id": 2,
+ "rope_theta": 1000000.0,
+ "rope_scaling": {
+ "original_max_position_embeddings": 8192,
+ "rope_type": "deepseek_yarn",
+ "factor": 120,
+ "beta_fast": 32,
+ "beta_slow": 1,
+ "mscale": 1,
+ "mscale_all_dim": 1,
+ },
+ "attention_method": "LSA",
+ "zero_expert_num": 128,
+ "zero_expert_type": "identity",
+ "moe_topk": 12,
+ "use_mla": 1,
+ "oe_vocab_size_ratio": 78,
+ "oe_neighbor_num": 4,
+ "oe_split_num": 4,
+ "mtp_num_layers": 1,
+ "index_n_heads": 16,
+ "index_head_dim": 128,
+ "index_topk": 2048,
+ "index_k_norm_type": "rms",
+ "cli_factor": 2,
+ "index_local_tokens": 1024,
+ "index_init_tokens": 16,
+}
+
+TINY = dict(
+ vocab_size = 128,
+ hidden_size = 96,
+ ffn_hidden_size = 64,
+ expert_ffn_hidden_size = 16,
+ num_layers = 2,
+ num_attention_heads = 2,
+ kv_lora_rank = 16,
+ q_lora_rank = 24,
+ qk_rope_head_dim = 8,
+ v_head_dim = 8,
+ qk_nope_head_dim = 8,
+ n_routed_experts = 4,
+ zero_expert_num = 2,
+ moe_topk = 2,
+ oe_vocab_size_ratio = 3,
+ index_n_heads = 2,
+ index_head_dim = 8,
+)
+
+
+def _write_tiny(
+ path,
+ seed = 0,
+ **overrides,
+):
+ cfg = dict(REAL_CONFIG, **TINY, **overrides)
+ g = torch.Generator().manual_seed(seed)
+
+ def w(*shape, scale = 0.1):
+ return torch.randn(*shape, generator = g) * scale
+
+ def norm(n):
+ return 1.0 + 0.1 * torch.randn(n, generator = g)
+
+ H, V = cfg["hidden_size"], cfg["vocab_size"]
+ E, Z = cfg["n_routed_experts"], cfg["zero_expert_num"]
+ I, Fh = cfg["expert_ffn_hidden_size"], cfg["ffn_hidden_size"]
+ nh, ql, kl = cfg["num_attention_heads"], cfg["q_lora_rank"], cfg["kv_lora_rank"]
+ rope, nope, vd = cfg["qk_rope_head_dim"], cfg["qk_nope_head_dim"], cfg["v_head_dim"]
+ sd = {
+ "model.embed_tokens.weight": w(V, H, scale = 0.5),
+ "lm_head.weight": w(V, H),
+ "model.norm.weight": norm(H),
+ }
+ m = int(cfg["oe_vocab_size_ratio"] * V)
+ tables = cfg["oe_split_num"] * (cfg["oe_neighbor_num"] - 1)
+ for i in range(tables):
+ sd[f"model.oe_embed_tokens{i}.weight"] = w(m + 2 * i + 1, H // tables, scale = 0.5)
+ sd[f"model.oe_embed_proj{i}.weight"] = w(H, H // tables)
+
+ def attn(prefix, indexer):
+ sd[prefix + "q_a_proj.weight"] = w(ql, H)
+ sd[prefix + "q_a_layernorm.weight"] = norm(ql)
+ sd[prefix + "q_b_proj.weight"] = w(nh * (nope + rope), ql)
+ sd[prefix + "kv_a_proj_with_mqa.weight"] = w(kl + rope, H)
+ sd[prefix + "kv_a_layernorm.weight"] = norm(kl)
+ sd[prefix + "kv_b_proj.weight"] = w(nh * (nope + vd), kl)
+ sd[prefix + "o_proj.weight"] = w(H, nh * vd)
+ if indexer:
+ d, h = cfg["index_head_dim"], cfg["index_n_heads"]
+ sd[prefix + "indexer.wq_b.weight"] = w(h * d, ql)
+ sd[prefix + "indexer.wk.weight"] = w(d, H)
+ sd[prefix + "indexer.k_norm.weight"] = norm(d)
+ sd[prefix + "indexer.weights_proj.weight"] = w(h, H)
+
+ for layer in range(cfg["num_layers"]):
+ p = f"model.layers.{layer}."
+ for s in range(2):
+ attn(p + f"self_attn.{s}.", indexer = s == 0)
+ sd[p + f"input_layernorm.{s}.weight"] = norm(H)
+ sd[p + f"post_attention_layernorm.{s}.weight"] = norm(H)
+ sd[p + f"mlps.{s}.gate_proj.weight"] = w(Fh, H)
+ sd[p + f"mlps.{s}.up_proj.weight"] = w(Fh, H)
+ sd[p + f"mlps.{s}.down_proj.weight"] = w(H, Fh)
+ sd[p + "mlp.router.classifier.weight"] = w(E + Z, H, scale = 0.5)
+ sd[p + "mlp.router.e_score_correction_bias"] = 0.01 * torch.randn(E + Z, generator = g)
+ for e in range(E):
+ sd[p + f"mlp.experts.{e}.gate_proj.weight"] = w(I, H)
+ sd[p + f"mlp.experts.{e}.up_proj.weight"] = w(I, H)
+ sd[p + f"mlp.experts.{e}.down_proj.weight"] = w(H, I)
+ sd["model.mtp.norm.weight"] = norm(H)
+ sd["model.mtp.layers.0.eh_proj.weight"] = w(H, 2 * H)
+ stored = {
+ k: (v if ("router" in k or "weights_proj" in k) else v.to(torch.bfloat16)).contiguous()
+ for k, v in sd.items()
+ }
+ os.makedirs(path, exist_ok = True)
+ safetensors_torch.save_file(
+ stored, os.path.join(path, "model.safetensors"), metadata = {"format": "pt"}
+ )
+ with open(os.path.join(path, "config.json"), "w") as f:
+ json.dump(cfg, f)
+ return cfg, {k: v.float() for k, v in stored.items()}
+
+
+def _load(path, dtype = torch.float32):
+ register_longcat_lsa()
+ return AutoModelForCausalLM.from_pretrained(path, dtype = dtype)
+
+
+# Reference from SGLang longcat_flash.py, deepseek_v2 MLA, deepseek_yarn, ngram_embedding.cuh.
+
+
+def _ref_ngram_ids(cfg, tokens):
+ V, eos = cfg["vocab_size"], cfg["eos_token_id"]
+ k_, n_ = cfg["oe_split_num"], cfg["oe_neighbor_num"]
+ m = int(cfg["oe_vocab_size_ratio"] * V)
+ ids = torch.zeros(len(tokens), (n_ - 1) * k_, dtype = torch.long)
+ for n in range(n_ - 1):
+ for k in range(k_):
+ idx = n * k_ + k
+ mod = m + 2 * idx + 1
+ for i in range(len(tokens)):
+ acc = 0
+ for j in range(n + 2):
+ if i - j < 0:
+ break
+ tok = int(tokens[i - j])
+ if tok == eos and j > 0: # the context never crosses an earlier EOS
+ break
+ acc += (tok * pow(V, j, mod)) % mod
+ ids[i, idx] = acc % mod
+ return ids
+
+
+def _ref_forward(cfg, sd, tokens):
+ eps, H = cfg["rms_norm_eps"], cfg["hidden_size"]
+ nh, nope, rope, vd = (
+ cfg["num_attention_heads"],
+ cfg["qk_nope_head_dim"],
+ cfg["qk_rope_head_dim"],
+ cfg["v_head_dim"],
+ )
+ T = len(tokens)
+
+ def rms(x, w):
+ return x * torch.rsqrt(x.pow(2).mean(-1, keepdim = True) + eps) * w
+
+ def mscale(scale, m):
+ return 1.0 if scale <= 1 else 0.1 * m * math.log(scale) + 1.0
+
+ rs = cfg["rope_scaling"]
+ factor, omax, base = rs["factor"], rs["original_max_position_embeddings"], cfg["rope_theta"]
+
+ def corr(rot):
+ return (rope * math.log(omax / (rot * 2 * math.pi))) / (2 * math.log(base))
+
+ low = max(math.floor(corr(rs["beta_fast"])), 0)
+ high = min(math.ceil(corr(rs["beta_slow"])), rope - 1)
+ pos_freqs = base ** (torch.arange(0, rope, 2, dtype = torch.float) / rope)
+ keep = 1 - ((torch.arange(rope // 2, dtype = torch.float) - low) / (high - low)).clamp(0, 1)
+ inv_freq = (1.0 / (factor * pos_freqs)) * (1 - keep) + (1.0 / pos_freqs) * keep
+ freqs = torch.arange(T, dtype = torch.float)[:, None] * inv_freq[None]
+ cos, sin = freqs.cos()[:, None], freqs.sin()[:, None]
+
+ def rotate(x): # interleaved pairs (is_neox_style = False)
+ x1, x2 = x[..., 0::2], x[..., 1::2]
+ return torch.stack((x1 * cos - x2 * sin, x2 * cos + x1 * sin), -1).flatten(-2)
+
+ scaling = (nope + rope) ** -0.5 * mscale(factor, rs["mscale_all_dim"]) ** 2
+ q_scale, kv_scale = (H / cfg["q_lora_rank"]) ** 0.5, (H / cfg["kv_lora_rank"]) ** 0.5
+
+ def mla(p, x):
+ q = rms(x @ sd[p + "q_a_proj.weight"].T, sd[p + "q_a_layernorm.weight"] * q_scale)
+ q_nope, q_pe = (q @ sd[p + "q_b_proj.weight"].T).view(T, nh, -1).split([nope, rope], -1)
+ kv_a, k_pe = (x @ sd[p + "kv_a_proj_with_mqa.weight"].T).split(
+ [cfg["kv_lora_rank"], rope], -1
+ )
+ kv = rms(kv_a, sd[p + "kv_a_layernorm.weight"] * kv_scale) @ sd[p + "kv_b_proj.weight"].T
+ k_nope, v = kv.view(T, nh, -1).split([nope, vd], -1)
+ q = torch.cat([q_nope, rotate(q_pe)], -1)
+ k = torch.cat([k_nope, rotate(k_pe[:, None]).expand(T, nh, rope)], -1)
+ s = torch.einsum("qhd,khd->hqk", q, k) * scaling
+ # kv_len <= index_topk: the indexer's top-k selects every causal position.
+ s = s.masked_fill(~torch.ones(T, T, dtype = torch.bool).tril(), float("-inf"))
+ o = torch.einsum("hqk,khd->qhd", s.softmax(-1), v).reshape(T, -1)
+ return o @ sd[p + "o_proj.weight"].T
+
+ def mlp(p, x):
+ gate = F.silu(x @ sd[p + "gate_proj.weight"].T)
+ return (gate * (x @ sd[p + "up_proj.weight"].T)) @ sd[p + "down_proj.weight"].T
+
+ def moe(p, x):
+ E, rsf = cfg["n_routed_experts"], cfg["routed_scaling_factor"]
+ scores = (x @ sd[p + "router.classifier.weight"].T).softmax(-1)
+ choice = scores + sd[p + "router.e_score_correction_bias"][None]
+ ids = torch.topk(choice, cfg["moe_topk"], dim = -1)[1]
+ weights = scores.gather(1, ids) * rsf # identity experts scaled as in transformers / vLLM
+ out = torch.zeros_like(x)
+ for t in range(T):
+ for j in range(ids.shape[1]):
+ e = int(ids[t, j])
+ y = mlp(p + f"experts.{e}.", x[t : t + 1])[0] if e < E else x[t]
+ out[t] += weights[t, j] * y
+ return out
+
+ ng = _ref_ngram_ids(cfg, tokens)
+ parts = [sd["model.embed_tokens.weight"][tokens]]
+ for i in range(ng.shape[1]):
+ rows = sd[f"model.oe_embed_tokens{i}.weight"][ng[:, i]]
+ parts.append(rows @ sd[f"model.oe_embed_proj{i}.weight"].T)
+ h, residual = torch.stack(parts).mean(0), None
+
+ def add_norm(h, residual, w):
+ residual = h if residual is None else h + residual
+ return rms(residual, w), residual
+
+ for layer in range(cfg["num_layers"]):
+ p = f"model.layers.{layer}."
+ x, residual = add_norm(h, residual, sd[p + "input_layernorm.0.weight"])
+ x, residual = add_norm(
+ mla(p + "self_attn.0.", x), residual, sd[p + "post_attention_layernorm.0.weight"]
+ )
+ shortcut = moe(p + "mlp.", x)
+ x, residual = add_norm(mlp(p + "mlps.0.", x), residual, sd[p + "input_layernorm.1.weight"])
+ x, residual = add_norm(
+ mla(p + "self_attn.1.", x), residual, sd[p + "post_attention_layernorm.1.weight"]
+ )
+ h = shortcut + mlp(p + "mlps.1.", x)
+ x, _ = add_norm(h, residual, sd["model.norm.weight"])
+ return x @ sd["lm_head.weight"].T
+
+
+def _tokens(
+ cfg,
+ batch,
+ length,
+ seed = 1,
+):
+ g = torch.Generator().manual_seed(seed)
+ ids = torch.randint(3, cfg["vocab_size"], (batch, length), generator = g)
+ eos = cfg["eos_token_id"]
+ ids[0, 5] = eos
+ ids[0, 6] = eos # back-to-back EOS
+ if batch > 1:
+ ids[1, 11] = eos
+ return ids
+
+
[email protected](scope = "module")
+def tiny(tmp_path_factory):
+ path = str(tmp_path_factory.mktemp("longcat_lsa_tiny"))
+ cfg, sd = _write_tiny(path)
+ return path, cfg, sd
+
+
+def test_recognises_only_this_config():
+ assert is_longcat_lsa_config_dict(REAL_CONFIG)
+ assert is_longcat_lsa_config_dict({"model_type": LONGCAT_LSA_MODEL_TYPE})
+ lite = dict(REAL_CONFIG, model_type = "longcat_flash_ngram")
+ assert not is_longcat_lsa_config_dict(lite)
+ no_ngram = {k: v for k, v in REAL_CONFIG.items() if k != "oe_vocab_size_ratio"}
+ assert not is_longcat_lsa_config_dict(no_ngram)
+ assert not is_longcat_lsa_config_dict({"architectures": ["LlamaForCausalLM"]})
+ assert not is_longcat_lsa_config_dict(None)
+
+
+def test_autoconfig_loads_the_published_config(tmp_path):
+ fix_transformers_longcat_lsa_config()
+ with open(tmp_path / "config.json", "w") as f:
+ json.dump(REAL_CONFIG, f)
+ config = AutoConfig.from_pretrained(str(tmp_path))
+ assert config.model_type == LONGCAT_LSA_MODEL_TYPE
+ rope = getattr(config, "rope_parameters", None) or config.rope_scaling
+ assert rope["rope_type"] == "yarn" # SGLang's deepseek_yarn is transformers' yarn
+ assert config.head_dim == config.qk_rope_head_dim == 64
+ assert config.oe_vocab_size_ratio == 78 and config.index_topk == 2048
+
+ other = tmp_path / "other"
+ other.mkdir()
+ with open(other / "config.json", "w") as f:
+ json.dump({"architectures": ["SomethingElse"]}, f)
+ with pytest.raises(ValueError):
+ AutoConfig.from_pretrained(str(other))
+
+
+def test_every_checkpoint_key_loads(tiny):
+ path, cfg, sd = tiny
+ register_longcat_lsa()
+ model, info = AutoModelForCausalLM.from_pretrained(
+ path, dtype = torch.float32, output_loading_info = True
+ )
+ assert not info["missing_keys"], info["missing_keys"]
+ assert all(k.startswith("model.mtp.") for k in info["unexpected_keys"]), info
+ assert not info.get("mismatched_keys"), info["mismatched_keys"]
+ layer = model.model.layers[0]
+ experts = layer.mlp.experts
+ p = "model.layers.0.mlp.experts.3."
+ if hasattr(experts, "gate_up_proj"): # transformers 5 stacks the experts
+ assert experts.gate_up_proj.shape[0] == cfg["n_routed_experts"]
+ gate_up = torch.cat([sd[p + "gate_proj.weight"], sd[p + "up_proj.weight"]], 0)
+ assert torch.equal(experts.gate_up_proj[3], gate_up)
+ assert torch.equal(experts.down_proj[3], sd[p + "down_proj.weight"])
+ else:
+ assert torch.equal(experts[3].gate_proj.weight, sd[p + "gate_proj.weight"])
+ ngram = model.model.ngram_embeddings
+ assert torch.equal(ngram.embedders[7].weight, sd["model.oe_embed_tokens7.weight"])
+ assert torch.equal(ngram.post_projs[11].weight, sd["model.oe_embed_proj11.weight"])
+ indexer = layer.self_attn[0].indexer
+ assert torch.equal(indexer.wq_b.weight, sd["model.layers.0.self_attn.0.indexer.wq_b.weight"])
+ assert not indexer.wq_b.weight.requires_grad
+
+
+def test_forward_matches_sglang_reference(tiny):
+ path, cfg, sd = tiny
+ model = _load(path).eval()
+ ids = _tokens(cfg, 2, 19)
+ with torch.no_grad():
+ logits = model(input_ids = ids).logits
+ for b in range(ids.shape[0]):
+ ref = _ref_forward(cfg, sd, ids[b])
+ torch.testing.assert_close(logits[b], ref, atol = 2e-4, rtol = 1e-4)
+
+
+def test_ngram_ids_follow_the_sglang_kernel(tiny):
+ path, cfg, sd = tiny
+ model = _load(path).eval()
+ seen = []
+ handles = [
+ e.register_forward_hook(lambda mod, args, out: seen.append(args[0].clone()))
+ for e in model.model.ngram_embeddings.embedders
+ ]
+ ids = _tokens(cfg, 2, 15)
+ with torch.no_grad():
+ model.model.ngram_embeddings(model.model.embed_tokens(ids), ids)
+ for h in handles:
+ h.remove()
+ ours = torch.stack(seen, -1)
+ for b in range(ids.shape[0]):
+ assert torch.equal(ours[b], _ref_ngram_ids(cfg, ids[b]))
+
+
[email protected](not has_real_cuda(), reason = "needs a CUDA device")
+def test_split_model_keeps_the_token_table_where_accelerate_put_it(tiny):
+ # Accelerate hooks move every `.to`-able arg: never pass the embed_tokens module itself.
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+
+ path, cfg, sd = tiny
+ model = _load(path).eval().to("cuda")
+ ids = _tokens(cfg, 2, 15).to("cuda")
+ with torch.no_grad():
+ expected = model(input_ids = ids).logits.cpu()
+ model.model.embed_tokens.to("cpu")
+ add_hook_to_module(model.model.embed_tokens, AlignDevicesHook(execution_device = "cpu"))
+ add_hook_to_module(model.model.ngram_embeddings, AlignDevicesHook(execution_device = "cuda"))
+ with torch.no_grad():
+ got = model(input_ids = ids).logits.cpu()
+ assert model.model.embed_tokens.weight.device.type == "cpu"
+ torch.testing.assert_close(got, expected, atol = 1e-4, rtol = 1e-4)
+
+
+def test_cached_decode_matches_full_forward(tiny):
+ path, cfg, sd = tiny
+ model = _load(path).eval()
+ ids = _tokens(cfg, 1, 14)
+ with torch.no_grad():
+ full = model(input_ids = ids).logits[0]
+ out = model(input_ids = ids[:, :9], use_cache = True)
+ cache, steps = out.past_key_values, [out.logits[0]]
+ for t in range(9, ids.shape[1]): # decode across the n-gram context boundary
+ out = model(input_ids = ids[:, t : t + 1], past_key_values = cache, use_cache = True)
+ cache = out.past_key_values
+ steps.append(out.logits[0])
+ torch.testing.assert_close(torch.cat(steps), full, atol = 1e-4, rtol = 1e-4)
+
+
+def test_packed_training_rows_do_not_see_each_other(tiny):
+ path, cfg, sd = tiny
+ model = _load(path).train()
+ a, b = _tokens(cfg, 2, 12, seed = 3)
+ b[0] = a[-1] # b's first n-grams would read a's tail if the n-gram crossed the boundary
+ packed = torch.cat([a, b])[None]
+ position_ids = torch.cat([torch.arange(12), torch.arange(12)])[None]
+ with torch.no_grad():
+ out = model(input_ids = packed, position_ids = position_ids)
+ alone = model(input_ids = b[None]).logits[0]
+ assert out.past_key_values is None
+ torch.testing.assert_close(out.logits[0, 12:], alone, atol = 1e-4, rtol = 1e-4)
+
+
+def test_left_padded_generation_matches_unpadded(tiny):
+ path, cfg, sd = tiny
+ model = _load(path).eval()
+ ids = _tokens(cfg, 1, 10, seed = 4)
+ padded = torch.cat([torch.full((1, 3), 7), ids], dim = -1)
+ mask = torch.cat([torch.zeros(1, 3, dtype = torch.long), torch.ones_like(ids)], dim = -1)
+ kwargs = dict(max_new_tokens = 5, do_sample = False, pad_token_id = 0)
+ with torch.no_grad():
+ want = model.generate(input_ids = ids, **kwargs)[0, 10:]
+ got = model.generate(input_ids = padded, attention_mask = mask, **kwargs)[0, 13:]
+ assert got.tolist() == want.tolist()
+
+
+def test_left_padded_mask_without_position_ids_matches_unpadded(tiny):
+ path, cfg, sd = tiny
+ model = _load(path).eval()
+ ids = _tokens(cfg, 1, 10, seed = 4)
+ padded = torch.cat([torch.full((1, 3), 7), ids], dim = -1)
+ mask = torch.cat([torch.zeros(1, 3, dtype = torch.long), torch.ones_like(ids)], dim = -1)
+ with torch.no_grad():
+ want = model(input_ids = ids).logits[0]
+ got = model(input_ids = padded, attention_mask = mask).logits[0, 3:]
+ torch.testing.assert_close(got, want, atol = 1e-4, rtol = 1e-4)
+
+
+def test_resized_vocab_saves_and_reloads(tiny, tmp_path):
+ path, cfg, sd = tiny
+ model = _load(path).eval()
+ model.resize_token_embeddings(cfg["vocab_size"] + 8)
+ model.save_pretrained(str(tmp_path))
+ reloaded = _load(str(tmp_path)).eval()
+ ids = _tokens(cfg, 1, 12, seed = 5)
+ with torch.no_grad():
+ torch.testing.assert_close(
+ reloaded(input_ids = ids).logits, model(input_ids = ids).logits, atol = 1e-5, rtol = 1e-5
+ )
+
+
+def test_ngram_history_follows_beam_reorder_and_crop(tiny):
+ path, cfg, sd = tiny
+ model = _load(path).eval()
+ ids = _tokens(cfg, 2, 14)
+ swapped = ids.flip(0)
+ with torch.no_grad():
+ full = model(input_ids = swapped).logits
+ cache = model(input_ids = ids[:, :9], use_cache = True).past_key_values
+ cache.reorder_cache(torch.tensor([1, 0]))
+ out = model(input_ids = swapped[:, 9:10], past_key_values = cache, use_cache = True)
+ torch.testing.assert_close(out.logits[:, -1], full[:, 9], atol = 1e-4, rtol = 1e-4)
+ cache = out.past_key_values
+ cache.crop(9)
+ out = model(input_ids = swapped[:, 9:11], past_key_values = cache, use_cache = True)
+ torch.testing.assert_close(out.logits[:, -1], full[:, 10], atol = 1e-4, rtol = 1e-4)
+
+
+def test_cache_reset_clears_the_ngram_history(tiny):
+ import transformers
+ from transformers.cache_utils import StaticCache
+
+ if int(transformers.__version__.split(".")[0]) < 5:
+ pytest.skip("transformers 4's StaticCache cannot hold MLA's differing key/value head dims")
+ path, cfg, sd = tiny
+ model = _load(path).eval()
+ first, second = _tokens(cfg, 1, 9, seed = 1), _tokens(cfg, 1, 9, seed = 2)
+ try:
+ cache = StaticCache(config = model.config, max_cache_len = 32)
+ except TypeError:
+ cache = StaticCache(config = model.config, max_batch_size = 1, max_cache_len = 32)
+ with torch.no_grad():
+ expected = model(input_ids = second).logits
+ model(input_ids = first, past_key_values = cache, use_cache = True)
+ cache.reset()
+ got = model(input_ids = second, past_key_values = cache, use_cache = True).logits
+ torch.testing.assert_close(got, expected, atol = 1e-4, rtol = 1e-4)
+
+
+def test_inputs_embeds_is_refused(tiny):
+ path, cfg, sd = tiny
+ model = _load(path).eval()
+ ids = _tokens(cfg, 1, 9)
+ with pytest.raises(ValueError, match = "input_ids"):
+ model(inputs_embeds = model.model.embed_tokens(ids))
+
+
+def test_bf16_keeps_router_fp32_and_sglang_norm_eps(tiny):
+ path, cfg, sd = tiny
+ model = _load(path, dtype = torch.bfloat16)
+ layer = model.model.layers[0]
+ assert layer.mlp.router.classifier.weight.dtype == torch.float32
+ assert layer.self_attn[0].indexer.weights_proj.weight.dtype == torch.float32
+ assert layer.self_attn[0].q_a_proj.weight.dtype == torch.bfloat16
+ for attn in layer.self_attn:
+ assert attn.q_a_layernorm.variance_epsilon == cfg["rms_norm_eps"]
+ assert attn.kv_a_layernorm.variance_epsilon == cfg["rms_norm_eps"]
+
+
+def test_save_writes_the_published_layout(tiny, tmp_path):
+ path, cfg, sd = tiny
+ model = _load(path, dtype = torch.bfloat16)
+ model.save_pretrained(str(tmp_path))
+ saved = {}
+ for name in os.listdir(tmp_path):
+ if name.endswith(".safetensors"):
+ saved.update(safetensors_torch.load_file(str(tmp_path / name)))
+ original = safetensors_torch.load_file(os.path.join(path, "model.safetensors"))
+ expected = {k for k in original if not k.startswith("model.mtp.")}
+ assert set(saved) == expected
+ for key in expected:
+ assert saved[key].dtype == original[key].dtype, key
+ assert torch.equal(saved[key], original[key]), key
+ with open(tmp_path / "config.json") as f:
+ assert json.load(f)["model_type"] == LONGCAT_LSA_MODEL_TYPE
+
+
+def test_peft_converts_per_expert_adapters_like_longcat_flash():
+ register_longcat_lsa()
+ conversion = pytest.importorskip("transformers.conversion_mapping")
+ table = getattr(conversion, "_MODEL_TO_CONVERSION_PATTERN", None)
+ if not isinstance(table, dict) or "longcat_flash" not in table:
+ pytest.skip("this transformers has no model-type conversion table")
+ assert table[LONGCAT_LSA_MODEL_TYPE] == table["longcat_flash"]
+ try:
+ from peft.utils import transformers_weight_conversion as peft_conversion
+ except Exception:
+ return
+ assert (
+ peft_conversion._MODEL_TO_CONVERSION_PATTERN.get(LONGCAT_LSA_MODEL_TYPE)
+ == (table["longcat_flash"])
+ )
+
+
+def test_long_sequence_warns_once(tmp_path):
+ cfg, sd = _write_tiny(str(tmp_path), index_topk = 8)
+ model = _load(str(tmp_path)).eval()
+ ids = _tokens(cfg, 1, 12)
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ with torch.no_grad():
+ model(input_ids = ids)
+ model(input_ids = ids)
+ messages = [str(w.message) for w in caught if "sparse attention" in str(w.message)]
+ assert len(messages) == 1, messages
+
+
+def test_cached_decode_past_index_topk_warns(tmp_path):
+ cfg, sd = _write_tiny(str(tmp_path), index_topk = 8)
+ model = _load(str(tmp_path)).eval()
+ ids = _tokens(cfg, 1, 12)
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
+ with torch.no_grad():
+ cache = model(input_ids = ids[:, :8], use_cache = True).past_key_values
+ model(input_ids = ids[:, 8:9], past_key_values = cache, use_cache = True)
+ messages = [str(w.message) for w in caught if "sparse attention" in str(w.message)]
+ assert len(messages) == 1 and "9 token" in messages[0], messages
+
+
+def test_4bit_keeps_the_mla_up_projections_in_16bit():
+ # The device-map planner must size q_b_proj / kv_b_proj unquantized, as the load keeps them.
+ import ast
+
+ from unsloth.models.vision import _architecture_skip_modules
+
+ for model_type in ("longcat_flash", "longcat_flash_lsa"):
+ assert {"q_b_proj", "kv_b_proj"} <= set(_architecture_skip_modules([model_type]))
+ assert _architecture_skip_modules(["llama"]) == []
+ path = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ "unsloth",
+ "models",
+ "vision.py",
+ )
+ with open(path, encoding = "utf-8") as f:
+ tree = ast.parse(f.read())
+ planner = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "id", None) == "planner_quantization_kwargs"
+ ]
+ assert planner
+ for call in planner:
+ extra = next(k.value for k in call.keywords if k.arg == "extra_skip_modules")
+ assert "_architecture_skip_modules" in ast.unparse(extra)
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 88e4fadc5e1..53f30b15998 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -254,6 +254,7 @@ from .import_fixes import (
fix_transformers_composite_prefix_renaming,
fix_transformers_fully_masked_rows,
fix_transformers_chunked_mask_block_sequence_ids,
+ fix_transformers_longcat_lsa_config,
fix_transformers_rope_scaling_drops_theta,
fix_transformers_validate_rope_ignore_keys,
fix_transformers5_remote_code_legacy_defaults,
@@ -322,6 +323,7 @@ fix_transformers_rope_scaling_drops_theta()
fix_transformers_validate_rope_ignore_keys()
fix_transformers5_remote_code_legacy_defaults()
fix_transformers_config_only_remote_code()
+fix_transformers_longcat_lsa_config()
# Remote code written for 4.x reads plain RoPE as rope_scaling None and imports is_torch_fx_available.
fix_transformers_remote_rope_scaling_none()
fix_transformers_is_torch_fx_available()
@@ -381,6 +383,7 @@ patch_accelerate_recursively_apply()
del fix_transformers5_bare_annotation_configs
del fix_transformers_rope_scaling_drops_theta
del fix_transformers_validate_rope_ignore_keys
+del fix_transformers_longcat_lsa_config
del fix_transformers_remote_rope_scaling_none
del fix_transformers_is_torch_fx_available
del fix_xformers_performance_issue
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index a923e00b2e8..f0d5ea35f2e 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -10186,3 +10186,63 @@ def disable_sentencepiece_on_windows():
f"{DISABLE_SENTENCEPIECE_VARIABLE}=0 to import it again."
)
return True
+
+
+def fix_transformers_longcat_lsa_config():
+ """Answer AutoConfig's "Unrecognized model" on a LongcatCausalLM config (no model_type,
+ auto_map or modeling code) with ``models/longcat_lsa.py``; all other loads are untouched."""
+ try:
+ from transformers import AutoConfig
+ from transformers.configuration_utils import PretrainedConfig
+ except Exception:
+ return
+ current = AutoConfig.__dict__.get("from_pretrained")
+ original = getattr(current, "__func__", None)
+ if original is None or getattr(original, "_unsloth_longcat_lsa", False):
+ return
+ try:
+ import transformers.models.longcat_flash # noqa: F401
+ except Exception:
+ return
+
+ def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
+ try:
+ return original(cls, pretrained_model_name_or_path, *args, **kwargs)
+ except ValueError as error:
+ message = str(error)
+ if "Unrecognized model" not in message and "longcat_flash_lsa" not in message:
+ raise
+ from .models.longcat_lsa import (
+ is_longcat_lsa_config_dict,
+ load_longcat_lsa_config,
+ )
+
+ hub_kwargs = {
+ key: kwargs[key]
+ for key in (
+ "cache_dir",
+ "force_download",
+ "local_files_only",
+ "token",
+ "revision",
+ "subfolder",
+ )
+ if key in kwargs
+ }
+ try:
+ config_dict, _ = PretrainedConfig.get_config_dict(
+ pretrained_model_name_or_path, **hub_kwargs
+ )
+ except Exception:
+ raise error
+ if not is_longcat_lsa_config_dict(config_dict):
+ raise
+ return load_longcat_lsa_config(pretrained_model_name_or_path, *args, **kwargs)
+
+ from_pretrained._unsloth_longcat_lsa = True
+ from_pretrained.__wrapped__ = original
+ AutoConfig.from_pretrained = classmethod(from_pretrained)
+ if UNSLOTH_ENABLE_LOGGING:
+ logger.info(
+ "Unsloth: LongcatCausalLM configs without a model_type load on transformers' longcat_flash."
+ )
diff --git a/unsloth/models/longcat_lsa.py b/unsloth/models/longcat_lsa.py
new file mode 100644
index 00000000000..4bc53fd27c6
--- /dev/null
+++ b/unsloth/models/longcat_lsa.py
@@ -0,0 +1,510 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""LongCat-Flash-Lite-Sparse (``LongcatCausalLM``, no model_type/auto_map/modeling code) on transformers' ``longcat_flash``.
+
+At seq_len <= index_topk the LSA indexer selects every causal position, so the model is the dense
+Lite network; the indexer weights are kept frozen only to load/save. Longer sequences train dense
+(differs from sparse inference; warned once). Identity experts are scaled by routed_scaling_factor
+as in transformers/vLLM (SGLang leaves them unscaled). The MTP head is not loaded.
+"""
+
+import sys
+import warnings
+
+import torch
+from torch import nn
+
+__all__ = [
+ "LONGCAT_LSA_MODEL_TYPE",
+ "is_longcat_lsa_config_dict",
+ "register_longcat_lsa",
+]
+
+LONGCAT_LSA_MODEL_TYPE = "longcat_flash_lsa"
+_LONGCAT_LSA_ARCHITECTURES = ("LongcatCausalLM",)
+
+# Plain prefixes (no regex groups) so the mapping also runs backwards on save.
+_OE_RENAMES = (
+ ("model.oe_embed_tokens", "model.ngram_embeddings.embedders."),
+ ("model.oe_embed_proj", "model.ngram_embeddings.post_projs."),
+)
+
+
+def is_longcat_lsa_config_dict(config_dict) -> bool:
+ """Config saved by this module, or the published one (``LongcatCausalLM``, no model_type)."""
+ if not isinstance(config_dict, dict):
+ return False
+ model_type = config_dict.get("model_type")
+ if model_type == LONGCAT_LSA_MODEL_TYPE:
+ return True
+ if model_type:
+ return False
+ architectures = config_dict.get("architectures") or []
+ if not any(arch in _LONGCAT_LSA_ARCHITECTURES for arch in architectures):
+ return False
+ return config_dict.get("oe_vocab_size_ratio") is not None or (
+ config_dict.get("ngram_vocab_size_ratio") is not None
+ )
+
+
+def _translate_rope_scaling(rope_scaling):
+ if not isinstance(rope_scaling, dict):
+ return rope_scaling
+ rope_scaling = dict(rope_scaling)
+ rope_type = rope_scaling.pop("type", None) or rope_scaling.get("rope_type")
+ if rope_type == "deepseek_yarn":
+ rope_type = "yarn"
+ if rope_type is not None:
+ rope_scaling["rope_type"] = rope_type
+ return rope_scaling
+
+
+_CLASSES = None
+
+
+_NGRAM_TRACKING_CACHES = {}
+
+
+def _ngram_tracking_cache_class(cls):
+ """``cls`` whose reorder/select/repeat/crop also move the n-gram token history."""
+ tracked = _NGRAM_TRACKING_CACHES.get(cls)
+ if tracked is not None:
+ return tracked
+
+ class NgramTrackingCache(cls):
+ def reorder_cache(self, beam_idx, *args, **kwargs):
+ out = super().reorder_cache(beam_idx, *args, **kwargs)
+ history = getattr(self, "_unsloth_ngram_history", None)
+ if history is not None:
+ self._unsloth_ngram_history = history.index_select(0, beam_idx.to(history.device))
+ return out
+
+ def batch_select_indices(self, indices, *args, **kwargs):
+ out = super().batch_select_indices(indices, *args, **kwargs)
+ history = getattr(self, "_unsloth_ngram_history", None)
+ if history is not None:
+ if torch.is_tensor(indices):
+ indices = indices.to(history.device)
+ self._unsloth_ngram_history = history[indices]
+ return out
+
+ def batch_repeat_interleave(self, repeats, *args, **kwargs):
+ out = super().batch_repeat_interleave(repeats, *args, **kwargs)
+ history = getattr(self, "_unsloth_ngram_history", None)
+ if history is not None:
+ self._unsloth_ngram_history = history.repeat_interleave(repeats, dim = 0)
+ return out
+
+ def reset(self, *args, **kwargs):
+ out = super().reset(*args, **kwargs)
+ self._unsloth_ngram_history = None
+ return out
+
+ def crop(self, *args, **kwargs):
+ out = super().crop(*args, **kwargs)
+ history = getattr(self, "_unsloth_ngram_history", None)
+ if history is not None:
+ self._unsloth_ngram_history = history[..., : self.get_seq_length()]
+ return out
+
+ NgramTrackingCache.__name__ = cls.__name__
+ NgramTrackingCache.__qualname__ = cls.__qualname__
+ _NGRAM_TRACKING_CACHES[cls] = NgramTrackingCache
+ return NgramTrackingCache
+
+
+def _classes():
+ """Build classes lazily so importing Unsloth does not import longcat_flash."""
+ global _CLASSES
+ if _CLASSES is not None:
+ return _CLASSES
+
+ from transformers.models.longcat_flash.configuration_longcat_flash import (
+ LongcatFlashConfig,
+ )
+ from transformers.models.longcat_flash.modeling_longcat_flash import (
+ LongcatFlashForCausalLM,
+ LongcatFlashModel,
+ LongcatFlashPreTrainedModel,
+ )
+
+ class LongcatLsaConfig(LongcatFlashConfig):
+ model_type = LONGCAT_LSA_MODEL_TYPE
+
+ def __init__(
+ self,
+ oe_vocab_size_ratio = None,
+ oe_neighbor_num = None,
+ oe_split_num = None,
+ index_n_heads = None,
+ index_head_dim = None,
+ index_topk = None,
+ oe_ngram_vocab_size = None,
+ **kwargs,
+ ):
+ if oe_vocab_size_ratio is None:
+ oe_vocab_size_ratio = kwargs.pop("ngram_vocab_size_ratio", None)
+ if oe_neighbor_num is None:
+ oe_neighbor_num = kwargs.pop("emb_neighbor_num", None)
+ if oe_split_num is None:
+ oe_split_num = kwargs.pop("emb_split_num", None)
+ self.oe_vocab_size_ratio = oe_vocab_size_ratio
+ self.oe_neighbor_num = oe_neighbor_num
+ self.oe_split_num = oe_split_num
+ self.index_n_heads = index_n_heads
+ self.index_head_dim = index_head_dim
+ self.index_topk = index_topk
+ if "rope_scaling" in kwargs:
+ kwargs["rope_scaling"] = _translate_rope_scaling(kwargs["rope_scaling"])
+ if isinstance(kwargs.get("rope_parameters"), dict):
+ kwargs["rope_parameters"] = _translate_rope_scaling(kwargs["rope_parameters"])
+ # Configs omit head_dim; the rotary table must use qk_rope_head_dim.
+ if "head_dim" not in kwargs and "qk_rope_head_dim" in kwargs:
+ kwargs["head_dim"] = kwargs["qk_rope_head_dim"]
+ super().__init__(**kwargs)
+ # The n-gram hash radix and table sizes; kept apart from vocab_size so a resized vocab reloads.
+ self.oe_ngram_vocab_size = oe_ngram_vocab_size or self.vocab_size
+
+ class _FrozenWeight(nn.Module):
+ """Bare ``weight``, not a Linear, so LoRA all-linear and bitsandbytes skip it."""
+
+ def __init__(self, *shape):
+ super().__init__()
+ self.weight = nn.Parameter(torch.empty(*shape), requires_grad = False)
+
+ class LongcatLsaIndexer(nn.Module):
+ """Lightning indexer weights; unused by the dense forward."""
+
+ def __init__(self, config):
+ super().__init__()
+ heads, head_dim = config.index_n_heads, config.index_head_dim
+ self.wq_b = _FrozenWeight(heads * head_dim, config.q_lora_rank)
+ self.wk = _FrozenWeight(head_dim, config.hidden_size)
+ self.k_norm = _FrozenWeight(head_dim)
+ self.weights_proj = _FrozenWeight(heads, config.hidden_size)
+
+ class LongcatNgramEmbedding(nn.Module):
+ """Lite's ``NgramEmbedding``. Takes embed_tokens' output, not the module: an accelerate
+ hook would otherwise move the whole token table to this card."""
+
+ def __init__(self, config):
+ super().__init__()
+ self.vocab_size = getattr(config, "oe_ngram_vocab_size", None) or config.vocab_size
+ self.m = int(config.oe_vocab_size_ratio * self.vocab_size)
+ self.k = int(config.oe_split_num)
+ self.n = int(config.oe_neighbor_num)
+ eos = config.eos_token_id
+ self.eos_token_id = eos[0] if isinstance(eos, (list, tuple)) else eos
+ num_embedders = self.k * (self.n - 1)
+ emb_dim = config.hidden_size // num_embedders
+ self.embedders = nn.ModuleList(
+ nn.Embedding(int(self.m + i * 2 + 1), emb_dim) for i in range(num_embedders)
+ )
+ self.post_projs = nn.ModuleList(
+ nn.Linear(emb_dim, config.hidden_size, bias = False) for _ in range(num_embedders)
+ )
+
+ def _shifted(
+ self,
+ context,
+ shift,
+ resets = None,
+ ):
+ """``context`` shifted right within each segment, zero outside it. A segment ends at an
+ EOS or a -1 history mark, and starts again where ``resets`` is set."""
+ length = context.shape[-1]
+ positions = torch.arange(length, device = context.device)
+ ends = torch.where(
+ (context == self.eos_token_id) | (context < 0),
+ positions + 1,
+ torch.zeros_like(positions),
+ )
+ starts = torch.cummax(ends, dim = -1).values
+ starts = torch.cat([torch.zeros_like(starts[..., :1]), starts[..., :-1]], dim = -1)
+ if resets is not None:
+ reset_starts = torch.where(resets, positions, torch.zeros_like(positions))
+ starts = torch.maximum(starts, torch.cummax(reset_starts, dim = -1).values)
+ source = positions - shift
+ valid = source >= starts
+ gathered = torch.gather(
+ context, -1, source.clamp(min = 0).expand_as(context).contiguous()
+ )
+ return torch.where(valid, gathered, torch.zeros_like(gathered))
+
+ def forward(
+ self,
+ word_embeds,
+ input_ids,
+ ngram_context = None,
+ resets = None,
+ ):
+ seq_len = input_ids.shape[-1]
+ context = input_ids
+ if ngram_context is not None:
+ context = torch.cat([ngram_context[..., -(self.n - 1) :], input_ids], dim = -1)
+ if resets is not None:
+ resets = torch.cat(
+ [
+ resets.new_zeros(resets.shape[:-1] + (context.shape[-1] - seq_len,)),
+ resets,
+ ],
+ dim = -1,
+ )
+ x = word_embeds
+ context = context.long()
+ shifted = {i: self._shifted(context, i - 1, resets) for i in range(2, self.n + 1)}
+ for i in range(2, self.n + 1):
+ for j in range(self.k):
+ index = (i - 2) * self.k + j
+ mod = int(self.m + index * 2 + 1)
+ ngram_ids = context.clone()
+ power = 1
+ for order in range(2, i + 1):
+ power = (power * self.vocab_size) % mod
+ ngram_ids = ngram_ids + shifted[order] * power
+ ngram_ids = (ngram_ids % mod)[..., -seq_len:]
+ embedder = self.embedders[index]
+ projected = self.post_projs[index](
+ embedder(ngram_ids.to(embedder.weight.device))
+ )
+ x = x + projected.to(x.device)
+ return x / (1 + self.k * (self.n - 1))
+
+ class LongcatLsaModel(LongcatFlashModel):
+ config_class = LongcatLsaConfig
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.ngram_embeddings = LongcatNgramEmbedding(config)
+ # transformers 5 sizes gate_up_proj for routed + zero experts; the checkpoint has routed only.
+ routed = config.n_routed_experts
+ for layer in self.layers:
+ # SGLang uses rms_norm_eps for the MLA latent norms; longcat_flash uses 1e-6.
+ for attn in getattr(layer, "self_attn", ()):
+ for norm_name in ("q_a_layernorm", "kv_a_layernorm"):
+ norm = getattr(attn, norm_name, None)
+ if norm is not None and hasattr(norm, "variance_epsilon"):
+ norm.variance_epsilon = config.rms_norm_eps
+ experts = getattr(getattr(layer, "mlp", None), "experts", None)
+ gate_up = getattr(experts, "gate_up_proj", None)
+ if isinstance(gate_up, nn.Parameter) and gate_up.shape[0] > routed:
+ experts.gate_up_proj = nn.Parameter(
+ torch.empty(
+ routed, *gate_up.shape[1:], dtype = gate_up.dtype, device = gate_up.device
+ )
+ )
+ if getattr(config, "index_topk", None) and getattr(config, "index_n_heads", None):
+ for layer in self.layers:
+ layer.self_attn[0].indexer = LongcatLsaIndexer(config)
+ self.post_init()
+
+ def forward(
+ self,
+ input_ids = None,
+ attention_mask = None,
+ position_ids = None,
+ past_key_values = None,
+ inputs_embeds = None,
+ use_cache = None,
+ **kwargs,
+ ):
+ if inputs_embeds is not None:
+ raise ValueError(
+ "Unsloth: LongCat-Flash-Lite-Sparse builds its input embeddings from token ids "
+ "(word plus n-gram embeddings), so pass `input_ids`, not `inputs_embeds`."
+ )
+ if input_ids is not None:
+ topk = getattr(self.config, "index_topk", None)
+ cached = 0
+ if past_key_values is not None:
+ try:
+ cached = int(past_key_values.get_seq_length())
+ except Exception:
+ cached = 0
+ total = cached + input_ids.shape[-1]
+ if topk and total > topk and not getattr(self, "_unsloth_lsa_warned", False):
+ self._unsloth_lsa_warned = True
+ warnings.warn(
+ f"Unsloth: LongCat sparse attention selects every key only up to {topk} "
+ f"tokens. This {total} token sequence runs dense attention, "
+ "which differs from the sparse attention used at inference.",
+ stacklevel = 2,
+ )
+ history = getattr(past_key_values, "_unsloth_ngram_history", None)
+ keep = self.ngram_embeddings.n - 1
+ context = None if history is None else history[..., -keep:]
+ # position_ids restarts (packing, left padding) begin a new n-gram segment.
+ resets = None
+ pos = position_ids
+ if pos is None and attention_mask is not None and attention_mask.dim() == 2:
+ # Same positions generate() derives from the mask; only the n-gram uses them.
+ mask = attention_mask.long()
+ pos = (mask.cumsum(-1) - 1).masked_fill(mask == 0, 0)[
+ ..., -input_ids.shape[-1] :
+ ]
+ if pos is not None and pos.shape[-1] == input_ids.shape[-1]:
+ pos = pos.expand(input_ids.shape[0], -1)
+ resets = torch.diff(pos, prepend = pos[..., :1] - 1, dim = -1) != 1
+ inputs_embeds = self.ngram_embeddings(
+ self.embed_tokens(input_ids), input_ids, context, resets
+ )
+ # No cache in training: transformers' packed mask needs past_key_values None.
+ if use_cache is None:
+ use_cache = getattr(self.config, "use_cache", False) and not self.training
+ if use_cache and past_key_values is None:
+ from transformers.cache_utils import DynamicCache
+ try:
+ past_key_values = DynamicCache(config = self.config)
+ except TypeError:
+ past_key_values = DynamicCache()
+ if past_key_values is not None:
+ # Id history rides on the cache so beam reorders and crops move it.
+ seen = input_ids if history is None else torch.cat([history, input_ids], dim = -1)
+ if resets is not None:
+ # Mark ids before each row's last restart -1 so later decode steps skip them.
+ offset = seen.shape[-1] - input_ids.shape[-1]
+ idx = torch.arange(input_ids.shape[-1], device = resets.device) + offset
+ last = torch.where(resets, idx, torch.zeros_like(idx)).amax(
+ -1, keepdim = True
+ )
+ before = torch.arange(seen.shape[-1], device = seen.device) < last.to(
+ seen.device
+ )
+ seen = seen.masked_fill(before, -1)
+ past_key_values._unsloth_ngram_history = seen
+ cache_class = type(past_key_values)
+ if cache_class not in _NGRAM_TRACKING_CACHES.values():
+ past_key_values.__class__ = _ngram_tracking_cache_class(cache_class)
+ return super().forward(
+ input_ids = None,
+ attention_mask = attention_mask,
+ position_ids = position_ids,
+ past_key_values = past_key_values,
+ inputs_embeds = inputs_embeds,
+ use_cache = use_cache,
+ **kwargs,
+ )
+
+ class LongcatCausalLM(LongcatFlashForCausalLM):
+ config_class = LongcatLsaConfig
+ _keys_to_ignore_on_load_unexpected = [r"model\.mtp\..*"]
+ # Router and indexer weights_proj stay F32 as in SGLang.
+ _keep_in_fp32_modules_strict = ["router", "weights_proj"]
+
+ def __init__(self, config):
+ LongcatFlashPreTrainedModel.__init__(self, config)
+ self.model = LongcatLsaModel(config)
+ self.vocab_size = config.vocab_size
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias = False)
+ self.post_init()
+
+ # transformers 4 applies key_mapping only to VLM classes: pass it on load, undo it on save.
+ @classmethod
+ def from_pretrained(cls, *args, **kwargs):
+ if not _has_conversion_registry() and kwargs.get("key_mapping") is None:
+ kwargs["key_mapping"] = {"^" + src: dst for src, dst in _OE_RENAMES}
+ loaded = super().from_pretrained(*args, **kwargs)
+ # Loading drops requires_grad = False; re-freeze the indexer.
+ model = loaded[0] if isinstance(loaded, tuple) else loaded
+ for module in model.modules():
+ if isinstance(module, LongcatLsaIndexer):
+ module.requires_grad_(False)
+ return loaded
+
+ def save_pretrained(self, *args, **kwargs):
+ if not _has_conversion_registry():
+ state_dict = kwargs.get("state_dict")
+ if state_dict is None:
+ state_dict = self.state_dict()
+ kwargs["state_dict"] = {
+ _to_checkpoint_key(key): value for key, value in state_dict.items()
+ }
+ return super().save_pretrained(*args, **kwargs)
+
+ _CLASSES = (LongcatLsaConfig, LongcatCausalLM)
+ return _CLASSES
+
+
+def _has_conversion_registry():
+ try:
+ from transformers.conversion_mapping import register_checkpoint_conversion_mapping
+ except Exception:
+ return False
+ return True
+
+
+def _to_checkpoint_key(key):
+ for source, target in _OE_RENAMES:
+ if key.startswith(target):
+ return source + key[len(target) :]
+ return key
+
+
+_CONVERSIONS_REGISTERED = False
+
+
+def _register_conversions():
+ """Give this model_type longcat_flash's transformers 5 conversions plus the n-gram renames."""
+ global _CONVERSIONS_REGISTERED
+ if _CONVERSIONS_REGISTERED:
+ return
+ try:
+ from transformers.conversion_mapping import (
+ get_checkpoint_conversion_mapping,
+ register_checkpoint_conversion_mapping,
+ )
+ from transformers.core_model_loading import WeightRenaming
+ except Exception:
+ return
+ renames = [
+ WeightRenaming(source_patterns = source, target_patterns = target)
+ for source, target in _OE_RENAMES
+ ]
+ base = get_checkpoint_conversion_mapping("longcat_flash") or []
+ register_checkpoint_conversion_mapping(
+ LONGCAT_LSA_MODEL_TYPE, renames + list(base), overwrite = True
+ )
+ _CONVERSIONS_REGISTERED = True
+ # peft's per-expert LoRA conversion table; must come after the registration above.
+ for module_name in (
+ "transformers.conversion_mapping",
+ "peft.utils.transformers_weight_conversion",
+ ):
+ table = getattr(sys.modules.get(module_name), "_MODEL_TO_CONVERSION_PATTERN", None)
+ if isinstance(table, dict) and "longcat_flash" in table:
+ table.setdefault(LONGCAT_LSA_MODEL_TYPE, table["longcat_flash"])
+
+
+def register_longcat_lsa():
+ """Register the config and model classes with the Auto factories (idempotent)."""
+ from transformers import AutoConfig, AutoModelForCausalLM
+
+ config_class, model_class = _classes()
+ try:
+ AutoConfig.register(LONGCAT_LSA_MODEL_TYPE, config_class)
+ except ValueError:
+ pass
+ try:
+ AutoModelForCausalLM.register(config_class, model_class)
+ except ValueError:
+ pass
+ _register_conversions()
+ return config_class, model_class
+
+
+def load_longcat_lsa_config(pretrained_model_name_or_path, *args, **kwargs):
+ config_class, _ = register_longcat_lsa()
+ kwargs.pop("trust_remote_code", None)
+ kwargs.pop("code_revision", None)
+ return config_class.from_pretrained(pretrained_model_name_or_path, *args, **kwargs)
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index dae92111181..11bf9a9d097 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -1269,6 +1269,19 @@ def _get_total_transformer_layers(model):
return None
+def _architecture_skip_modules(model_types):
+ """Extra modules kept out of bitsandbytes; shared by the device-map planner and the load."""
+ model_types = model_types or []
+ skip = []
+ # Nemotron-H uses 'mixer' (not 'mamba') for Mamba layers, whose fused kernels pass out_proj.weight straight to F.linear and fail with quantized Params4bit, so skip out_proj.
+ if any(mt == "nemotron_h" for mt in model_types):
+ skip.append("out_proj")
+ # LongCat-Flash MLA: NF4 on q_b_proj / kv_b_proj badly hurts loss; both are small.
+ if any(mt in ("longcat_flash", "longcat_flash_lsa") for mt in model_types):
+ skip.extend(("q_b_proj", "kv_b_proj"))
+ return skip
+
+
def _cast_unquantized_floats(model, dtype):
"""Cast every floating parameter and buffer that is not a quantized weight."""
for tensor in list(model.parameters()) + list(model.buffers()):
@@ -1902,9 +1915,7 @@ class FastBaseModel:
load_in_4bit = load_in_4bit,
load_in_8bit = load_in_8bit,
quantization_config = user_quantization_config,
- extra_skip_modules = ["out_proj"]
- if any(mt == "nemotron_h" for mt in (model_types or []))
- else None,
+ extra_skip_modules = _architecture_skip_modules(model_types) or None,
),
)
@@ -1957,10 +1968,7 @@ class FastBaseModel:
tokenizer_only = True,
)
- _skip_modules = SKIP_QUANTIZATION_MODULES.copy()
- # Nemotron-H uses 'mixer' (not 'mamba') for Mamba layers, whose fused kernels pass out_proj.weight straight to F.linear and fail with quantized Params4bit, so skip out_proj.
- if any(mt == "nemotron_h" for mt in (model_types or [])):
- _skip_modules.append("out_proj")
+ _skip_modules = SKIP_QUANTIZATION_MODULES.copy() + _architecture_skip_modules(model_types)
if load_in_4bit:
bnb_config = BitsAndBytesConfig(