Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Load and fine-tune LongCat-Flash-Lite-Sparse on transformers' longcat_flash (#11620)
Daniel Hancccba4413e64
diff --git a/tests/test_longcat_lsa.py b/tests/test_longcat_lsa.pynew file mode 100644index 00000000000..2c57d1a943c--- /dev/null+++ b/tests/test_longcat_lsa.py@@ -0,0 +1,648 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""LongCat-Flash-Lite-Sparse loads via ``unsloth/models/longcat_lsa.py``: load, forward vs an+SGLang-transcribed reference, cache, and save back to the published layout, on a tiny checkpoint."""++import json+import math+import os+import warnings++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("transformers")+pytest.importorskip("transformers.models.longcat_flash")+safetensors_torch = pytest.importorskip("safetensors.torch")++import torch.nn.functional as F # noqa: E402+from real_accelerator import has_real_cuda # noqa: E402+from transformers import AutoConfig, AutoModelForCausalLM # noqa: E402++from unsloth.import_fixes import fix_transformers_longcat_lsa_config # noqa: E402+from unsloth.models.longcat_lsa import ( # noqa: E402+ LONGCAT_LSA_MODEL_TYPE,+ is_longcat_lsa_config_dict,+ register_longcat_lsa,+)++# The published config.json, verbatim apart from the dimensions shrunk below.+REAL_CONFIG = {+ "architectures": ["LongcatCausalLM"],+ "attention_bias": False,+ "attention_dropout": 0.0,+ "vocab_size": 131072,+ "hidden_size": 3072,+ "ffn_hidden_size": 6144,+ "expert_ffn_hidden_size": 1024,+ "num_layers": 14,+ "num_attention_heads": 32,+ "kv_lora_rank": 512,+ "q_lora_rank": 1536,+ "qk_rope_head_dim": 64,+ "v_head_dim": 128,+ "qk_nope_head_dim": 128,+ "mla_scale_q_lora": True,+ "mla_scale_kv_lora": True,+ "routed_scaling_factor": 6.0,+ "n_routed_experts": 256,+ "max_position_embeddings": 983040,+ "rms_norm_eps": 1e-05,+ "use_cache": True,+ "bos_token_id": 1,+ "eos_token_id": 2,+ "rope_theta": 1000000.0,+ "rope_scaling": {+ "original_max_position_embeddings": 8192,+ "rope_type": "deepseek_yarn",+ "factor": 120,+ "beta_fast": 32,+ "beta_slow": 1,+ "mscale": 1,+ "mscale_all_dim": 1,+ },+ "attention_method": "LSA",+ "zero_expert_num": 128,+ "zero_expert_type": "identity",+ "moe_topk": 12,+ "use_mla": 1,+ "oe_vocab_size_ratio": 78,+ "oe_neighbor_num": 4,+ "oe_split_num": 4,+ "mtp_num_layers": 1,+ "index_n_heads": 16,+ "index_head_dim": 128,+ "index_topk": 2048,+ "index_k_norm_type": "rms",+ "cli_factor": 2,+ "index_local_tokens": 1024,+ "index_init_tokens": 16,+}++TINY = dict(+ vocab_size = 128,+ hidden_size = 96,+ ffn_hidden_size = 64,+ expert_ffn_hidden_size = 16,+ num_layers = 2,+ num_attention_heads = 2,+ kv_lora_rank = 16,+ q_lora_rank = 24,+ qk_rope_head_dim = 8,+ v_head_dim = 8,+ qk_nope_head_dim = 8,+ n_routed_experts = 4,+ zero_expert_num = 2,+ moe_topk = 2,+ oe_vocab_size_ratio = 3,+ index_n_heads = 2,+ index_head_dim = 8,+)+++def _write_tiny(+ path,+ seed = 0,+ **overrides,+):+ cfg = dict(REAL_CONFIG, **TINY, **overrides)+ g = torch.Generator().manual_seed(seed)++ def w(*shape, scale = 0.1):+ return torch.randn(*shape, generator = g) * scale++ def norm(n):+ return 1.0 + 0.1 * torch.randn(n, generator = g)++ H, V = cfg["hidden_size"], cfg["vocab_size"]+ E, Z = cfg["n_routed_experts"], cfg["zero_expert_num"]+ I, Fh = cfg["expert_ffn_hidden_size"], cfg["ffn_hidden_size"]+ nh, ql, kl = cfg["num_attention_heads"], cfg["q_lora_rank"], cfg["kv_lora_rank"]+ rope, nope, vd = cfg["qk_rope_head_dim"], cfg["qk_nope_head_dim"], cfg["v_head_dim"]+ sd = {+ "model.embed_tokens.weight": w(V, H, scale = 0.5),+ "lm_head.weight": w(V, H),+ "model.norm.weight": norm(H),+ }+ m = int(cfg["oe_vocab_size_ratio"] * V)+ tables = cfg["oe_split_num"] * (cfg["oe_neighbor_num"] - 1)+ for i in range(tables):+ sd[f"model.oe_embed_tokens{i}.weight"] = w(m + 2 * i + 1, H // tables, scale = 0.5)+ sd[f"model.oe_embed_proj{i}.weight"] = w(H, H // tables)++ def attn(prefix, indexer):+ sd[prefix + "q_a_proj.weight"] = w(ql, H)+ sd[prefix + "q_a_layernorm.weight"] = norm(ql)+ sd[prefix + "q_b_proj.weight"] = w(nh * (nope + rope), ql)+ sd[prefix + "kv_a_proj_with_mqa.weight"] = w(kl + rope, H)+ sd[prefix + "kv_a_layernorm.weight"] = norm(kl)+ sd[prefix + "kv_b_proj.weight"] = w(nh * (nope + vd), kl)+ sd[prefix + "o_proj.weight"] = w(H, nh * vd)+ if indexer:+ d, h = cfg["index_head_dim"], cfg["index_n_heads"]+ sd[prefix + "indexer.wq_b.weight"] = w(h * d, ql)+ sd[prefix + "indexer.wk.weight"] = w(d, H)+ sd[prefix + "indexer.k_norm.weight"] = norm(d)+ sd[prefix + "indexer.weights_proj.weight"] = w(h, H)++ for layer in range(cfg["num_layers"]):+ p = f"model.layers.{layer}."+ for s in range(2):+ attn(p + f"self_attn.{s}.", indexer = s == 0)+ sd[p + f"input_layernorm.{s}.weight"] = norm(H)+ sd[p + f"post_attention_layernorm.{s}.weight"] = norm(H)+ sd[p + f"mlps.{s}.gate_proj.weight"] = w(Fh, H)+ sd[p + f"mlps.{s}.up_proj.weight"] = w(Fh, H)+ sd[p + f"mlps.{s}.down_proj.weight"] = w(H, Fh)+ sd[p + "mlp.router.classifier.weight"] = w(E + Z, H, scale = 0.5)+ sd[p + "mlp.router.e_score_correction_bias"] = 0.01 * torch.randn(E + Z, generator = g)+ for e in range(E):+ sd[p + f"mlp.experts.{e}.gate_proj.weight"] = w(I, H)+ sd[p + f"mlp.experts.{e}.up_proj.weight"] = w(I, H)+ sd[p + f"mlp.experts.{e}.down_proj.weight"] = w(H, I)+ sd["model.mtp.norm.weight"] = norm(H)+ sd["model.mtp.layers.0.eh_proj.weight"] = w(H, 2 * H)+ stored = {+ k: (v if ("router" in k or "weights_proj" in k) else v.to(torch.bfloat16)).contiguous()+ for k, v in sd.items()+ }+ os.makedirs(path, exist_ok = True)+ safetensors_torch.save_file(+ stored, os.path.join(path, "model.safetensors"), metadata = {"format": "pt"}+ )+ with open(os.path.join(path, "config.json"), "w") as f:+ json.dump(cfg, f)+ return cfg, {k: v.float() for k, v in stored.items()}+++def _load(path, dtype = torch.float32):+ register_longcat_lsa()+ return AutoModelForCausalLM.from_pretrained(path, dtype = dtype)+++# Reference from SGLang longcat_flash.py, deepseek_v2 MLA, deepseek_yarn, ngram_embedding.cuh.+++def _ref_ngram_ids(cfg, tokens):+ V, eos = cfg["vocab_size"], cfg["eos_token_id"]+ k_, n_ = cfg["oe_split_num"], cfg["oe_neighbor_num"]+ m = int(cfg["oe_vocab_size_ratio"] * V)+ ids = torch.zeros(len(tokens), (n_ - 1) * k_, dtype = torch.long)+ for n in range(n_ - 1):+ for k in range(k_):+ idx = n * k_ + k+ mod = m + 2 * idx + 1+ for i in range(len(tokens)):+ acc = 0+ for j in range(n + 2):+ if i - j < 0:+ break+ tok = int(tokens[i - j])+ if tok == eos and j > 0: # the context never crosses an earlier EOS+ break+ acc += (tok * pow(V, j, mod)) % mod+ ids[i, idx] = acc % mod+ return ids+++def _ref_forward(cfg, sd, tokens):+ eps, H = cfg["rms_norm_eps"], cfg["hidden_size"]+ nh, nope, rope, vd = (+ cfg["num_attention_heads"],+ cfg["qk_nope_head_dim"],+ cfg["qk_rope_head_dim"],+ cfg["v_head_dim"],+ )+ T = len(tokens)++ def rms(x, w):+ return x * torch.rsqrt(x.pow(2).mean(-1, keepdim = True) + eps) * w++ def mscale(scale, m):+ return 1.0 if scale <= 1 else 0.1 * m * math.log(scale) + 1.0++ rs = cfg["rope_scaling"]+ factor, omax, base = rs["factor"], rs["original_max_position_embeddings"], cfg["rope_theta"]++ def corr(rot):+ return (rope * math.log(omax / (rot * 2 * math.pi))) / (2 * math.log(base))++ low = max(math.floor(corr(rs["beta_fast"])), 0)+ high = min(math.ceil(corr(rs["beta_slow"])), rope - 1)+ pos_freqs = base ** (torch.arange(0, rope, 2, dtype = torch.float) / rope)+ keep = 1 - ((torch.arange(rope // 2, dtype = torch.float) - low) / (high - low)).clamp(0, 1)+ inv_freq = (1.0 / (factor * pos_freqs)) * (1 - keep) + (1.0 / pos_freqs) * keep+ freqs = torch.arange(T, dtype = torch.float)[:, None] * inv_freq[None]+ cos, sin = freqs.cos()[:, None], freqs.sin()[:, None]++ def rotate(x): # interleaved pairs (is_neox_style = False)+ x1, x2 = x[..., 0::2], x[..., 1::2]+ return torch.stack((x1 * cos - x2 * sin, x2 * cos + x1 * sin), -1).flatten(-2)++ scaling = (nope + rope) ** -0.5 * mscale(factor, rs["mscale_all_dim"]) ** 2+ q_scale, kv_scale = (H / cfg["q_lora_rank"]) ** 0.5, (H / cfg["kv_lora_rank"]) ** 0.5++ def mla(p, x):+ q = rms(x @ sd[p + "q_a_proj.weight"].T, sd[p + "q_a_layernorm.weight"] * q_scale)+ q_nope, q_pe = (q @ sd[p + "q_b_proj.weight"].T).view(T, nh, -1).split([nope, rope], -1)+ kv_a, k_pe = (x @ sd[p + "kv_a_proj_with_mqa.weight"].T).split(+ [cfg["kv_lora_rank"], rope], -1+ )+ kv = rms(kv_a, sd[p + "kv_a_layernorm.weight"] * kv_scale) @ sd[p + "kv_b_proj.weight"].T+ k_nope, v = kv.view(T, nh, -1).split([nope, vd], -1)+ q = torch.cat([q_nope, rotate(q_pe)], -1)+ k = torch.cat([k_nope, rotate(k_pe[:, None]).expand(T, nh, rope)], -1)+ s = torch.einsum("qhd,khd->hqk", q, k) * scaling+ # kv_len <= index_topk: the indexer's top-k selects every causal position.+ s = s.masked_fill(~torch.ones(T, T, dtype = torch.bool).tril(), float("-inf"))+ o = torch.einsum("hqk,khd->qhd", s.softmax(-1), v).reshape(T, -1)+ return o @ sd[p + "o_proj.weight"].T++ def mlp(p, x):+ gate = F.silu(x @ sd[p + "gate_proj.weight"].T)+ return (gate * (x @ sd[p + "up_proj.weight"].T)) @ sd[p + "down_proj.weight"].T++ def moe(p, x):+ E, rsf = cfg["n_routed_experts"], cfg["routed_scaling_factor"]+ scores = (x @ sd[p + "router.classifier.weight"].T).softmax(-1)+ choice = scores + sd[p + "router.e_score_correction_bias"][None]+ ids = torch.topk(choice, cfg["moe_topk"], dim = -1)[1]+ weights = scores.gather(1, ids) * rsf # identity experts scaled as in transformers / vLLM+ out = torch.zeros_like(x)+ for t in range(T):+ for j in range(ids.shape[1]):+ e = int(ids[t, j])+ y = mlp(p + f"experts.{e}.", x[t : t + 1])[0] if e < E else x[t]+ out[t] += weights[t, j] * y+ return out++ ng = _ref_ngram_ids(cfg, tokens)+ parts = [sd["model.embed_tokens.weight"][tokens]]+ for i in range(ng.shape[1]):+ rows = sd[f"model.oe_embed_tokens{i}.weight"][ng[:, i]]+ parts.append(rows @ sd[f"model.oe_embed_proj{i}.weight"].T)+ h, residual = torch.stack(parts).mean(0), None++ def add_norm(h, residual, w):+ residual = h if residual is None else h + residual+ return rms(residual, w), residual++ for layer in range(cfg["num_layers"]):+ p = f"model.layers.{layer}."+ x, residual = add_norm(h, residual, sd[p + "input_layernorm.0.weight"])+ x, residual = add_norm(+ mla(p + "self_attn.0.", x), residual, sd[p + "post_attention_layernorm.0.weight"]+ )+ shortcut = moe(p + "mlp.", x)+ x, residual = add_norm(mlp(p + "mlps.0.", x), residual, sd[p + "input_layernorm.1.weight"])+ x, residual = add_norm(+ mla(p + "self_attn.1.", x), residual, sd[p + "post_attention_layernorm.1.weight"]+ )+ h = shortcut + mlp(p + "mlps.1.", x)+ x, _ = add_norm(h, residual, sd["model.norm.weight"])+ return x @ sd["lm_head.weight"].T+++def _tokens(+ cfg,+ batch,+ length,+ seed = 1,+):+ g = torch.Generator().manual_seed(seed)+ ids = torch.randint(3, cfg["vocab_size"], (batch, length), generator = g)+ eos = cfg["eos_token_id"]+ ids[0, 5] = eos+ ids[0, 6] = eos # back-to-back EOS+ if batch > 1:+ ids[1, 11] = eos+ return ids++[email protected](scope = "module")+def tiny(tmp_path_factory):+ path = str(tmp_path_factory.mktemp("longcat_lsa_tiny"))+ cfg, sd = _write_tiny(path)+ return path, cfg, sd+++def test_recognises_only_this_config():+ assert is_longcat_lsa_config_dict(REAL_CONFIG)+ assert is_longcat_lsa_config_dict({"model_type": LONGCAT_LSA_MODEL_TYPE})+ lite = dict(REAL_CONFIG, model_type = "longcat_flash_ngram")+ assert not is_longcat_lsa_config_dict(lite)+ no_ngram = {k: v for k, v in REAL_CONFIG.items() if k != "oe_vocab_size_ratio"}+ assert not is_longcat_lsa_config_dict(no_ngram)+ assert not is_longcat_lsa_config_dict({"architectures": ["LlamaForCausalLM"]})+ assert not is_longcat_lsa_config_dict(None)+++def test_autoconfig_loads_the_published_config(tmp_path):+ fix_transformers_longcat_lsa_config()+ with open(tmp_path / "config.json", "w") as f:+ json.dump(REAL_CONFIG, f)+ config = AutoConfig.from_pretrained(str(tmp_path))+ assert config.model_type == LONGCAT_LSA_MODEL_TYPE+ rope = getattr(config, "rope_parameters", None) or config.rope_scaling+ assert rope["rope_type"] == "yarn" # SGLang's deepseek_yarn is transformers' yarn+ assert config.head_dim == config.qk_rope_head_dim == 64+ assert config.oe_vocab_size_ratio == 78 and config.index_topk == 2048++ other = tmp_path / "other"+ other.mkdir()+ with open(other / "config.json", "w") as f:+ json.dump({"architectures": ["SomethingElse"]}, f)+ with pytest.raises(ValueError):+ AutoConfig.from_pretrained(str(other))+++def test_every_checkpoint_key_loads(tiny):+ path, cfg, sd = tiny+ register_longcat_lsa()+ model, info = AutoModelForCausalLM.from_pretrained(+ path, dtype = torch.float32, output_loading_info = True+ )+ assert not info["missing_keys"], info["missing_keys"]+ assert all(k.startswith("model.mtp.") for k in info["unexpected_keys"]), info+ assert not info.get("mismatched_keys"), info["mismatched_keys"]+ layer = model.model.layers[0]+ experts = layer.mlp.experts+ p = "model.layers.0.mlp.experts.3."+ if hasattr(experts, "gate_up_proj"): # transformers 5 stacks the experts+ assert experts.gate_up_proj.shape[0] == cfg["n_routed_experts"]+ gate_up = torch.cat([sd[p + "gate_proj.weight"], sd[p + "up_proj.weight"]], 0)+ assert torch.equal(experts.gate_up_proj[3], gate_up)+ assert torch.equal(experts.down_proj[3], sd[p + "down_proj.weight"])+ else:+ assert torch.equal(experts[3].gate_proj.weight, sd[p + "gate_proj.weight"])+ ngram = model.model.ngram_embeddings+ assert torch.equal(ngram.embedders[7].weight, sd["model.oe_embed_tokens7.weight"])+ assert torch.equal(ngram.post_projs[11].weight, sd["model.oe_embed_proj11.weight"])+ indexer = layer.self_attn[0].indexer+ assert torch.equal(indexer.wq_b.weight, sd["model.layers.0.self_attn.0.indexer.wq_b.weight"])+ assert not indexer.wq_b.weight.requires_grad+++def test_forward_matches_sglang_reference(tiny):+ path, cfg, sd = tiny+ model = _load(path).eval()+ ids = _tokens(cfg, 2, 19)+ with torch.no_grad():+ logits = model(input_ids = ids).logits+ for b in range(ids.shape[0]):+ ref = _ref_forward(cfg, sd, ids[b])+ torch.testing.assert_close(logits[b], ref, atol = 2e-4, rtol = 1e-4)+++def test_ngram_ids_follow_the_sglang_kernel(tiny):+ path, cfg, sd = tiny+ model = _load(path).eval()+ seen = []+ handles = [+ e.register_forward_hook(lambda mod, args, out: seen.append(args[0].clone()))+ for e in model.model.ngram_embeddings.embedders+ ]+ ids = _tokens(cfg, 2, 15)+ with torch.no_grad():+ model.model.ngram_embeddings(model.model.embed_tokens(ids), ids)+ for h in handles:+ h.remove()+ ours = torch.stack(seen, -1)+ for b in range(ids.shape[0]):+ assert torch.equal(ours[b], _ref_ngram_ids(cfg, ids[b]))++[email protected](not has_real_cuda(), reason = "needs a CUDA device")+def test_split_model_keeps_the_token_table_where_accelerate_put_it(tiny):+ # Accelerate hooks move every `.to`-able arg: never pass the embed_tokens module itself.+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module++ path, cfg, sd = tiny+ model = _load(path).eval().to("cuda")+ ids = _tokens(cfg, 2, 15).to("cuda")+ with torch.no_grad():+ expected = model(input_ids = ids).logits.cpu()+ model.model.embed_tokens.to("cpu")+ add_hook_to_module(model.model.embed_tokens, AlignDevicesHook(execution_device = "cpu"))+ add_hook_to_module(model.model.ngram_embeddings, AlignDevicesHook(execution_device = "cuda"))+ with torch.no_grad():+ got = model(input_ids = ids).logits.cpu()+ assert model.model.embed_tokens.weight.device.type == "cpu"+ torch.testing.assert_close(got, expected, atol = 1e-4, rtol = 1e-4)+++def test_cached_decode_matches_full_forward(tiny):+ path, cfg, sd = tiny+ model = _load(path).eval()+ ids = _tokens(cfg, 1, 14)+ with torch.no_grad():+ full = model(input_ids = ids).logits[0]+ out = model(input_ids = ids[:, :9], use_cache = True)+ cache, steps = out.past_key_values, [out.logits[0]]+ for t in range(9, ids.shape[1]): # decode across the n-gram context boundary+ out = model(input_ids = ids[:, t : t + 1], past_key_values = cache, use_cache = True)+ cache = out.past_key_values+ steps.append(out.logits[0])+ torch.testing.assert_close(torch.cat(steps), full, atol = 1e-4, rtol = 1e-4)+++def test_packed_training_rows_do_not_see_each_other(tiny):+ path, cfg, sd = tiny+ model = _load(path).train()+ a, b = _tokens(cfg, 2, 12, seed = 3)+ b[0] = a[-1] # b's first n-grams would read a's tail if the n-gram crossed the boundary+ packed = torch.cat([a, b])[None]+ position_ids = torch.cat([torch.arange(12), torch.arange(12)])[None]+ with torch.no_grad():+ out = model(input_ids = packed, position_ids = position_ids)+ alone = model(input_ids = b[None]).logits[0]+ assert out.past_key_values is None+ torch.testing.assert_close(out.logits[0, 12:], alone, atol = 1e-4, rtol = 1e-4)+++def test_left_padded_generation_matches_unpadded(tiny):+ path, cfg, sd = tiny+ model = _load(path).eval()+ ids = _tokens(cfg, 1, 10, seed = 4)+ padded = torch.cat([torch.full((1, 3), 7), ids], dim = -1)+ mask = torch.cat([torch.zeros(1, 3, dtype = torch.long), torch.ones_like(ids)], dim = -1)+ kwargs = dict(max_new_tokens = 5, do_sample = False, pad_token_id = 0)+ with torch.no_grad():+ want = model.generate(input_ids = ids, **kwargs)[0, 10:]+ got = model.generate(input_ids = padded, attention_mask = mask, **kwargs)[0, 13:]+ assert got.tolist() == want.tolist()+++def test_left_padded_mask_without_position_ids_matches_unpadded(tiny):+ path, cfg, sd = tiny+ model = _load(path).eval()+ ids = _tokens(cfg, 1, 10, seed = 4)+ padded = torch.cat([torch.full((1, 3), 7), ids], dim = -1)+ mask = torch.cat([torch.zeros(1, 3, dtype = torch.long), torch.ones_like(ids)], dim = -1)+ with torch.no_grad():+ want = model(input_ids = ids).logits[0]+ got = model(input_ids = padded, attention_mask = mask).logits[0, 3:]+ torch.testing.assert_close(got, want, atol = 1e-4, rtol = 1e-4)+++def test_resized_vocab_saves_and_reloads(tiny, tmp_path):+ path, cfg, sd = tiny+ model = _load(path).eval()+ model.resize_token_embeddings(cfg["vocab_size"] + 8)+ model.save_pretrained(str(tmp_path))+ reloaded = _load(str(tmp_path)).eval()+ ids = _tokens(cfg, 1, 12, seed = 5)+ with torch.no_grad():+ torch.testing.assert_close(+ reloaded(input_ids = ids).logits, model(input_ids = ids).logits, atol = 1e-5, rtol = 1e-5+ )+++def test_ngram_history_follows_beam_reorder_and_crop(tiny):+ path, cfg, sd = tiny+ model = _load(path).eval()+ ids = _tokens(cfg, 2, 14)+ swapped = ids.flip(0)+ with torch.no_grad():+ full = model(input_ids = swapped).logits+ cache = model(input_ids = ids[:, :9], use_cache = True).past_key_values+ cache.reorder_cache(torch.tensor([1, 0]))+ out = model(input_ids = swapped[:, 9:10], past_key_values = cache, use_cache = True)+ torch.testing.assert_close(out.logits[:, -1], full[:, 9], atol = 1e-4, rtol = 1e-4)+ cache = out.past_key_values+ cache.crop(9)+ out = model(input_ids = swapped[:, 9:11], past_key_values = cache, use_cache = True)+ torch.testing.assert_close(out.logits[:, -1], full[:, 10], atol = 1e-4, rtol = 1e-4)+++def test_cache_reset_clears_the_ngram_history(tiny):+ import transformers+ from transformers.cache_utils import StaticCache++ if int(transformers.__version__.split(".")[0]) < 5:+ pytest.skip("transformers 4's StaticCache cannot hold MLA's differing key/value head dims")+ path, cfg, sd = tiny+ model = _load(path).eval()+ first, second = _tokens(cfg, 1, 9, seed = 1), _tokens(cfg, 1, 9, seed = 2)+ try:+ cache = StaticCache(config = model.config, max_cache_len = 32)+ except TypeError:+ cache = StaticCache(config = model.config, max_batch_size = 1, max_cache_len = 32)+ with torch.no_grad():+ expected = model(input_ids = second).logits+ model(input_ids = first, past_key_values = cache, use_cache = True)+ cache.reset()+ got = model(input_ids = second, past_key_values = cache, use_cache = True).logits+ torch.testing.assert_close(got, expected, atol = 1e-4, rtol = 1e-4)+++def test_inputs_embeds_is_refused(tiny):+ path, cfg, sd = tiny+ model = _load(path).eval()+ ids = _tokens(cfg, 1, 9)+ with pytest.raises(ValueError, match = "input_ids"):+ model(inputs_embeds = model.model.embed_tokens(ids))+++def test_bf16_keeps_router_fp32_and_sglang_norm_eps(tiny):+ path, cfg, sd = tiny+ model = _load(path, dtype = torch.bfloat16)+ layer = model.model.layers[0]+ assert layer.mlp.router.classifier.weight.dtype == torch.float32+ assert layer.self_attn[0].indexer.weights_proj.weight.dtype == torch.float32+ assert layer.self_attn[0].q_a_proj.weight.dtype == torch.bfloat16+ for attn in layer.self_attn:+ assert attn.q_a_layernorm.variance_epsilon == cfg["rms_norm_eps"]+ assert attn.kv_a_layernorm.variance_epsilon == cfg["rms_norm_eps"]+++def test_save_writes_the_published_layout(tiny, tmp_path):+ path, cfg, sd = tiny+ model = _load(path, dtype = torch.bfloat16)+ model.save_pretrained(str(tmp_path))+ saved = {}+ for name in os.listdir(tmp_path):+ if name.endswith(".safetensors"):+ saved.update(safetensors_torch.load_file(str(tmp_path / name)))+ original = safetensors_torch.load_file(os.path.join(path, "model.safetensors"))+ expected = {k for k in original if not k.startswith("model.mtp.")}+ assert set(saved) == expected+ for key in expected:+ assert saved[key].dtype == original[key].dtype, key+ assert torch.equal(saved[key], original[key]), key+ with open(tmp_path / "config.json") as f:+ assert json.load(f)["model_type"] == LONGCAT_LSA_MODEL_TYPE+++def test_peft_converts_per_expert_adapters_like_longcat_flash():+ register_longcat_lsa()+ conversion = pytest.importorskip("transformers.conversion_mapping")+ table = getattr(conversion, "_MODEL_TO_CONVERSION_PATTERN", None)+ if not isinstance(table, dict) or "longcat_flash" not in table:+ pytest.skip("this transformers has no model-type conversion table")+ assert table[LONGCAT_LSA_MODEL_TYPE] == table["longcat_flash"]+ try:+ from peft.utils import transformers_weight_conversion as peft_conversion+ except Exception:+ return+ assert (+ peft_conversion._MODEL_TO_CONVERSION_PATTERN.get(LONGCAT_LSA_MODEL_TYPE)+ == (table["longcat_flash"])+ )+++def test_long_sequence_warns_once(tmp_path):+ cfg, sd = _write_tiny(str(tmp_path), index_topk = 8)+ model = _load(str(tmp_path)).eval()+ ids = _tokens(cfg, 1, 12)+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ with torch.no_grad():+ model(input_ids = ids)+ model(input_ids = ids)+ messages = [str(w.message) for w in caught if "sparse attention" in str(w.message)]+ assert len(messages) == 1, messages+++def test_cached_decode_past_index_topk_warns(tmp_path):+ cfg, sd = _write_tiny(str(tmp_path), index_topk = 8)+ model = _load(str(tmp_path)).eval()+ ids = _tokens(cfg, 1, 12)+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ with torch.no_grad():+ cache = model(input_ids = ids[:, :8], use_cache = True).past_key_values+ model(input_ids = ids[:, 8:9], past_key_values = cache, use_cache = True)+ messages = [str(w.message) for w in caught if "sparse attention" in str(w.message)]+ assert len(messages) == 1 and "9 token" in messages[0], messages+++def test_4bit_keeps_the_mla_up_projections_in_16bit():+ # The device-map planner must size q_b_proj / kv_b_proj unquantized, as the load keeps them.+ import ast++ from unsloth.models.vision import _architecture_skip_modules++ for model_type in ("longcat_flash", "longcat_flash_lsa"):+ assert {"q_b_proj", "kv_b_proj"} <= set(_architecture_skip_modules([model_type]))+ assert _architecture_skip_modules(["llama"]) == []+ path = os.path.join(+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),+ "unsloth",+ "models",+ "vision.py",+ )+ with open(path, encoding = "utf-8") as f:+ tree = ast.parse(f.read())+ planner = [+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.Call)+ and getattr(node.func, "id", None) == "planner_quantization_kwargs"+ ]+ assert planner+ for call in planner:+ extra = next(k.value for k in call.keywords if k.arg == "extra_skip_modules")+ assert "_architecture_skip_modules" in ast.unparse(extra)diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 88e4fadc5e1..53f30b15998 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -254,6 +254,7 @@ from .import_fixes import (fix_transformers_composite_prefix_renaming,fix_transformers_fully_masked_rows,fix_transformers_chunked_mask_block_sequence_ids,+ fix_transformers_longcat_lsa_config,fix_transformers_rope_scaling_drops_theta,fix_transformers_validate_rope_ignore_keys,fix_transformers5_remote_code_legacy_defaults,@@ -322,6 +323,7 @@ fix_transformers_rope_scaling_drops_theta()fix_transformers_validate_rope_ignore_keys()fix_transformers5_remote_code_legacy_defaults()fix_transformers_config_only_remote_code()+fix_transformers_longcat_lsa_config()# Remote code written for 4.x reads plain RoPE as rope_scaling None and imports is_torch_fx_available.fix_transformers_remote_rope_scaling_none()fix_transformers_is_torch_fx_available()@@ -381,6 +383,7 @@ patch_accelerate_recursively_apply()del fix_transformers5_bare_annotation_configsdel fix_transformers_rope_scaling_drops_thetadel fix_transformers_validate_rope_ignore_keys+del fix_transformers_longcat_lsa_configdel fix_transformers_remote_rope_scaling_nonedel fix_transformers_is_torch_fx_availabledel fix_xformers_performance_issuediff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex a923e00b2e8..f0d5ea35f2e 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -10186,3 +10186,63 @@ def disable_sentencepiece_on_windows():f"{DISABLE_SENTENCEPIECE_VARIABLE}=0 to import it again.")return True+++def fix_transformers_longcat_lsa_config():+ """Answer AutoConfig's "Unrecognized model" on a LongcatCausalLM config (no model_type,+ auto_map or modeling code) with ``models/longcat_lsa.py``; all other loads are untouched."""+ try:+ from transformers import AutoConfig+ from transformers.configuration_utils import PretrainedConfig+ except Exception:+ return+ current = AutoConfig.__dict__.get("from_pretrained")+ original = getattr(current, "__func__", None)+ if original is None or getattr(original, "_unsloth_longcat_lsa", False):+ return+ try:+ import transformers.models.longcat_flash # noqa: F401+ except Exception:+ return++ def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):+ try:+ return original(cls, pretrained_model_name_or_path, *args, **kwargs)+ except ValueError as error:+ message = str(error)+ if "Unrecognized model" not in message and "longcat_flash_lsa" not in message:+ raise+ from .models.longcat_lsa import (+ is_longcat_lsa_config_dict,+ load_longcat_lsa_config,+ )++ hub_kwargs = {+ key: kwargs[key]+ for key in (+ "cache_dir",+ "force_download",+ "local_files_only",+ "token",+ "revision",+ "subfolder",+ )+ if key in kwargs+ }+ try:+ config_dict, _ = PretrainedConfig.get_config_dict(+ pretrained_model_name_or_path, **hub_kwargs+ )+ except Exception:+ raise error+ if not is_longcat_lsa_config_dict(config_dict):+ raise+ return load_longcat_lsa_config(pretrained_model_name_or_path, *args, **kwargs)++ from_pretrained._unsloth_longcat_lsa = True+ from_pretrained.__wrapped__ = original+ AutoConfig.from_pretrained = classmethod(from_pretrained)+ if UNSLOTH_ENABLE_LOGGING:+ logger.info(+ "Unsloth: LongcatCausalLM configs without a model_type load on transformers' longcat_flash."+ )diff --git a/unsloth/models/longcat_lsa.py b/unsloth/models/longcat_lsa.pynew file mode 100644index 00000000000..4bc53fd27c6--- /dev/null+++ b/unsloth/models/longcat_lsa.py@@ -0,0 +1,510 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""LongCat-Flash-Lite-Sparse (``LongcatCausalLM``, no model_type/auto_map/modeling code) on transformers' ``longcat_flash``.++At seq_len <= index_topk the LSA indexer selects every causal position, so the model is the dense+Lite network; the indexer weights are kept frozen only to load/save. Longer sequences train dense+(differs from sparse inference; warned once). Identity experts are scaled by routed_scaling_factor+as in transformers/vLLM (SGLang leaves them unscaled). The MTP head is not loaded.+"""++import sys+import warnings++import torch+from torch import nn++__all__ = [+ "LONGCAT_LSA_MODEL_TYPE",+ "is_longcat_lsa_config_dict",+ "register_longcat_lsa",+]++LONGCAT_LSA_MODEL_TYPE = "longcat_flash_lsa"+_LONGCAT_LSA_ARCHITECTURES = ("LongcatCausalLM",)++# Plain prefixes (no regex groups) so the mapping also runs backwards on save.+_OE_RENAMES = (+ ("model.oe_embed_tokens", "model.ngram_embeddings.embedders."),+ ("model.oe_embed_proj", "model.ngram_embeddings.post_projs."),+)+++def is_longcat_lsa_config_dict(config_dict) -> bool:+ """Config saved by this module, or the published one (``LongcatCausalLM``, no model_type)."""+ if not isinstance(config_dict, dict):+ return False+ model_type = config_dict.get("model_type")+ if model_type == LONGCAT_LSA_MODEL_TYPE:+ return True+ if model_type:+ return False+ architectures = config_dict.get("architectures") or []+ if not any(arch in _LONGCAT_LSA_ARCHITECTURES for arch in architectures):+ return False+ return config_dict.get("oe_vocab_size_ratio") is not None or (+ config_dict.get("ngram_vocab_size_ratio") is not None+ )+++def _translate_rope_scaling(rope_scaling):+ if not isinstance(rope_scaling, dict):+ return rope_scaling+ rope_scaling = dict(rope_scaling)+ rope_type = rope_scaling.pop("type", None) or rope_scaling.get("rope_type")+ if rope_type == "deepseek_yarn":+ rope_type = "yarn"+ if rope_type is not None:+ rope_scaling["rope_type"] = rope_type+ return rope_scaling+++_CLASSES = None+++_NGRAM_TRACKING_CACHES = {}+++def _ngram_tracking_cache_class(cls):+ """``cls`` whose reorder/select/repeat/crop also move the n-gram token history."""+ tracked = _NGRAM_TRACKING_CACHES.get(cls)+ if tracked is not None:+ return tracked++ class NgramTrackingCache(cls):+ def reorder_cache(self, beam_idx, *args, **kwargs):+ out = super().reorder_cache(beam_idx, *args, **kwargs)+ history = getattr(self, "_unsloth_ngram_history", None)+ if history is not None:+ self._unsloth_ngram_history = history.index_select(0, beam_idx.to(history.device))+ return out++ def batch_select_indices(self, indices, *args, **kwargs):+ out = super().batch_select_indices(indices, *args, **kwargs)+ history = getattr(self, "_unsloth_ngram_history", None)+ if history is not None:+ if torch.is_tensor(indices):+ indices = indices.to(history.device)+ self._unsloth_ngram_history = history[indices]+ return out++ def batch_repeat_interleave(self, repeats, *args, **kwargs):+ out = super().batch_repeat_interleave(repeats, *args, **kwargs)+ history = getattr(self, "_unsloth_ngram_history", None)+ if history is not None:+ self._unsloth_ngram_history = history.repeat_interleave(repeats, dim = 0)+ return out++ def reset(self, *args, **kwargs):+ out = super().reset(*args, **kwargs)+ self._unsloth_ngram_history = None+ return out++ def crop(self, *args, **kwargs):+ out = super().crop(*args, **kwargs)+ history = getattr(self, "_unsloth_ngram_history", None)+ if history is not None:+ self._unsloth_ngram_history = history[..., : self.get_seq_length()]+ return out++ NgramTrackingCache.__name__ = cls.__name__+ NgramTrackingCache.__qualname__ = cls.__qualname__+ _NGRAM_TRACKING_CACHES[cls] = NgramTrackingCache+ return NgramTrackingCache+++def _classes():+ """Build classes lazily so importing Unsloth does not import longcat_flash."""+ global _CLASSES+ if _CLASSES is not None:+ return _CLASSES++ from transformers.models.longcat_flash.configuration_longcat_flash import (+ LongcatFlashConfig,+ )+ from transformers.models.longcat_flash.modeling_longcat_flash import (+ LongcatFlashForCausalLM,+ LongcatFlashModel,+ LongcatFlashPreTrainedModel,+ )++ class LongcatLsaConfig(LongcatFlashConfig):+ model_type = LONGCAT_LSA_MODEL_TYPE++ def __init__(+ self,+ oe_vocab_size_ratio = None,+ oe_neighbor_num = None,+ oe_split_num = None,+ index_n_heads = None,+ index_head_dim = None,+ index_topk = None,+ oe_ngram_vocab_size = None,+ **kwargs,+ ):+ if oe_vocab_size_ratio is None:+ oe_vocab_size_ratio = kwargs.pop("ngram_vocab_size_ratio", None)+ if oe_neighbor_num is None:+ oe_neighbor_num = kwargs.pop("emb_neighbor_num", None)+ if oe_split_num is None:+ oe_split_num = kwargs.pop("emb_split_num", None)+ self.oe_vocab_size_ratio = oe_vocab_size_ratio+ self.oe_neighbor_num = oe_neighbor_num+ self.oe_split_num = oe_split_num+ self.index_n_heads = index_n_heads+ self.index_head_dim = index_head_dim+ self.index_topk = index_topk+ if "rope_scaling" in kwargs:+ kwargs["rope_scaling"] = _translate_rope_scaling(kwargs["rope_scaling"])+ if isinstance(kwargs.get("rope_parameters"), dict):+ kwargs["rope_parameters"] = _translate_rope_scaling(kwargs["rope_parameters"])+ # Configs omit head_dim; the rotary table must use qk_rope_head_dim.+ if "head_dim" not in kwargs and "qk_rope_head_dim" in kwargs:+ kwargs["head_dim"] = kwargs["qk_rope_head_dim"]+ super().__init__(**kwargs)+ # The n-gram hash radix and table sizes; kept apart from vocab_size so a resized vocab reloads.+ self.oe_ngram_vocab_size = oe_ngram_vocab_size or self.vocab_size++ class _FrozenWeight(nn.Module):+ """Bare ``weight``, not a Linear, so LoRA all-linear and bitsandbytes skip it."""++ def __init__(self, *shape):+ super().__init__()+ self.weight = nn.Parameter(torch.empty(*shape), requires_grad = False)++ class LongcatLsaIndexer(nn.Module):+ """Lightning indexer weights; unused by the dense forward."""++ def __init__(self, config):+ super().__init__()+ heads, head_dim = config.index_n_heads, config.index_head_dim+ self.wq_b = _FrozenWeight(heads * head_dim, config.q_lora_rank)+ self.wk = _FrozenWeight(head_dim, config.hidden_size)+ self.k_norm = _FrozenWeight(head_dim)+ self.weights_proj = _FrozenWeight(heads, config.hidden_size)++ class LongcatNgramEmbedding(nn.Module):+ """Lite's ``NgramEmbedding``. Takes embed_tokens' output, not the module: an accelerate+ hook would otherwise move the whole token table to this card."""++ def __init__(self, config):+ super().__init__()+ self.vocab_size = getattr(config, "oe_ngram_vocab_size", None) or config.vocab_size+ self.m = int(config.oe_vocab_size_ratio * self.vocab_size)+ self.k = int(config.oe_split_num)+ self.n = int(config.oe_neighbor_num)+ eos = config.eos_token_id+ self.eos_token_id = eos[0] if isinstance(eos, (list, tuple)) else eos+ num_embedders = self.k * (self.n - 1)+ emb_dim = config.hidden_size // num_embedders+ self.embedders = nn.ModuleList(+ nn.Embedding(int(self.m + i * 2 + 1), emb_dim) for i in range(num_embedders)+ )+ self.post_projs = nn.ModuleList(+ nn.Linear(emb_dim, config.hidden_size, bias = False) for _ in range(num_embedders)+ )++ def _shifted(+ self,+ context,+ shift,+ resets = None,+ ):+ """``context`` shifted right within each segment, zero outside it. A segment ends at an+ EOS or a -1 history mark, and starts again where ``resets`` is set."""+ length = context.shape[-1]+ positions = torch.arange(length, device = context.device)+ ends = torch.where(+ (context == self.eos_token_id) | (context < 0),+ positions + 1,+ torch.zeros_like(positions),+ )+ starts = torch.cummax(ends, dim = -1).values+ starts = torch.cat([torch.zeros_like(starts[..., :1]), starts[..., :-1]], dim = -1)+ if resets is not None:+ reset_starts = torch.where(resets, positions, torch.zeros_like(positions))+ starts = torch.maximum(starts, torch.cummax(reset_starts, dim = -1).values)+ source = positions - shift+ valid = source >= starts+ gathered = torch.gather(+ context, -1, source.clamp(min = 0).expand_as(context).contiguous()+ )+ return torch.where(valid, gathered, torch.zeros_like(gathered))++ def forward(+ self,+ word_embeds,+ input_ids,+ ngram_context = None,+ resets = None,+ ):+ seq_len = input_ids.shape[-1]+ context = input_ids+ if ngram_context is not None:+ context = torch.cat([ngram_context[..., -(self.n - 1) :], input_ids], dim = -1)+ if resets is not None:+ resets = torch.cat(+ [+ resets.new_zeros(resets.shape[:-1] + (context.shape[-1] - seq_len,)),+ resets,+ ],+ dim = -1,+ )+ x = word_embeds+ context = context.long()+ shifted = {i: self._shifted(context, i - 1, resets) for i in range(2, self.n + 1)}+ for i in range(2, self.n + 1):+ for j in range(self.k):+ index = (i - 2) * self.k + j+ mod = int(self.m + index * 2 + 1)+ ngram_ids = context.clone()+ power = 1+ for order in range(2, i + 1):+ power = (power * self.vocab_size) % mod+ ngram_ids = ngram_ids + shifted[order] * power+ ngram_ids = (ngram_ids % mod)[..., -seq_len:]+ embedder = self.embedders[index]+ projected = self.post_projs[index](+ embedder(ngram_ids.to(embedder.weight.device))+ )+ x = x + projected.to(x.device)+ return x / (1 + self.k * (self.n - 1))++ class LongcatLsaModel(LongcatFlashModel):+ config_class = LongcatLsaConfig++ def __init__(self, config):+ super().__init__(config)+ self.ngram_embeddings = LongcatNgramEmbedding(config)+ # transformers 5 sizes gate_up_proj for routed + zero experts; the checkpoint has routed only.+ routed = config.n_routed_experts+ for layer in self.layers:+ # SGLang uses rms_norm_eps for the MLA latent norms; longcat_flash uses 1e-6.+ for attn in getattr(layer, "self_attn", ()):+ for norm_name in ("q_a_layernorm", "kv_a_layernorm"):+ norm = getattr(attn, norm_name, None)+ if norm is not None and hasattr(norm, "variance_epsilon"):+ norm.variance_epsilon = config.rms_norm_eps+ experts = getattr(getattr(layer, "mlp", None), "experts", None)+ gate_up = getattr(experts, "gate_up_proj", None)+ if isinstance(gate_up, nn.Parameter) and gate_up.shape[0] > routed:+ experts.gate_up_proj = nn.Parameter(+ torch.empty(+ routed, *gate_up.shape[1:], dtype = gate_up.dtype, device = gate_up.device+ )+ )+ if getattr(config, "index_topk", None) and getattr(config, "index_n_heads", None):+ for layer in self.layers:+ layer.self_attn[0].indexer = LongcatLsaIndexer(config)+ self.post_init()++ def forward(+ self,+ input_ids = None,+ attention_mask = None,+ position_ids = None,+ past_key_values = None,+ inputs_embeds = None,+ use_cache = None,+ **kwargs,+ ):+ if inputs_embeds is not None:+ raise ValueError(+ "Unsloth: LongCat-Flash-Lite-Sparse builds its input embeddings from token ids "+ "(word plus n-gram embeddings), so pass `input_ids`, not `inputs_embeds`."+ )+ if input_ids is not None:+ topk = getattr(self.config, "index_topk", None)+ cached = 0+ if past_key_values is not None:+ try:+ cached = int(past_key_values.get_seq_length())+ except Exception:+ cached = 0+ total = cached + input_ids.shape[-1]+ if topk and total > topk and not getattr(self, "_unsloth_lsa_warned", False):+ self._unsloth_lsa_warned = True+ warnings.warn(+ f"Unsloth: LongCat sparse attention selects every key only up to {topk} "+ f"tokens. This {total} token sequence runs dense attention, "+ "which differs from the sparse attention used at inference.",+ stacklevel = 2,+ )+ history = getattr(past_key_values, "_unsloth_ngram_history", None)+ keep = self.ngram_embeddings.n - 1+ context = None if history is None else history[..., -keep:]+ # position_ids restarts (packing, left padding) begin a new n-gram segment.+ resets = None+ pos = position_ids+ if pos is None and attention_mask is not None and attention_mask.dim() == 2:+ # Same positions generate() derives from the mask; only the n-gram uses them.+ mask = attention_mask.long()+ pos = (mask.cumsum(-1) - 1).masked_fill(mask == 0, 0)[+ ..., -input_ids.shape[-1] :+ ]+ if pos is not None and pos.shape[-1] == input_ids.shape[-1]:+ pos = pos.expand(input_ids.shape[0], -1)+ resets = torch.diff(pos, prepend = pos[..., :1] - 1, dim = -1) != 1+ inputs_embeds = self.ngram_embeddings(+ self.embed_tokens(input_ids), input_ids, context, resets+ )+ # No cache in training: transformers' packed mask needs past_key_values None.+ if use_cache is None:+ use_cache = getattr(self.config, "use_cache", False) and not self.training+ if use_cache and past_key_values is None:+ from transformers.cache_utils import DynamicCache+ try:+ past_key_values = DynamicCache(config = self.config)+ except TypeError:+ past_key_values = DynamicCache()+ if past_key_values is not None:+ # Id history rides on the cache so beam reorders and crops move it.+ seen = input_ids if history is None else torch.cat([history, input_ids], dim = -1)+ if resets is not None:+ # Mark ids before each row's last restart -1 so later decode steps skip them.+ offset = seen.shape[-1] - input_ids.shape[-1]+ idx = torch.arange(input_ids.shape[-1], device = resets.device) + offset+ last = torch.where(resets, idx, torch.zeros_like(idx)).amax(+ -1, keepdim = True+ )+ before = torch.arange(seen.shape[-1], device = seen.device) < last.to(+ seen.device+ )+ seen = seen.masked_fill(before, -1)+ past_key_values._unsloth_ngram_history = seen+ cache_class = type(past_key_values)+ if cache_class not in _NGRAM_TRACKING_CACHES.values():+ past_key_values.__class__ = _ngram_tracking_cache_class(cache_class)+ return super().forward(+ input_ids = None,+ attention_mask = attention_mask,+ position_ids = position_ids,+ past_key_values = past_key_values,+ inputs_embeds = inputs_embeds,+ use_cache = use_cache,+ **kwargs,+ )++ class LongcatCausalLM(LongcatFlashForCausalLM):+ config_class = LongcatLsaConfig+ _keys_to_ignore_on_load_unexpected = [r"model\.mtp\..*"]+ # Router and indexer weights_proj stay F32 as in SGLang.+ _keep_in_fp32_modules_strict = ["router", "weights_proj"]++ def __init__(self, config):+ LongcatFlashPreTrainedModel.__init__(self, config)+ self.model = LongcatLsaModel(config)+ self.vocab_size = config.vocab_size+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias = False)+ self.post_init()++ # transformers 4 applies key_mapping only to VLM classes: pass it on load, undo it on save.+ @classmethod+ def from_pretrained(cls, *args, **kwargs):+ if not _has_conversion_registry() and kwargs.get("key_mapping") is None:+ kwargs["key_mapping"] = {"^" + src: dst for src, dst in _OE_RENAMES}+ loaded = super().from_pretrained(*args, **kwargs)+ # Loading drops requires_grad = False; re-freeze the indexer.+ model = loaded[0] if isinstance(loaded, tuple) else loaded+ for module in model.modules():+ if isinstance(module, LongcatLsaIndexer):+ module.requires_grad_(False)+ return loaded++ def save_pretrained(self, *args, **kwargs):+ if not _has_conversion_registry():+ state_dict = kwargs.get("state_dict")+ if state_dict is None:+ state_dict = self.state_dict()+ kwargs["state_dict"] = {+ _to_checkpoint_key(key): value for key, value in state_dict.items()+ }+ return super().save_pretrained(*args, **kwargs)++ _CLASSES = (LongcatLsaConfig, LongcatCausalLM)+ return _CLASSES+++def _has_conversion_registry():+ try:+ from transformers.conversion_mapping import register_checkpoint_conversion_mapping+ except Exception:+ return False+ return True+++def _to_checkpoint_key(key):+ for source, target in _OE_RENAMES:+ if key.startswith(target):+ return source + key[len(target) :]+ return key+++_CONVERSIONS_REGISTERED = False+++def _register_conversions():+ """Give this model_type longcat_flash's transformers 5 conversions plus the n-gram renames."""+ global _CONVERSIONS_REGISTERED+ if _CONVERSIONS_REGISTERED:+ return+ try:+ from transformers.conversion_mapping import (+ get_checkpoint_conversion_mapping,+ register_checkpoint_conversion_mapping,+ )+ from transformers.core_model_loading import WeightRenaming+ except Exception:+ return+ renames = [+ WeightRenaming(source_patterns = source, target_patterns = target)+ for source, target in _OE_RENAMES+ ]+ base = get_checkpoint_conversion_mapping("longcat_flash") or []+ register_checkpoint_conversion_mapping(+ LONGCAT_LSA_MODEL_TYPE, renames + list(base), overwrite = True+ )+ _CONVERSIONS_REGISTERED = True+ # peft's per-expert LoRA conversion table; must come after the registration above.+ for module_name in (+ "transformers.conversion_mapping",+ "peft.utils.transformers_weight_conversion",+ ):+ table = getattr(sys.modules.get(module_name), "_MODEL_TO_CONVERSION_PATTERN", None)+ if isinstance(table, dict) and "longcat_flash" in table:+ table.setdefault(LONGCAT_LSA_MODEL_TYPE, table["longcat_flash"])+++def register_longcat_lsa():+ """Register the config and model classes with the Auto factories (idempotent)."""+ from transformers import AutoConfig, AutoModelForCausalLM++ config_class, model_class = _classes()+ try:+ AutoConfig.register(LONGCAT_LSA_MODEL_TYPE, config_class)+ except ValueError:+ pass+ try:+ AutoModelForCausalLM.register(config_class, model_class)+ except ValueError:+ pass+ _register_conversions()+ return config_class, model_class+++def load_longcat_lsa_config(pretrained_model_name_or_path, *args, **kwargs):+ config_class, _ = register_longcat_lsa()+ kwargs.pop("trust_remote_code", None)+ kwargs.pop("code_revision", None)+ return config_class.from_pretrained(pretrained_model_name_or_path, *args, **kwargs)diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex dae92111181..11bf9a9d097 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -1269,6 +1269,19 @@ def _get_total_transformer_layers(model):return None+def _architecture_skip_modules(model_types):+ """Extra modules kept out of bitsandbytes; shared by the device-map planner and the load."""+ model_types = model_types or []+ skip = []+ # Nemotron-H uses 'mixer' (not 'mamba') for Mamba layers, whose fused kernels pass out_proj.weight straight to F.linear and fail with quantized Params4bit, so skip out_proj.+ if any(mt == "nemotron_h" for mt in model_types):+ skip.append("out_proj")+ # LongCat-Flash MLA: NF4 on q_b_proj / kv_b_proj badly hurts loss; both are small.+ if any(mt in ("longcat_flash", "longcat_flash_lsa") for mt in model_types):+ skip.extend(("q_b_proj", "kv_b_proj"))+ return skip++def _cast_unquantized_floats(model, dtype):"""Cast every floating parameter and buffer that is not a quantized weight."""for tensor in list(model.parameters()) + list(model.buffers()):@@ -1902,9 +1915,7 @@ class FastBaseModel:load_in_4bit = load_in_4bit,load_in_8bit = load_in_8bit,quantization_config = user_quantization_config,- extra_skip_modules = ["out_proj"]- if any(mt == "nemotron_h" for mt in (model_types or []))- else None,+ extra_skip_modules = _architecture_skip_modules(model_types) or None,),)@@ -1957,10 +1968,7 @@ class FastBaseModel:tokenizer_only = True,)- _skip_modules = SKIP_QUANTIZATION_MODULES.copy()- # Nemotron-H uses 'mixer' (not 'mamba') for Mamba layers, whose fused kernels pass out_proj.weight straight to F.linear and fail with quantized Params4bit, so skip out_proj.- if any(mt == "nemotron_h" for mt in (model_types or [])):- _skip_modules.append("out_proj")+ _skip_modules = SKIP_QUANTIZATION_MODULES.copy() + _architecture_skip_modules(model_types)if load_in_4bit:bnb_config = BitsAndBytesConfig(