Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

26.1 KB
Raw Blame
"""DeepSeek-V4-Flash from a llama.cpp ``deepseek4`` GGUF (unsloth ``DeepSeek-V4-Flash-0731-GGUF``
and the Vision-Exp release): config from the GGUF metadata, tensor names, and the dense state dict.

The GGUF geometry matches the HF checkpoint (43 MLA blocks with sliding-window sparse
attention, per-layer KV compressors and a Lightning Indexer on the ratio-4 blocks, mHC x4
residual streams, a 256-expert sqrtsoftplus MoE with hash routing on the first blocks and
a shared expert), so ``parse_gguf_config`` builds the family's own :class:`DeepseekV4Args`
from the GGUF KV metadata and reuses the shared ModelConfig construction
(:func:`deepseek_v4.config.model_config_from_args`).

The converter drops the MTP blocks and writes ``attention.compress_ratios`` as a single
placeholder 0, so the per-layer compressor layout is recovered from the tensor table: an
``indexer.*`` tensor means ratio 4, a compressor without an indexer means 128, unless its
``wkv`` carries the CSA overlap's double rows (also ratio 4). Routed experts serve from
the GGUF expert banks (``freetoken.models.gguf.experts``) like every GGUF family.

A ``deepseek4v`` mmproj beside the Vision-Exp text file adds the vision tower under
``visual.``: the converter splits the fused SwiGLU ``w1`` into ``ffn_gate`` / ``ffn_up``
rows and reshapes the patch embedding into the conv layout (both undone at load); ggml-org
writes the attention block fused and unsloth split. A vision trunk also carries the
``exp_probs_b_vl`` router biases and drops the hash layers' never-applied text bias.
"""

from __future__ import annotations

import os
import re
from dataclasses import replace
from typing import TYPE_CHECKING, Iterator

import numpy as np
import torch

from freetoken.layers.quantization.ggml_types import is_quantized
from freetoken.models.gguf.mmproj import Mmproj, find_mmproj, log_mmproj, mmproj_tensor
from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index, gguf_tensor_names, load_gguf_metadata
from freetoken.models.gguf.tensors import check_scales, fused, matrix
from freetoken.utils import init_logger

from .args import DeepseekV4Args
from .config import IMAGE_TOKEN_ID, model_config_from_args, parse_vision_config

if TYPE_CHECKING:
    from freetoken.models.config import ModelConfig
    from freetoken.models.gguf.config import GgufConfigShim

logger = init_logger(__name__)

# engine module leaf (after "model.layers.N.") -> GGUF tensor names (without ".weight")
_LAYER_LEAVES: dict[str, tuple[str, ...]] = {
    "attn.wq_a": ("attn_q_a",),
    "attn.wq_b": ("attn_q_b",),
    "attn.wkv": ("attn_kv",),
    "attn.wo_b": ("attn_output_b",),
    "attn.compressor.wkv": ("attn_compressor_kv",),
    "attn.compressor.wgate": ("attn_compressor_gate",),
    "attn.indexer.wq_b": ("indexer.attn_q_b",),
    "attn.indexer.weights_proj": ("indexer.proj",),
    "attn.indexer.compressor.wkv": ("indexer_compressor_kv",),
    "attn.indexer.compressor.wgate": ("indexer_compressor_gate",),
    "ffn.shared_experts.w1": ("ffn_gate_shexp",),
    "ffn.shared_experts.w3": ("ffn_up_shexp",),
    "ffn.shared_experts.w2": ("ffn_down_shexp",),
    # routed experts (GGUF expert banks), gate / up / down order
    "ffn.experts.0.w1": ("ffn_gate_exps",),
    "ffn.experts.0.w3": ("ffn_up_exps",),
    "ffn.experts.0.w2": ("ffn_down_exps",),
}
_TOP_LEAVES: dict[str, tuple[str, ...]] = {
    "model.embed": ("token_embd",),
    "model.head": ("output",),
}
_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")


def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
    """Engine module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""
    if name in _TOP_LEAVES:
        return _TOP_LEAVES[name]
    m = _LAYER_RE.match(name)
    if m is None:
        return ()
    return tuple(f"blk.{m.group(1)}.{leaf}" for leaf in _LAYER_LEAVES.get(m.group(2), ()))


def _compress_ratios(index: dict[str, GgufTensorInfo], head_dim: int, num_layers: int) -> tuple[int, ...]:
    """Per-layer compressor ratios from the tensor table, which every released file's
    ``attention.compress_ratios`` array (43 trunk entries ahead of 3 MTP zeros, which
    llama.cpp requires) confirms element for element.

    An ``indexer.proj`` tensor exists exactly on the ratio-4 blocks; a compressor without an
    indexer is a 128 block, unless its ``wkv`` has the CSA overlap's double rows (ratio 4)."""
    ratios = []
    for layer in range(num_layers):
        blk = f"blk.{layer}"
        if f"{blk}.indexer.proj.weight" in index:
            ratios.append(4)
            continue
        info = index.get(f"{blk}.attn_compressor_kv.weight")
        ratios.append(0 if info is None else 4 if info.ne[-1] == 2 * head_dim else 128)
    return tuple(ratios)


def parse_gguf_config(shim: "GgufConfigShim") -> ModelConfig:
    md = shim.metadata
    prefix = shim.model_type

    check_scales(shim.model_path)

    def kv(key: str, default=None):
        return md.get(f"{prefix}.{key}", default)

    def need(key: str):
        val = kv(key)
        if val is None:
            raise KeyError(f"missing GGUF metadata key {prefix}.{key}")
        return val

    names = gguf_tensor_names(shim.model_path)
    if any(n.startswith(("vision.", "aligner.")) for n in names):
        raise NotImplementedError(
            f"{shim.model_path}: deepseek4 GGUF with an inline vision tower is not served "
            "(llama.cpp ships the tower as a separate mmproj file)"
        )

    # a deepseek4v mmproj beside the file fills the shim's flat vision fields; a text-only
    # process nulls vision_n_layers on its copy, and None here is the same answer
    vc = parse_vision_config(shim)

    # the clamp is per-layer in the GGUF (and the shared expert carries its own array); the model
    # takes one value, so anything non-uniform is refused instead of silently taking the first
    clamp = need("swiglu_clamp_exp")
    clamp_vals = list(clamp) if isinstance(clamp, list) else [clamp]
    clamp_shexp = kv("swiglu_clamp_shexp")
    shexp_vals = list(clamp_shexp) if isinstance(clamp_shexp, list) else [clamp_shexp] if clamp_shexp is not None else []
    if len(set(map(float, clamp_vals + shexp_vals))) != 1:
        raise NotImplementedError(f"{shim.model_path}: non-uniform swiglu_clamp_exp / swiglu_clamp_shexp is not served")

    from freetoken.third_party.gguf.constants import ExpertGatingFuncType

    if int(need("expert_gating_func")) != int(ExpertGatingFuncType.SQRTSOFTPLUS):
        raise NotImplementedError(f"{shim.model_path}: expert_gating_func other than sqrtsoftplus is not served")

    # every official DeepSeek-V4 config carries the same yarn block (V4-Flash, 0731, Pro,
    # Pro-0813: factor 16 over 65536, betas 32 / 1). The no-yarn files on HF lost the whole
    # block in conversion -- llama.cpp serves them unscaled, which is wrong for them -- so a
    # wholly missing block falls back to the family values and a partly present one refuses.
    scaling = {k: kv(f"rope.scaling.{k}") for k in
               ("type", "factor", "original_context_length", "yarn_beta_fast", "yarn_beta_slow")}
    if scaling["type"] not in (None, "yarn"):
        raise NotImplementedError(f"{shim.model_path}: rope scaling other than yarn is not served")
    missing = [k for k, v in scaling.items() if v is None]
    if missing:
        if len(missing) != len(scaling):
            raise ValueError(
                f"{shim.model_path}: rope.scaling is only partly present "
                f"(missing {', '.join(missing)}); the file is inconsistent"
            )
        logger.warning(
            f"{shim.model_path}: no rope.scaling block; using the family values "
            "(yarn, factor 16, original 65536, betas 32 / 1)"
        )
        scaling = {"type": "yarn", "factor": 16.0, "original_context_length": 65536,
                   "yarn_beta_fast": 32.0, "yarn_beta_slow": 1.0}

    head_dim = int(need("attention.key_length"))
    if int(need("attention.value_length")) != head_dim:
        raise NotImplementedError(f"{shim.model_path}: attention key and value lengths differ")
    if int(kv("attention.head_count_kv", 1) or 1) != 1:
        raise NotImplementedError(f"{shim.model_path}: more than one latent KV head is not served")

    num_layers = int(need("block_count"))
    index = gguf_tensor_index(shim.model_path)

    # REAP exports prune experts per layer: as a per-layer expert_count array, or with the
    # scalar kept but the expert tensors shrunk. llama.cpp builds every layer from one
    # n_expert and refuses both shapes; the model has no per-layer expert count to express.
    experts_kv = need("expert_count")
    if isinstance(experts_kv, list):
        raise NotImplementedError(
            f"{shim.model_path}: a per-layer expert_count array (REAP-style prune) is not served"
        )
    n_experts = int(experts_kv)
    for name, info in index.items():
        if name.endswith(".ffn_gate_exps.weight") and info.ne[-1] != n_experts:
            raise NotImplementedError(
                f"{shim.model_path}: {name} carries {info.ne[-1]} experts but expert_count says "
                f"{n_experts} (REAP-style per-layer prunes are not served)"
            )
    args = DeepseekV4Args(
        max_batch_size=1,
        vocab_size=int(shim.vocab_size),
        dim=int(need("embedding_length")),
        moe_inter_dim=int(need("expert_feed_forward_length")),
        n_layers=num_layers,
        n_hash_layers=int(kv("hash_layer_count", 0) or 0),
        n_heads=int(need("attention.head_count")),
        n_routed_experts=int(need("expert_count")),
        n_shared_experts=int(kv("expert_shared_count", 1) or 1),
        n_activated_experts=int(need("expert_used_count")),
        score_func="sqrtsoftplus",
        route_scale=float(need("expert_weights_scale")),
        swiglu_limit=float(clamp_vals[0]),
        q_lora_rank=int(need("attention.q_lora_rank")),
        head_dim=head_dim,
        rope_head_dim=int(need("rope.dimension_count")),
        norm_eps=float(need("attention.layer_norm_rms_epsilon")),
        o_groups=int(need("attention.output_group_count")),
        o_lora_rank=int(need("attention.output_lora_rank")),
        window_size=int(need("attention.sliding_window")),
        compress_ratios=_compress_ratios(index, head_dim, num_layers),
        compress_rope_theta=float(kv("attention.compress_rope_freq_base", 160000.0) or 160000.0),
        original_seq_len=int(scaling["original_context_length"]),
        rope_theta=float(need("rope.freq_base")),
        rope_factor=float(scaling["factor"]),
        beta_fast=float(scaling["yarn_beta_fast"]),
        beta_slow=float(scaling["yarn_beta_slow"]),
        index_n_heads=int(need("attention.indexer.head_count")),
        index_head_dim=int(need("attention.indexer.key_length")),
        index_topk=int(need("attention.indexer.top_k")),
        hc_mult=int(need("hyper_connection.count")),
        hc_sinkhorn_iters=int(need("hyper_connection.sinkhorn_iterations")),
        hc_eps=float(need("hyper_connection.epsilon")),
        n_mtp_layers=0,  # the converter drops the MTP blocks
        # vision fields only when a mmproj filled them (the defaults are the text model's)
        **({} if vc is None else dict(
            vision_n_layers=vc.vision_n_layers,
            vision_dim=vc.vision_dim,
            vision_n_heads=vc.vision_n_heads,
            vision_inter_dim=vc.vision_inter_dim,
            vision_patch_size=vc.vision_patch_size,
            vision_rope_theta=vc.vision_rope_theta,
            vision_downsample_ratio=vc.vision_downsample_ratio,
            vision_max_n_token=vc.vision_max_n_token,
            vision_min_pixels=vc.vision_min_pixels,
            vision_max_wh_ratio=vc.vision_max_wh_ratio,
        )),
    )
    config = model_config_from_args(args, max_position=int(need("context_length")), vision_config=vc)
    return replace(config, expert_quant="gguf")


# ----------------------------------------------------------------------------------------------
# Vision tower (a deepseek4v mmproj)
# ----------------------------------------------------------------------------------------------

_PROJECTOR = "deepseek4v"
_IMAGE_TOKEN = "<|deepseek_image|>"
# no mmproj key carries these; llama.cpp hardcodes them and its converter asserts them
_ROPE_THETA = 10000.0
_MAX_N_TOKEN = 384
_MAX_WH_RATIO = 8

# top-level mmproj tensor -> the tower name under ``visual.``
_MM_TOP: dict[str, str] = {
    "v.patch_embd.bias": "vision.patch_embed.proj.bias",
    "v.post_ln.weight": "vision.norm.weight",
    "mm.1.weight": "aligner.w1.weight",
    "mm.1.bias": "aligner.w1.bias",
    "mm.2.weight": "aligner.w2.weight",
    "mm.2.bias": "aligner.w2.bias",
    "v.token_embd.img_start": "image_start",
    "v.token_embd.img_end": "image_end",
    "v.image_newline": "image_newline",
    "v.token_embd.img_pad": "image_pad",
}


def gguf_mm_config(text_md: dict, mmproj: Mmproj, vocab_size: int, *, model_path: str = "") -> tuple[dict, dict]:
    """The flat HF config keys (the vision dims sit at the top level on this family, like the
    checkpoint's inference config) and the preprocessor dict of the tower in ``mmproj``, for
    the text GGUF whose metadata is ``text_md``."""
    if mmproj.projector != _PROJECTOR:
        raise ValueError(f"mmproj {mmproj.path} is a {mmproj.projector!r} tower; DeepSeek-V4 needs {_PROJECTOR!r}")
    md, index = load_gguf_metadata(mmproj.path), gguf_tensor_index(mmproj.path)
    # the converter writes use_silu=True only for the SwiGLU tower; the module has no gelu MLP
    if md.get("clip.use_silu") is not True:
        raise NotImplementedError(f"mmproj {mmproj.path}: only the SwiGLU tower is served")

    def kv(key: str):
        return md[f"clip.vision.{key}"]

    arch = text_md["general.architecture"]
    text_hidden = int(text_md[f"{arch}.embedding_length"])
    if int(kv("projection_dim")) != text_hidden:
        raise ValueError(f"mmproj {mmproj.path} projects to {int(kv('projection_dim'))} but the text model is {text_hidden} wide")
    tokens = text_md["tokenizer.ggml.tokens"]
    if _IMAGE_TOKEN not in tokens:
        raise ValueError(
            f"the text GGUF tokenizer has no {_IMAGE_TOKEN} token (the 0731 text-only release "
            "uses another dialect; pair the mmproj with a Vision-Exp text GGUF)"
        )
    # the model config and the processor hardcode the family id; a file with the token at
    # another id would fail per request instead of here
    if tokens.index(_IMAGE_TOKEN) != IMAGE_TOKEN_ID:
        raise ValueError(
            f"the text GGUF carries {_IMAGE_TOKEN} at id {tokens.index(_IMAGE_TOKEN)}, not the family's {IMAGE_TOKEN_ID}"
        )
    # a ValueError so auto-discovery skips this mmproj (text only); only --mmproj fails
    if not any(n.endswith(".exp_probs_b_vl.bias") for n in gguf_tensor_names(model_path)):
        raise ValueError(
            f"the text trunk {os.path.basename(model_path)} predates vision routing "
            "(no exp_probs_b_vl; converted before llama.cpp #28154) -- pair a Vision-Exp "
            "text GGUF reconverted after 2026-09-02"
        )
    for sentinel in ("v.token_embd.img_start", "v.token_embd.img_end", "v.image_newline", "v.token_embd.img_pad"):
        if sentinel not in index:
            raise ValueError(f"mmproj {mmproj.path}: sentinel tensor {sentinel!r} is missing")
    # the tower hardcodes eps 1e-6 (vision.py); the converter warns it is NOT the text 1e-20
    if abs(float(kv("attention.layer_norm_epsilon")) - 1e-6) > 1e-9:
        raise NotImplementedError(f"mmproj {mmproj.path}: vision layer_norm_epsilon {kv('attention.layer_norm_epsilon')} is not the served 1e-6")
    # qkv dialect: ggml-org writes the fused attn_qkv, unsloth the split attn_q/k/v (clip.cpp loads both)
    split = all(f"v.blk.0.attn_{q}.{part}" in index for q in "qkv" for part in ("weight", "bias"))
    if split == ("v.blk.0.attn_qkv.weight" in index):
        raise ValueError(f"mmproj {mmproj.path}: the attn tensors are neither the fused attn_qkv nor the split attn_q/k/v layout (or carry both)")
    patch, dim = int(kv("patch_size")), int(kv("embedding_length"))
    # the converter reshapes the Linear's [dim, 3*p*p] kernel into the conv [p, p, 3, dim]
    if tuple(index["v.patch_embd.weight"].ne) != (patch, patch, 3, dim):
        raise ValueError(f"mmproj {mmproj.path}: v.patch_embd.weight {tuple(index['v.patch_embd.weight'].ne)} is not the [p, p, 3, dim] conv layout")
    # clip.vision.image_size (672) is deliberately unread: a converter compat/warmup value,
    # the tower is native-resolution

    hf_fields = {
        "vision_n_layers": int(kv("block_count")),
        "vision_dim": dim,
        "vision_n_heads": int(kv("attention.head_count")),
        "vision_inter_dim": int(kv("feed_forward_length")),
        "vision_patch_size": patch,
        "vision_rope_theta": _ROPE_THETA,
        "vision_downsample_ratio": int(kv("projector.scale_factor")),
        "vision_max_n_token": _MAX_N_TOKEN,
        "vision_min_pixels": int(kv("image_min_pixels")),
        "vision_max_wh_ratio": _MAX_WH_RATIO,
        # parse_vision_config reads the aligner's target width off hidden_size
        "hidden_size": text_hidden,
        "image_token_id": tokens.index(_IMAGE_TOKEN),
    }
    preprocessor = {
        # no transformers class exists; DSV4MMProcessor._image_processor returns None
        "image_processor_type": "DeepseekV4ImageProcessor",
        "patch_size": patch,
        "temporal_patch_size": 1,
        "merge_size": int(kv("projector.scale_factor")),
        "image_mean": [float(v) for v in kv("image_mean")],
        "image_std": [float(v) for v in kv("image_std")],
        "min_pixels": int(kv("image_min_pixels")),
        "max_n_token": _MAX_N_TOKEN,
    }
    return hf_fields, preprocessor


def iter_mmproj_weights(mmproj: Mmproj) -> Iterator[tuple[str, torch.Tensor]]:
    """The tower state dict under ``visual.``, named as the HF checkpoints load."""
    index = gguf_tensor_index(mmproj.path)
    fused = "v.blk.0.attn_qkv.weight" in index
    n_blocks = max(int(n.split(".")[2]) for n in index if n.startswith("v.blk.")) + 1

    pe = index["v.patch_embd.weight"]
    # ggml order [p, p, 3, out] -> torch [out, 3, p, p] -> the module's flat [out, 3*p*p]
    yield "visual.vision.patch_embed.proj.weight", mmproj_tensor(pe).reshape(pe.ne[3], 3 * pe.ne[0] * pe.ne[0])
    for gguf_name, ft_name in _MM_TOP.items():
        yield f"visual.{ft_name}", mmproj_tensor(index[gguf_name])

    consumed = {"v.patch_embd.weight", *_MM_TOP}
    for layer in range(n_blocks):
        blk, dst = f"v.blk.{layer}", f"visual.vision.blocks.{layer}"
        if fused:
            for part in ("weight", "bias"):
                yield f"{dst}.attn.wqkv.{part}", mmproj_tensor(index[f"{blk}.attn_qkv.{part}"])
            consumed.update((f"{blk}.attn_qkv.weight", f"{blk}.attn_qkv.bias"))
        else:
            for part in ("weight", "bias"):
                yield f"{dst}.attn.wqkv.{part}", torch.cat(
                    [mmproj_tensor(index[f"{blk}.attn_{q}.{part}"]) for q in "qkv"], dim=0
                )
            consumed.update(f"{blk}.attn_{q}.{part}" for q in "qkv" for part in ("weight", "bias"))
        for part in ("weight", "bias"):
            yield f"{dst}.attn.wo.{part}", mmproj_tensor(index[f"{blk}.attn_out.{part}"])
        # the converter chunks the fused SwiGLU w1 rows into gate / up; cat them back
        yield f"{dst}.mlp.w1.weight", torch.cat(
            [mmproj_tensor(index[f"{blk}.ffn_gate.weight"]), mmproj_tensor(index[f"{blk}.ffn_up.weight"])], dim=0
        )
        yield f"{dst}.mlp.w2.weight", mmproj_tensor(index[f"{blk}.ffn_down.weight"])
        yield f"{dst}.norm1.weight", mmproj_tensor(index[f"{blk}.ln1.weight"])
        yield f"{dst}.norm2.weight", mmproj_tensor(index[f"{blk}.ln2.weight"])
        consumed.update(
            f"{blk}.attn_out.{part}" for part in ("weight", "bias")
        )
        consumed.update(f"{blk}.ffn_{m}.weight" for m in ("gate", "up", "down"))
        consumed.update((f"{blk}.ln1.weight", f"{blk}.ln2.weight"))

    leftovers = set(index) - consumed
    if leftovers:
        raise ValueError(f"mmproj {mmproj.path}: unexpected tensors {sorted(leftovers)[:5]}")


# ----------------------------------------------------------------------------------------------
# Weights
# ----------------------------------------------------------------------------------------------

def _float(info: GgufTensorInfo) -> torch.Tensor:
    """A float tensor in torch order, dequantizing packed ggml rows: dynamic quants quantize
    tensors the module consumes as floats (the mHC mixers, the compressor gates, the router).
    Trailing unit dims in ``ne`` are dropped -- some writers ship tensors 4-D (llama.cpp's
    loaders try that shape first) and a trailing 1 carries no elements."""
    ne = info.ne
    while len(ne) > 1 and ne[-1] == 1:
        ne = ne[:-1]
    if not is_quantized(info.ggml_type):
        from freetoken.models.gguf.tensors import float_tensor

        return float_tensor(info).view(*reversed(ne))
    from freetoken.third_party.gguf import quants
    from freetoken.third_party.gguf.constants import GGMLQuantizationType

    flat = quants.dequantize(np.array(info.data()), GGMLQuantizationType(info.ggml_type))
    return torch.from_numpy(flat).view(*reversed(ne))


def _tid2eid(info: GgufTensorInfo) -> torch.Tensor:
    """The hash-route table as the module's int64 [vocab, top_k]."""
    ne = info.ne
    while len(ne) > 1 and ne[-1] == 1:
        ne = ne[:-1]
    raw = np.frombuffer(info.data(), dtype=np.int32).copy()
    return torch.from_numpy(raw).view(*reversed(ne)).to(torch.int64)


def iter_gguf_weights(
    model_path: str,
    device: torch.device,
    *,
    include_moe_experts: bool,
    include_non_moe: bool,
    include_vision: bool = True,
) -> Iterator[tuple[str, torch.Tensor]]:
    """The dense state dict from the GGUF, in the names the HF loader yields; the routed experts
    load as offload banks (``freetoken.models.gguf.experts``)."""
    if include_moe_experts:
        raise ValueError(
            "DeepSeek-V4 routed experts are served from the offload cache; "
            "run with --moe-strategy offload (include_moe_experts must be False)."
        )
    if not include_non_moe:
        return
    from freetoken.utils import cached_load_hf_config

    config = parse_gguf_config(cached_load_hf_config(model_path))
    vision = include_vision and config.is_multimodal
    log_mmproj(model_path, vision)
    args = config.dsv4_args
    index = gguf_tensor_index(model_path)
    hidden = config.hidden_size

    def t(name: str) -> GgufTensorInfo:
        return index[name if name in index else f"{name}.weight"]

    def bf16(name: str) -> torch.Tensor:
        return _float(t(name)).to(torch.bfloat16)

    yield from fused("model.embed", [t("token_embd")], [matrix(t("token_embd"))], hidden)
    yield from fused("model.head", [t("output")], [matrix(t("output"))], hidden)
    yield "model.norm.weight", bf16("output_norm")
    for part in ("fn", "base", "scale"):
        yield f"model.hc_head_{part}", _float(t(f"output_hc_{part}"))

    for layer in range(config.num_layers):
        blk, pre = f"blk.{layer}", f"model.layers.{layer}"
        m = f"{pre}.attn"
        yield f"{pre}.attn_norm.weight", bf16(f"{blk}.attn_norm")
        yield f"{pre}.ffn_norm.weight", bf16(f"{blk}.ffn_norm")
        # mHC mixing tensors, fp32 on every layer
        for sub in ("attn", "ffn"):
            for part in ("fn", "base", "scale"):
                yield f"{pre}.hc_{sub}_{part}", _float(t(f"{blk}.hc_{sub}_{part}"))

        for proj, src, k in (
            ("wq_a", "attn_q_a", hidden),
            ("wq_b", "attn_q_b", args.q_lora_rank),
            ("wkv", "attn_kv", hidden),
            ("wo_b", "attn_output_b", args.o_groups * args.o_lora_rank),
        ):
            info = t(f"{blk}.{src}")
            yield from fused(f"{m}.{proj}", [info], [matrix(info)], k)
        yield f"{m}.q_norm.weight", bf16(f"{blk}.attn_q_a_norm")
        yield f"{m}.kv_norm.weight", bf16(f"{blk}.attn_kv_a_norm")
        # wo_a feeds a bf16 einsum; the HF path dequantizes its fp8 the same way
        yield f"{m}.wo_a", _float(t(f"{blk}.attn_output_a")).to(torch.bfloat16)
        yield f"{m}.attn_sink", _float(t(f"{blk}.attn_sinks"))

        ratio = args.compress_ratios[layer]
        if ratio:
            compressors = ((f"{m}.compressor", f"{blk}.attn_compressor"),)
            if ratio == 4:
                compressors += ((f"{m}.indexer.compressor", f"{blk}.indexer_compressor"),)
            for c_dst, c_src in compressors:
                yield f"{c_dst}.ape", _float(t(f"{c_src}_ape"))
                # the compressors read wkv / wgate through a dense bf16 einsum, not the quant
                # method (the registry exempts them, like the HF spec), so a tensor the file
                # quantized dequantizes here
                for proj, suf in (("wkv", "kv"), ("wgate", "gate")):
                    yield f"{c_dst}.{proj}.weight", bf16(f"{c_src}_{suf}")
                yield f"{c_dst}.norm.weight", bf16(f"{c_src}_norm")
            if ratio == 4:
                for proj, src, k in (
                    ("indexer.wq_b", "indexer.attn_q_b", args.q_lora_rank),
                    ("indexer.weights_proj", "indexer.proj", hidden),
                ):
                    info = t(f"{blk}.{src}")
                    yield from fused(f"{m}.{proj}", [info], [matrix(info)], k)

        g = f"{pre}.ffn.gate"
        yield f"{g}.weight", bf16(f"{blk}.ffn_gate_inp")
        if layer < args.n_hash_layers:
            yield f"{g}.tid2eid", _tid2eid(t(f"{blk}.ffn_gate_tid2eid"))
            if vision:
                # the converter drops the hash layers' text bias (the reference never applies
                # it); the module builds the slot on a vision build, so it zero-fills
                yield f"{g}.bias", torch.zeros(args.n_routed_experts, dtype=torch.float32)
        else:
            yield f"{g}.bias", _float(t(f"{blk}.exp_probs_b.bias"))
        if vision:
            yield f"{g}.bias_vl", _float(t(f"{blk}.exp_probs_b_vl.bias"))
        for proj, src in (("w1", "ffn_gate_shexp"), ("w3", "ffn_up_shexp"), ("w2", "ffn_down_shexp")):
            info = t(f"{blk}.{src}")
            k = hidden if proj != "w2" else args.moe_inter_dim
            yield from fused(f"{pre}.ffn.shared_experts.{proj}", [info], [matrix(info)], k)

    if vision:
        yield from iter_mmproj_weights(find_mmproj(model_path))


__all__ = ["gguf_mm_config", "iter_gguf_weights", "iter_mmproj_weights", "parse_gguf_config", "to_gguf"]