Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
29.7 KB
Raw Blame
#!/usr/bin/env python3
"""Generate the deterministic GLM-5.3-Flash text oracle contract.

The fixture is deliberately produced by Hugging Face's official
Glm5NextTextModel.  It covers KDA, DSA, mHC, dense FFN and routed MoE before a
Colibri runtime exists, so the future engine has an independent target instead
of validating itself.
"""
from __future__ import annotations

import argparse
import json
import shutil
import types
from collections import OrderedDict
from pathlib import Path


SEED = 1243
VOCAB = 128
HIDDEN = 128
LAYERS = 4
HEADS = 4
HEAD_DIM = 32
EXPERTS = 4
TOPK = 2
MOE = 128


def require_dependencies():
    try:
        import torch
        import transformers
        from safetensors.torch import save_file
        from transformers import Glm5NextTextConfig
        from transformers.models.glm5_next.modeling_glm5_next import Glm5NextTextModel
    except Exception as exc:  # pragma: no cover - regeneration diagnostic
        raise SystemExit(
            "GLM-5.3 tiny generation requires the pinned PyTorch, "
            "Transformers and safetensors dependencies"
        ) from exc
    if transformers.__version__ != "5.16.1":
        raise SystemExit(
            f"expected Transformers 5.16.1, found {transformers.__version__}"
        )
    return torch, transformers, save_file, Glm5NextTextConfig, Glm5NextTextModel


def config_kwargs() -> dict[str, object]:
    return {
        "vocab_size": VOCAB,
        "hidden_size": HIDDEN,
        "intermediate_size": 256,
        "moe_intermediate_size": MOE,
        "num_hidden_layers": LAYERS,
        "num_attention_heads": HEADS,
        "num_key_value_heads": HEADS,
        "n_shared_experts": 1,
        "n_routed_experts": EXPERTS,
        "num_experts_per_tok": TOPK,
        "kv_lora_rank": 64,
        "q_lora_rank": 128,
        "qk_rope_head_dim": 0,
        "qk_nope_head_dim": HEAD_DIM,
        "v_head_dim": HEAD_DIM,
        "max_position_embeddings": 128,
        "layer_types": ["linear_attention"] * 3 + ["deepseek_sparse_attention"],
        "mlp_layer_types": ["dense"] * 3 + ["sparse"],
        "indexer_types": ["full"] * LAYERS,
        "index_topk": 4,
        "index_kpool": 2,
        "index_head_dim": HEAD_DIM,
        "index_n_heads": 2,
        "linear_head_dim": HEAD_DIM,
        "linear_num_heads": HEADS,
        "linear_conv_kernel_dim": 4,
        "hc_mult": 2,
        "hc_sinkhorn_iters": 3,
        "hc_eps": 1.0e-6,
        "rms_norm_eps": 1.0e-5,
        "routed_scaling_factor": 2.5,
        "swiglu_limit": 10.0,
        "pad_token_id": None,
        "bos_token_id": 0,
        "eos_token_id": 1,
        "tie_word_embeddings": False,
    }


def runtime_config(version: str, fp8_mlp: bool = False) -> dict[str, object]:
    text = config_kwargs()
    text.update({
        "model_type": "glm5_next_text",
        "linear_attn_config": {
            "num_heads": HEADS,
            "head_dim": HEAD_DIM,
            "short_conv_kernel_size": 4,
            "gate_lower_bound": -5.0,
            "kda_layers": [0, 1, 2],
        },
        "num_nextn_predict_layers": 0,
    })
    result = {
        "architectures": ["Glm5NextForConditionalGeneration"],
        "model_type": "glm5_next",
        "transformers_version": version,
        "torch_dtype": "float32",
        "text_config": text,
    }
    if fp8_mlp:
        result["quantization_config"] = {
            "quant_method": "fp8",
            "fmt": "e4m3",
            "activation_scheme": "dynamic",
            "weight_block_size": [128, 128],
        }
    return result


def production_layout(torch, model, head) -> OrderedDict[str, object]:
    """Recreate the official checkpoint's pre-Transformers disk layout."""
    output: OrderedDict[str, object] = OrderedDict()
    for name, tensor in model.state_dict().items():
        prefix = "model.language_model."
        if name.endswith("mlp.experts.gate_up_proj"):
            base = prefix + name.removesuffix("gate_up_proj")
            for expert in range(EXPERTS):
                output[f"{base}{expert}.gate_proj.weight"] = tensor[expert, :MOE].contiguous()
                output[f"{base}{expert}.up_proj.weight"] = tensor[expert, MOE:].contiguous()
        elif name.endswith("mlp.experts.down_proj"):
            base = prefix + name.removesuffix("down_proj")
            for expert in range(EXPERTS):
                output[f"{base}{expert}.down_proj.weight"] = tensor[expert].contiguous()
        elif name.endswith("self_attn.conv1d.weight"):
            base = prefix + name.removesuffix("conv1d.weight")
            q_conv, k_conv, v_conv = tensor.chunk(3, dim=0)
            output[f"{base}q_conv1d.weight"] = q_conv.contiguous()
            output[f"{base}k_conv1d.weight"] = k_conv.contiguous()
            output[f"{base}v_conv1d.weight"] = v_conv.contiguous()
        else:
            disk_name = name
            disk_name = disk_name.replace("attn_hc.fn", "hc_attn_fn")
            disk_name = disk_name.replace("attn_hc.base", "hc_attn_base")
            disk_name = disk_name.replace("attn_hc.scale", "hc_attn_scale")
            disk_name = disk_name.replace("ffn_hc.fn", "hc_ffn_fn")
            disk_name = disk_name.replace("ffn_hc.base", "hc_ffn_base")
            disk_name = disk_name.replace("ffn_hc.scale", "hc_ffn_scale")
            disk_name = disk_name.replace("self_attn.forget_gate.", "self_attn.")
            output[prefix + disk_name] = tensor.detach().contiguous()
    output["lm_head.weight"] = head.weight.detach().contiguous()
    return output


def initialize_contract_weights(torch, model, head) -> None:
    """Remove random-init degeneracies that make top-k ties implementation-defined."""
    with torch.no_grad():
        embeddings = model.embed_tokens.weight.detach().clone()
        for token in range(VOCAB):
            head.weight[(token + 1) % VOCAB].copy_(embeddings[token])
        indexer = model.layers[3].self_attn.indexer
        indexer.index_kpool_compress_gate.copy_(
            torch.linspace(-0.01, 0.01, indexer.index_kpool_compress_gate.numel()).view_as(
                indexer.index_kpool_compress_gate
            )
        )
        indexer.index_kpool_compress_ape.copy_(
            torch.linspace(-0.1, 0.1, indexer.index_kpool_compress_ape.numel()).view_as(
                indexer.index_kpool_compress_ape
            )
        )
        indexer.weights_proj.weight.copy_(
            torch.linspace(-0.02, 0.03, indexer.weights_proj.weight.numel()).view_as(
                indexer.weights_proj.weight
            )
        )
        indexer.wq_b.weight.zero_()
        indexer.wk.weight.zero_()
        for dim in range(indexer.head_dim):
            indexer.wk.weight[dim, dim] = 1.0
            for index_head in range(indexer.n_heads):
                indexer.wq_b.weight[index_head * indexer.head_dim + dim, dim] = 1.0


def quantize_block_weight(torch, weight):
    import torch.nn.functional as F

    rows, columns = weight.shape
    padded_rows = (rows + 127) // 128 * 128
    padded_columns = (columns + 127) // 128 * 128
    padded = F.pad(weight.float(), (0, padded_columns - columns, 0, padded_rows - rows))
    blocks = padded.reshape(padded_rows // 128, 128, padded_columns // 128, 128)
    maximum = blocks.abs().amax(dim=(1, 3))
    scales = torch.where(maximum > 0, maximum / 448.0, torch.ones_like(maximum))
    quantized = (blocks / scales[:, None, :, None]).clamp(-448, 448).to(torch.float8_e4m3fn)
    quantized_padded = quantized.reshape(padded_rows, padded_columns)
    dequantized_padded = (quantized.float() * scales[:, None, :, None]).reshape(padded_rows, padded_columns)
    quantized = quantized_padded[:rows, :columns].contiguous()
    dequantized = dequantized_padded[:rows, :columns].contiguous()
    return quantized, scales, dequantized


def dynamic_fp8_qdq(torch, value):
    shape = value.shape
    blocks = value.float().reshape(-1, shape[-1] // 128, 128)
    maximum = blocks.abs().amax(dim=-1)
    scales = torch.where(maximum > 0, maximum / 448.0, torch.ones_like(maximum))
    result = (blocks / scales[..., None]).clamp(-448, 448).to(torch.float8_e4m3fn).float()
    return (result * scales[..., None]).reshape(shape)


def apply_fp8_mlp_oracle(torch, model) -> None:
    import torch.nn.functional as F

    def quantized_linear(module):
        _, _, dequantized = quantize_block_weight(torch, module.weight.detach())
        module.weight.copy_(dequantized)

        def forward(self, value):
            return F.linear(dynamic_fp8_qdq(torch, value), self.weight)

        module.forward = types.MethodType(forward, module)

    with torch.no_grad():
        for layer in model.layers:
            if layer.block_type == "deepseek_sparse_attention":
                quantized_linear(layer.self_attn.q_a_proj)
                quantized_linear(layer.self_attn.q_b_proj)
                quantized_linear(layer.self_attn.kv_a_proj_with_mqa)
                quantized_linear(layer.self_attn.o_proj)
            mlp = layer.mlp
            if hasattr(mlp, "experts"):
                quantized_linear(mlp.shared_experts.gate_proj)
                quantized_linear(mlp.shared_experts.up_proj)
                quantized_linear(mlp.shared_experts.down_proj)
                experts = mlp.experts
                gate, up = experts.gate_up_proj.chunk(2, dim=1)
                for expert in range(EXPERTS):
                    _, _, gate[expert] = quantize_block_weight(torch, gate[expert])
                    _, _, up[expert] = quantize_block_weight(torch, up[expert])
                    _, _, experts.down_proj[expert] = quantize_block_weight(torch, experts.down_proj[expert])

                def expert_forward(self, hidden_states, top_k_index, top_k_weights):
                    final = torch.zeros_like(hidden_states)
                    mask = F.one_hot(top_k_index, num_classes=self.num_experts).permute(2, 1, 0)
                    for expert_index in torch.greater(mask.sum(dim=(-1, -2)), 0).nonzero().flatten():
                        top_k_pos, token_index = torch.where(mask[expert_index])
                        current = dynamic_fp8_qdq(torch, hidden_states[token_index])
                        current = self._apply_gate(F.linear(current, self.gate_up_proj[expert_index]))
                        current = F.linear(
                            dynamic_fp8_qdq(torch, current), self.down_proj[expert_index]
                        ) * top_k_weights[token_index, top_k_pos, None]
                        final.index_add_(0, token_index, current.to(final.dtype))
                    return final

                experts.forward = types.MethodType(expert_forward, experts)
            else:
                quantized_linear(mlp.gate_proj)
                quantized_linear(mlp.up_proj)
                quantized_linear(mlp.down_proj)


def quantize_disk_weights(torch, model, weights: OrderedDict[str, object]) -> None:
    replacements = []
    for name, tensor in weights.items():
        mlp = name.endswith(("gate_proj.weight", "up_proj.weight", "down_proj.weight"))
        dsa_suffix = name.endswith(("self_attn.q_a_proj.weight", "self_attn.q_b_proj.weight",
                                    "self_attn.kv_a_proj_with_mqa.weight", "self_attn.o_proj.weight"))
        layer_index = int(name.split(".layers.", 1)[1].split(".", 1)[0]) if ".layers." in name else -1
        dsa = dsa_suffix and layer_index >= 0 and model.layers[layer_index].block_type == "deepseek_sparse_attention"
        if not (mlp or dsa):
            continue
        quantized, scales, _ = quantize_block_weight(torch, tensor)
        replacements.append((name, quantized, scales))
    for name, quantized, scales in replacements:
        weights[name] = quantized
        weights[name + "_scale_inv"] = scales


def make_tokenizer() -> dict[str, object]:
    added = [
        {"id": token, "content": f"<t{token:03d}>", "single_word": False,
         "lstrip": False, "rstrip": False, "normalized": False, "special": True}
        for token in range(VOCAB)
    ]
    return {
        "version": "1.0", "truncation": None, "padding": None,
        "added_tokens": added, "normalizer": None, "pre_tokenizer": None,
        "post_processor": None, "decoder": None,
        "model": {"type": "BPE", "dropout": None, "unk_token": None,
                  "continuing_subword_prefix": "", "end_of_word_suffix": "",
                  "fuse_unk": False, "byte_fallback": False,
                  "ignore_merges": True, "vocab": {"x": VOCAB - 1}, "merges": []},
    }


def oracle(torch, transformers, model, head) -> dict[str, object]:
    prompt = [5, 7, 9, 11, 13, 17, 19, 23]
    ids = torch.tensor([prompt], dtype=torch.long)
    with torch.no_grad():
        full_output = model(ids, use_cache=False, output_hidden_states=True)
        full_hidden = full_output.last_hidden_state
        full_logits = head(full_hidden)
        prefix = model(ids[:, :-1], use_cache=True)
        incremental = model(
            ids[:, -1:], use_cache=True, past_key_values=prefix.past_key_values
        )
        incremental_logits = head(incremental.last_hidden_state)[:, -1]
    delta = float((full_logits[:, -1] - incremental_logits).abs().max())
    if delta > 1e-5:
        raise RuntimeError(f"prefill/incremental cache mismatch: {delta}")
    top_values, top_indices = full_logits[0, -1].topk(8)
    generated = []
    sequence = list(prompt)
    with torch.no_grad():
        for _ in range(4):
            greedy_ids = torch.tensor([sequence], dtype=torch.long)
            next_logits = head(model(greedy_ids, use_cache=False).last_hidden_state)[0, -1]
            token = int(next_logits.argmax())
            generated.append(token)
            sequence.append(token)
    cache_shapes = []
    for layer in prefix.past_key_values.layers:
        if hasattr(layer, "recurrent_states"):
            cache_shapes.append({
                "kind": "kda",
                "conv": list(layer.conv_states[0].shape),
                "recurrent": list(layer.recurrent_states[0].shape),
            })
        else:
            cache_shapes.append({
                "kind": "dsa",
                "keys": list(layer.keys.shape),
                "values": list(layer.values.shape),
                "indexer": list(layer.indexer_keys.shape),
            })
    return {
        "schema_version": 1,
        "source": "transformers",
        "transformers_version": transformers.__version__,
        "torch_version": torch.__version__,
        "seed": SEED,
        "prompt_ids": prompt,
        "teacher_forcing_ids": full_logits[0].argmax(-1).tolist(),
        "last_logits_top_ids": top_indices.tolist(),
        "last_logits_top_values": top_values.tolist(),
        "last_logits": full_logits[0, -1].tolist(),
        "greedy_new_ids": generated,
        "hidden_state_stats": [
            {"sum": float(state.float().sum()), "square_sum": float(state.float().square().sum())}
            for state in full_output.hidden_states
        ],
        "cache_max_abs_delta": delta,
        "cache_shapes": cache_shapes,
    }


def write_c_array(stream, name: str, tensor, dimensions: str) -> None:
    values = tensor.detach().float().reshape(-1).tolist()
    stream.write(f"static const float {name}{dimensions} = {{\n")
    for start in range(0, len(values), 8):
        literals = []
        for value in values[start:start + 8]:
            literal = f"{value:.9g}"
            if "." not in literal and "e" not in literal:
                literal += ".0"
            literals.append(literal + "f")
        row = ", ".join(literals)
        stream.write(f"  {row},\n")
    stream.write("};\n\n")


def write_u8_array(torch, stream, name: str, tensor, dimensions: str) -> None:
    values = tensor.detach().view(torch.uint8).reshape(-1).tolist()
    stream.write(f"static const unsigned char {name}{dimensions} = {{\n")
    for start in range(0, len(values), 16):
        stream.write("  " + ", ".join(str(value) for value in values[start:start + 16]) + ",\n")
    stream.write("};\n\n")


def write_fp8_case(torch, model, output: Path) -> None:
    weight = model.layers[0].mlp.gate_proj.weight.detach().float()
    rows, columns = weight.shape
    blocks = weight.reshape(rows // 128, 128, columns // 128, 128)
    maximum = blocks.abs().amax(dim=(1, 3))
    scales = torch.where(maximum > 0, maximum / 448.0, torch.ones_like(maximum))
    quantized = (blocks / scales[:, None, :, None]).clamp(-448, 448).to(torch.float8_e4m3fn)
    quantized = quantized.reshape_as(weight)
    activation = torch.linspace(-1.25, 1.75, columns)
    activation_blocks = activation.reshape(-1, 128)
    activation_maximum = activation_blocks.abs().amax(dim=1)
    activation_scales = torch.where(
        activation_maximum > 0, activation_maximum / 448.0, torch.ones_like(activation_maximum)
    )
    activation_qdq = (
        (activation_blocks / activation_scales[:, None]).clamp(-448, 448).to(torch.float8_e4m3fn).float()
        * activation_scales[:, None]
    ).reshape(-1)
    dequantized = quantized.float().reshape(rows // 128, 128, columns // 128, 128)
    dequantized = (dequantized * scales[:, None, :, None]).reshape_as(weight)
    expected = torch.mv(dequantized, activation_qdq)
    with output.open("w", encoding="utf-8") as stream:
        stream.write("#ifndef COLIBRI_GLM53_FP8_CASE_H\n#define COLIBRI_GLM53_FP8_CASE_H\n\n")
        stream.write(f"#define GLM53_FP8_ROWS {rows}\n#define GLM53_FP8_COLUMNS {columns}\n\n")
        write_u8_array(torch, stream, "glm53_fp8_weight", quantized,
                       "[GLM53_FP8_ROWS * GLM53_FP8_COLUMNS]")
        write_c_array(stream, "glm53_fp8_scales", scales,
                      "[(GLM53_FP8_ROWS / 128) * (GLM53_FP8_COLUMNS / 128)]")
        write_c_array(stream, "glm53_fp8_input", activation, "[GLM53_FP8_COLUMNS]")
        write_c_array(stream, "glm53_fp8_expected", expected, "[GLM53_FP8_ROWS]")
        stream.write("#endif\n")


def write_kda_case(torch, model, output: Path) -> None:
    """Emit a C fixture from the official recurrent KDA fallback."""
    from transformers.models.glm5_next.modeling_glm5_next import (
        causal_conv1d_fn, recurrent_kimi_delta_attention,
    )
    attention = model.layers[0].self_attn
    hidden = model.embed_tokens(torch.tensor([[5, 7]], dtype=torch.long))
    raw = torch.cat([
        attention.q_proj(hidden), attention.k_proj(hidden), attention.v_proj(hidden)
    ], dim=-1)
    convolved = causal_conv1d_fn(
        raw.transpose(1, 2), attention.conv1d.weight.squeeze(1),
        bias=None, activation=attention.activation,
    ).transpose(1, 2)
    query, key, value = convolved.split(HEADS * HEAD_DIM, dim=-1)
    decay = attention.forget_gate(hidden).view(1, 2, HEADS, HEAD_DIM)
    beta = torch.sigmoid(attention.b_proj(hidden))
    expected, _ = recurrent_kimi_delta_attention(
        query.view(1, 2, HEADS, HEAD_DIM),
        key.view(1, 2, HEADS, HEAD_DIM),
        value.view(1, 2, HEADS, HEAD_DIM),
        g=decay, beta=beta, initial_state=None, output_final_state=True,
        use_qk_l2norm_in_kernel=True,
    )
    with output.open("w", encoding="utf-8") as stream:
        stream.write("#ifndef COLIBRI_GLM53_KDA_CASE_H\n#define COLIBRI_GLM53_KDA_CASE_H\n\n")
        stream.write(f"#define GLM53_KDA_STEPS 2\n#define GLM53_KDA_HEADS {HEADS}\n")
        stream.write(f"#define GLM53_KDA_DIM {HEAD_DIM}\n#define GLM53_KDA_KERNEL 4\n\n")
        write_c_array(stream, "glm53_kda_qkv", raw[0],
                      "[2 * 3 * GLM53_KDA_HEADS * GLM53_KDA_DIM]")
        write_c_array(stream, "glm53_kda_conv", attention.conv1d.weight.squeeze(1),
                      "[3 * GLM53_KDA_HEADS * GLM53_KDA_DIM * GLM53_KDA_KERNEL]")
        write_c_array(stream, "glm53_kda_decay", decay[0],
                      "[2 * GLM53_KDA_HEADS * GLM53_KDA_DIM]")
        write_c_array(stream, "glm53_kda_beta", beta[0], "[2 * GLM53_KDA_HEADS]")
        write_c_array(stream, "glm53_kda_output", expected[0],
                      "[2 * GLM53_KDA_HEADS * GLM53_KDA_DIM]")
        stream.write("#endif\n")


def write_indexer_case(torch, model, output: Path) -> None:
    attention = model.layers[3].self_attn
    indexer = attention.indexer
    ids = torch.arange(8).view(1, -1)
    values = torch.arange(8 * HIDDEN, dtype=torch.float32).view(1, 8, HIDDEN)
    hidden = torch.sin(values * 0.013) + 0.2 * torch.cos(values * 0.031)
    base = torch.linspace(-1.0, 1.0, indexer.head_dim)
    perturb = torch.sin(torch.arange(indexer.head_dim, dtype=torch.float32) * 0.37)
    for token in range(ids.shape[1]):
        hidden[0, token, :indexer.head_dim] = base + (token + 1) * 0.01 * perturb
    q_resid = torch.zeros_like(hidden)
    q_resid[..., :indexer.head_dim] = base
    with torch.no_grad():
        indexer.wk.weight.zero_()
        indexer.wq_b.weight.zero_()
        for dim in range(indexer.head_dim):
            indexer.wk.weight[dim, dim] = 1.0
            for head in range(indexer.n_heads):
                indexer.wq_b.weight[head * indexer.head_dim + dim, dim] = 1.0
        indexer.weights_proj.weight.copy_(
            torch.linspace(0.001, 0.01, indexer.weights_proj.weight.numel()).view_as(
                indexer.weights_proj.weight
            )
        )
        indexer.index_kpool_compress_gate.copy_(
            torch.linspace(-0.01, 0.01, indexer.index_kpool_compress_gate.numel()).view_as(
                indexer.index_kpool_compress_gate
            )
        )
        indexer.index_kpool_compress_ape.copy_(
            torch.linspace(-0.1, 0.1, indexer.index_kpool_compress_ape.numel()).view_as(
                indexer.index_kpool_compress_ape
            )
        )
    queries = indexer.wq_b(q_resid).view(1, ids.shape[1], indexer.n_heads, indexer.head_dim)
    keys = indexer.k_norm(indexer.wk(hidden))
    gates = torch.nn.functional.linear(hidden, indexer.index_kpool_compress_gate)
    weights = indexer.weights_proj(hidden) * (indexer.n_heads ** -0.5)
    valid = torch.ones_like(ids, dtype=torch.bool)
    with torch.no_grad():
        expected = indexer(hidden, q_resid, valid, None)
    with output.open("w", encoding="utf-8") as stream:
        stream.write("#ifndef COLIBRI_GLM53_INDEXER_CASE_H\n#define COLIBRI_GLM53_INDEXER_CASE_H\n\n")
        stream.write(f"#define GLM53_INDEX_SEQUENCE {ids.shape[1]}\n")
        stream.write(f"#define GLM53_INDEX_HEADS {indexer.n_heads}\n")
        stream.write(f"#define GLM53_INDEX_DIM {indexer.head_dim}\n")
        stream.write(f"#define GLM53_INDEX_POOL {indexer.index_kpool}\n")
        stream.write(f"#define GLM53_INDEX_TOPK {indexer.index_topk}\n")
        stream.write("#define GLM53_INDEX_WIDTH (GLM53_INDEX_TOPK + GLM53_INDEX_POOL - 1)\n\n")
        write_c_array(stream, "glm53_index_queries", queries,
                      "[GLM53_INDEX_SEQUENCE * GLM53_INDEX_HEADS * GLM53_INDEX_DIM]")
        write_c_array(stream, "glm53_index_keys", keys,
                      "[GLM53_INDEX_SEQUENCE * GLM53_INDEX_DIM]")
        write_c_array(stream, "glm53_index_gates", gates,
                      "[GLM53_INDEX_SEQUENCE * GLM53_INDEX_DIM]")
        write_c_array(stream, "glm53_index_weights", weights,
                      "[GLM53_INDEX_SEQUENCE * GLM53_INDEX_HEADS]")
        write_c_array(stream, "glm53_index_ape", indexer.index_kpool_compress_ape,
                      "[GLM53_INDEX_POOL * GLM53_INDEX_DIM]")
        valid_values = ", ".join("1" if value else "0" for value in valid.reshape(-1).tolist())
        stream.write(f"static const unsigned char glm53_index_valid[GLM53_INDEX_SEQUENCE] = {{{valid_values}}};\n")
        expected_values = ", ".join(str(value) for value in expected.reshape(-1).tolist())
        stream.write("static const int glm53_index_expected[GLM53_INDEX_SEQUENCE * GLM53_INDEX_WIDTH] = {\n")
        stream.write(f"  {expected_values}\n}};\n\n#endif\n")


def write_sparse_attention_case(torch, model, output: Path) -> None:
    attention = model.layers[3].self_attn
    ids = torch.tensor([[5, 7, 9, 11, 13, 17, 19, 23]], dtype=torch.long)
    hidden = model.embed_tokens(ids)
    q_resid = attention.q_a_layernorm(attention.q_a_proj(hidden))
    queries = attention.q_b_proj(q_resid).view(1, 8, HEADS, HEAD_DIM)
    compressed = attention.kv_a_proj_with_mqa(hidden)
    latent = attention.kv_a_layernorm(compressed).view(1, 1, 8, 64)
    keys, values = attention.expand_kv(latent, compressed.new_empty(1, 1, 8, 0))
    keys = keys.transpose(1, 2).contiguous()
    values = values.transpose(1, 2).contiguous()
    valid = torch.ones_like(ids, dtype=torch.bool)
    indices = attention.indexer(hidden, q_resid, valid, None)
    width = indices.shape[-1]
    expected = torch.zeros(1, 8, HEADS, HEAD_DIM)
    with torch.no_grad():
        for position in range(8):
            selected = indices[0, position]
            selected = selected[selected.ge(0)]
            for head in range(HEADS):
                scores = queries[0, position, head] @ keys[0, selected, head].T
                probabilities = torch.softmax(scores.float() / (HEAD_DIM ** 0.5), dim=-1)
                expected[0, position, head] = probabilities @ values[0, selected, head].float()
    with output.open("w", encoding="utf-8") as stream:
        stream.write("#ifndef COLIBRI_GLM53_SPARSE_ATTENTION_CASE_H\n#define COLIBRI_GLM53_SPARSE_ATTENTION_CASE_H\n\n")
        stream.write("#define GLM53_SA_SEQUENCE 8\n")
        stream.write(f"#define GLM53_SA_HEADS {HEADS}\n#define GLM53_SA_KEY_DIM {HEAD_DIM}\n")
        stream.write(f"#define GLM53_SA_VALUE_DIM {HEAD_DIM}\n#define GLM53_SA_WIDTH {width}\n\n")
        write_c_array(stream, "glm53_sa_queries", queries,
                      "[GLM53_SA_SEQUENCE * GLM53_SA_HEADS * GLM53_SA_KEY_DIM]")
        write_c_array(stream, "glm53_sa_keys", keys,
                      "[GLM53_SA_SEQUENCE * GLM53_SA_HEADS * GLM53_SA_KEY_DIM]")
        write_c_array(stream, "glm53_sa_values", values,
                      "[GLM53_SA_SEQUENCE * GLM53_SA_HEADS * GLM53_SA_VALUE_DIM]")
        index_values = ", ".join(str(value) for value in indices.reshape(-1).tolist())
        stream.write("static const int glm53_sa_indices[GLM53_SA_SEQUENCE * GLM53_SA_WIDTH] = {\n")
        stream.write(f"  {index_values}\n}};\n\n")
        write_c_array(stream, "glm53_sa_expected", expected,
                      "[GLM53_SA_SEQUENCE * GLM53_SA_HEADS * GLM53_SA_VALUE_DIM]")
        stream.write("#endif\n")


def write_mhc_case(torch, model, output: Path) -> None:
    mhc = model.layers[0].attn_hc
    streams = model.embed_tokens(torch.tensor([[5]])).unsqueeze(2).expand(-1, -1, 2, -1).contiguous()
    with torch.no_grad():
        post, comb, collapsed = mhc(streams)
        branch = torch.sin(torch.arange(HIDDEN, dtype=torch.float32) * 0.07).view(1, 1, HIDDEN)
        expanded = post.to(branch.dtype).unsqueeze(-1) * branch.unsqueeze(-2) + torch.matmul(
            comb.to(streams.dtype), streams.unsqueeze(-3)
        ).squeeze(-2)
    with output.open("w", encoding="utf-8") as stream:
        stream.write("#ifndef COLIBRI_GLM53_MHC_CASE_H\n#define COLIBRI_GLM53_MHC_CASE_H\n\n")
        stream.write(f"#define GLM53_MHC_COPIES 2\n#define GLM53_MHC_DIM {HIDDEN}\n")
        stream.write("#define GLM53_MHC_ITERATIONS 3\n\n")
        write_c_array(stream, "glm53_mhc_streams", streams,
                      "[GLM53_MHC_COPIES * GLM53_MHC_DIM]")
        write_c_array(stream, "glm53_mhc_function", mhc.fn,
                      "[((2 + GLM53_MHC_COPIES) * GLM53_MHC_COPIES) * GLM53_MHC_COPIES * GLM53_MHC_DIM]")
        write_c_array(stream, "glm53_mhc_scale", mhc.scale, "[3]")
        write_c_array(stream, "glm53_mhc_base", mhc.base,
                      "[(2 + GLM53_MHC_COPIES) * GLM53_MHC_COPIES]")
        write_c_array(stream, "glm53_mhc_branch", branch, "[GLM53_MHC_DIM]")
        write_c_array(stream, "glm53_mhc_collapsed", collapsed, "[GLM53_MHC_DIM]")
        write_c_array(stream, "glm53_mhc_post", post, "[GLM53_MHC_COPIES]")
        write_c_array(stream, "glm53_mhc_comb", comb,
                      "[GLM53_MHC_COPIES * GLM53_MHC_COPIES]")
        write_c_array(stream, "glm53_mhc_output", expanded,
                      "[GLM53_MHC_COPIES * GLM53_MHC_DIM]")
        stream.write("#endif\n")


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    default = Path(__file__).resolve().parents[1] / "glm53_tiny"
    parser.add_argument("--output", type=Path, default=default)
    parser.add_argument("--force", action="store_true")
    parser.add_argument("--fp8-mlp", action="store_true")
    args = parser.parse_args()
    torch, transformers, save_file, Config, Model = require_dependencies()
    torch.manual_seed(SEED)
    torch.set_num_threads(1)
    model = Model(Config(**config_kwargs())).eval()
    head = torch.nn.Linear(HIDDEN, VOCAB, bias=False).eval()
    initialize_contract_weights(torch, model, head)
    if args.fp8_mlp:
        apply_fp8_mlp_oracle(torch, model)
    weights = production_layout(torch, model, head)
    if args.fp8_mlp:
        quantize_disk_weights(torch, model, weights)
    reference = oracle(torch, transformers, model, head)
    output = args.output.resolve()
    if output.exists():
        if not args.force:
            raise SystemExit(f"output exists (use --force): {output}")
        shutil.rmtree(output)
    output.mkdir(parents=True)
    (output / "config.json").write_text(
        json.dumps(runtime_config(transformers.__version__, args.fp8_mlp), indent=2) + "\n",
        encoding="utf-8",
    )
    (output / "tokenizer.json").write_text(
        json.dumps(make_tokenizer(), separators=(",", ":")) + "\n", encoding="utf-8"
    )
    save_file(weights, output / "model.safetensors", metadata={
        "format": "pt", "generator": "c/tools/make_glm53_tiny.py"
    })
    (output / "ref.json").write_text(
        json.dumps(reference, indent=2) + "\n", encoding="utf-8"
    )
    write_kda_case(torch, model, output / "glm53_kda_case.h")
    write_sparse_attention_case(torch, model, output / "glm53_sparse_attention_case.h")
    write_mhc_case(torch, model, output / "glm53_mhc_case.h")
    write_indexer_case(torch, model, output / "glm53_indexer_case.h")
    write_fp8_case(torch, model, output / "glm53_fp8_case.h")
    size = sum(path.stat().st_size for path in output.iterdir())
    print(f"wrote {output} ({len(weights)} tensors, {size} bytes)")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())