Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
15.8 KB
Raw Blame
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""repack_r3_8b.py — 8b (BailingMoeV3/Ling-3) → bailingmoe3 canonical-naming GGUF.

The contract is the full reference table (conversion/bailingmoe3.py +
llama-arch.cpp + bailingmoe3.cpp):
  name strings (llama-arch.cpp table as-is): attn_q_a/attn_q_a_norm/attn_q_b/
    attn_kv_a_mqa/attn_kv_a_norm/attn_k_b/attn_v_b/attn_gate/
    ssm_conv1d_{q,k,v}/ssm_a(no .weight)/ssm_dt(.bias)/ssm_beta/ssm_f_a/ssm_g_a/
    ssm_norm/exp_probs_b(.bias)
  transforms (checked line-by-line against bailingmoe3.py):
    conv [C,1,4] → reshape(1,C,1,4) → ne{4,1,C,1}, bytes unchanged, F32
    ssm_a = torch.exp(A_log).reshape(-1,1)   ← NOTE +exp here (35b uses −exp; do not mix them up)
    dt_bias [2048] elementwise → ssm_dt.bias stored directly as F32
    kv_b_proj → split + transpose (view(16,256,512) → [128 noope | 128 v] → k transpose(1,2))
      k_b/v_b are stored as **F32, not Q4_1**: the transpose scatters the quantization
      grouping axis (dim0=128 spans 128 original rows), so lossless repack is
      physically impossible; 6 layers x 8MB = 48MB cost, negligible.
    f_proj→ssm_f_a; g_proj→(KDA)ssm_g_a /(MLA)attn_gate; dense(MLA)→attn_output
    gate BF16→F32 stored directly (8b gate is NOT quantized, unlike 35b int8);
    expert_bias→exp_probs_b.bias F32
    all norms stored directly (bailing_hybrid sanitize has no +1 and no moveaxis;
    the source of truth is the final value multiplied directly)
  metadata: head_count_kv is a 24-element array ((i+1)%4==0 →1 else 0, bailingmoe3.py:55-56);
    key_length=576(=kv_lora+qk_rope) value_length=128 key_length_mla=192 value_length_mla=128;
    expert_gating_func=2(SIGMOID, llama-hparams.h:19) expert_weights_scale=2.5
    (config routed_scaling_factor) expert_weights_norm=true; kda.head_dim=128
    kda.safe_gate=true kda.gate_lower_bound=-5; leading_dense_block_count=1;
    rope.dimension_count=64; **tokenizer.ggml.pre="bailingmoe2"** (verified by hashing
    tokenizer.json against the fork, NOT qwen2); architecture string
    general.architecture=bailingmoe3.
pregate x66 stripped (sidecar feeds the prefetch router). Gates: tensor-count
identity / sentinel sha / kv_b transform reversibility spot-check / exp dual-path.
"""
import argparse, json, math, os, re, struct, sys, time
import numpy as np
if hasattr(sys.stdout, "reconfigure"):                      # issue #128: Windows ANSI-codepage stdout hardening
    sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from repack_mlx_to_gguf import (StIndex, GgufWriter, KV, T_F32, T_F16, T_Q4_1, QK41, DT_Q41,
                                mlx_q_bytes, pack_q41_rows, deq_mlx_side, deq_gguf_side, sha16)

FULL = lambda i: (i + 1) % 4 == 0            # MLA layer (bailingmoe3.py:39-42 same rule)
NL, E, KDA_DIM, HEADS = 24, 128, 2048, 16
R3VER = "r3-8b-1.0"


def q41_src(idx, name):
    W = idx.load(name); base = name[:-7]
    return W, idx.load(base + ".scales"), idx.load(base + ".biases")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--dir", default="models/edge0-8b")
    ap.add_argument("--out", default="models/edge0-8b-gguf")
    a = ap.parse_args()
    idx = StIndex([os.path.join(a.dir, f) for f in sorted(os.listdir(a.dir))
                   if re.fullmatch(r"model(-\d+-of-\d+)?\.safetensors", f)])
    cfg = json.load(open(os.path.join(a.dir, "config.json"), encoding="utf8"))
    tkj = json.load(open(os.path.join(a.dir, "tokenizer.json"), encoding="utf8"))
    A = "bailingmoe3."
    P = "model.layers.{}."
    ATT = P + "attention."
    MLP = P + "mlp."

    # ---- tensor table: (dst_name, kind, src); kind ∈ q41/f32raw/conv/sa/kvb ----
    table = [("token_embd.weight", "q41", "model.word_embeddings.weight"),
             ("output.weight", "q41", "lm_head.weight"),
             ("output_norm.weight", "f32raw", "model.norm.weight")]
    for i in range(NL):
        at_, mp = ATT.format(i), MLP.format(i)
        table += [(f"blk.{i}.attn_norm.weight", "f32raw", P.format(i) + "input_layernorm.weight"),
                  (f"blk.{i}.ffn_norm.weight", "f32raw", P.format(i) + "post_attention_layernorm.weight")]
        if FULL(i):
            table += [(f"blk.{i}.attn_q_a.weight", "q41", at_ + "q_a_proj.weight"),
                      (f"blk.{i}.attn_q_a_norm.weight", "f32raw", at_ + "q_a_layernorm.weight"),
                      (f"blk.{i}.attn_q_b.weight", "q41", at_ + "q_b_proj.weight"),
                      (f"blk.{i}.attn_kv_a_mqa.weight", "q41", at_ + "kv_a_proj_with_mqa.weight"),
                      (f"blk.{i}.attn_kv_a_norm.weight", "f32raw", at_ + "kv_a_layernorm.weight"),
                      (f"blk.{i}.attn_k_b.weight", "kvb_k", at_ + "kv_b_proj.weight"),
                      (f"blk.{i}.attn_v_b.weight", "kvb_v", at_ + "kv_b_proj.weight"),
                      (f"blk.{i}.attn_gate.weight", "q41", at_ + "g_proj.weight"),
                      (f"blk.{i}.attn_output.weight", "q41", at_ + "dense.weight")]
        else:
            table += [(f"blk.{i}.attn_q.weight", "q41", at_ + "q_proj.weight"),
                      (f"blk.{i}.attn_k.weight", "q41", at_ + "k_proj.weight"),
                      (f"blk.{i}.attn_v.weight", "q41", at_ + "v_proj.weight"),
                      (f"blk.{i}.ssm_conv1d_q.weight", "conv", at_ + "q_conv1d.weight"),
                      (f"blk.{i}.ssm_conv1d_k.weight", "conv", at_ + "k_conv1d.weight"),
                      (f"blk.{i}.ssm_conv1d_v.weight", "conv", at_ + "v_conv1d.weight"),
                      (f"blk.{i}.ssm_a", "sa", at_ + "A_log"),
                      (f"blk.{i}.ssm_dt.bias", "f32raw", at_ + "dt_bias"),
                      (f"blk.{i}.ssm_beta.weight", "q41", at_ + "b_proj.weight"),
                      (f"blk.{i}.ssm_f_a.weight", "q41", at_ + "f_proj.weight"),
                      (f"blk.{i}.ssm_g_a.weight", "q41", at_ + "g_proj.weight"),
                      (f"blk.{i}.ssm_norm.weight", "f32raw", at_ + "o_norm.weight"),
                      (f"blk.{i}.attn_output.weight", "q41", at_ + "o_proj.weight")]
        if i == 0:
            table += [(f"blk.0.ffn_gate.weight", "q41", mp + "gate_proj.weight"),
                      (f"blk.0.ffn_up.weight", "q41", mp + "up_proj.weight"),
                      (f"blk.0.ffn_down.weight", "q41", mp + "down_proj.weight")]
        else:
            table += [(f"blk.{i}.ffn_gate_inp.weight", "f32raw", mp + "gate.weight"),
                      (f"blk.{i}.exp_probs_b.bias", "f32raw", mp + "gate.expert_bias"),
                      (f"blk.{i}.ffn_gate_exps.weight", "q41", mp + "experts.gate_proj.weight"),
                      (f"blk.{i}.ffn_up_exps.weight", "q41", mp + "experts.up_proj.weight"),
                      (f"blk.{i}.ffn_down_exps.weight", "q41", mp + "experts.down_proj.weight"),
                      (f"blk.{i}.ffn_gate_shexp.weight", "q41", mp + "shared_experts.gate_proj.weight"),
                      (f"blk.{i}.ffn_up_shexp.weight", "q41", mp + "shared_experts.up_proj.weight"),
                      (f"blk.{i}.ffn_down_shexp.weight", "q41", mp + "shared_experts.down_proj.weight")]
    miss = [t for t in table if t[2] not in idx.t]
    assert not miss, f"source tensor missing: {miss[:5]}"
    names = [t[0] for t in table]
    assert len(names) == len(set(names))

    # ---- metadata ----
    os.makedirs(a.out, exist_ok=True)
    stem = "edge0-8b"
    gp = os.path.join(a.out, f"{stem}.gguf")
    w = GgufWriter(gp)
    w.kv_s("general.architecture", "bailingmoe3")
    w.kv_s("general.type", "model")
    w.kv_s("general.name", "edge0-8b")
    w.kv("general.alignment", KV["u32"], struct.pack("<I", 32))
    u = lambda v: struct.pack("<I", v)
    f32 = lambda v: struct.pack("<f", v)
    b1 = lambda v: b"\x01" if v else b"\x00"
    for k, ty, v in [
        (A + "block_count", KV["u32"], u(NL)), (A + "context_length", KV["u32"], u(cfg["max_position_embeddings"])),
        (A + "embedding_length", KV["u32"], u(cfg["hidden_size"])), (A + "feed_forward_length", KV["u32"], u(cfg["intermediate_size"])),
        (A + "attention.head_count", KV["u32"], u(HEADS)),
        (A + "attention.key_length", KV["u32"], u(cfg["kv_lora_rank"] + cfg["qk_rope_head_dim"])),
        (A + "attention.value_length", KV["u32"], u(cfg["head_dim"])),
        (A + "attention.key_length_mla", KV["u32"], u(cfg["qk_head_dim"])),
        (A + "attention.value_length_mla", KV["u32"], u(cfg["v_head_dim"])),
        (A + "attention.q_lora_rank", KV["u32"], u(cfg["q_lora_rank"])),
        (A + "attention.kv_lora_rank", KV["u32"], u(cfg["kv_lora_rank"])),
        (A + "rope.freq_base", KV["f32"], f32(float(cfg["rope_theta"]))),
        (A + "rope.dimension_count", KV["u32"], u(cfg["qk_rope_head_dim"])),
        (A + "attention.layer_norm_rms_epsilon", KV["f32"], f32(cfg["rms_norm_eps"])),
        (A + "vocab_size", KV["u32"], u(cfg["vocab_size"])),
        (A + "ssm.conv_kernel", KV["u32"], u(cfg["short_conv_kernel_size"])),
        (A + "kda.head_dim", KV["u32"], u(cfg["head_dim"])),
        (A + "expert_count", KV["u32"], u(E)), (A + "expert_used_count", KV["u32"], u(cfg["num_experts_per_tok"])),
        (A + "expert_group_count", KV["u32"], u(cfg["n_group"])), (A + "expert_group_used_count", KV["u32"], u(cfg["topk_group"])),
        (A + "expert_gating_func", KV["u32"], u(2)),
        (A + "expert_weights_scale", KV["f32"], f32(cfg["routed_scaling_factor"])),
        (A + "expert_feed_forward_length", KV["u32"], u(cfg["moe_intermediate_size"])),
        (A + "expert_shared_feed_forward_length", KV["u32"], u(cfg.get("shared_expert_intermediate_size", cfg["moe_intermediate_size"]))),
        (A + "expert_shared_count", KV["u32"], u(1)),
        (A + "leading_dense_block_count", KV["u32"], u(cfg["first_k_dense_replace"])),
    ]:
        w.kv(k, ty, v)
    w.kv(A + "kda.safe_gate", KV["bool"], b1(True))
    w.kv(A + "kda.gate_lower_bound", KV["f32"], f32(cfg["kda_lower_bound"]))
    w.kv(A + "expert_weights_norm", KV["bool"], b1(cfg.get("norm_topk_prob", True)))
    w.kv(A + "attention.head_count_kv", KV["arr"], struct.pack("<I", KV["u32"]) + struct.pack("<Q", NL)
         + b"".join(u(1 if FULL(i) else 0) for i in range(NL)))
    w.kv_s("tokenizer.ggml.model", "gpt2")
    w.kv_s("tokenizer.ggml.pre", "bailingmoe2")

    # tokenizer arrays
    vocab = tkj["model"]["vocab"]; toks = [None] * cfg["vocab_size"]
    for s, i2 in vocab.items():
        toks[i2] = s
    for at in tkj["added_tokens"]:
        toks[at["id"]] = at["content"]
    tt = [1] * len(toks)
    for at in tkj["added_tokens"]:
        tt[at["id"]] = 3
    n_pad = sum(1 for x in toks if x is None)
    for j, x in enumerate(toks):
        if x is None:
            toks[j] = f"<0x{j:06X}>"

    def arr(key, et, elems):
        w.kv(key, KV["arr"], struct.pack("<I", et) + struct.pack("<Q", len(elems)) + b"".join(elems))
    arr("tokenizer.ggml.tokens", KV["str"], [w._str(s) for s in toks])
    arr("tokenizer.ggml.merges", KV["str"],
        [w._str(" ".join(m) if isinstance(m, list) else m) for m in tkj["model"]["merges"]])
    arr("tokenizer.ggml.scores", KV["f32"], [f32(0.0) for _ in toks])
    arr("tokenizer.ggml.token_type", KV["i32"], [struct.pack("<i", x) for x in tt])
    w.kv("tokenizer.ggml.bos_token_id", KV["u32"], u(156891))
    w.kv("tokenizer.ggml.eos_token_id", KV["u32"], u(cfg.get("eos_token_id", 156895)))
    w.kv("tokenizer.ggml.padding_token_id", KV["u32"], u(cfg.get("pad_token_id", 156892)))
    w.kv("tokenizer.ggml.add_bos_token", KV["bool"], b1(False))
    w.kv("tokenizer.ggml.add_eos_token", KV["bool"], b1(False))
    w.kv("tokenizer.ggml.byte_fallback", KV["bool"], b1(True))
    ct = os.path.join(a.dir, "chat_template.jinja")
    if os.path.isfile(ct):
        w.kv_s("tokenizer.chat_template", open(ct, encoding="utf8").read())

    # ---- two-pass tensors ----
    def spec(kind, src):
        shp = idx.t[src][4]
        if kind == "kvb_k":
            return T_F32, (cfg["qk_nope_head_dim"], cfg["kv_lora_rank"], HEADS), HEADS * cfg["kv_lora_rank"] * cfg["qk_nope_head_dim"] * 4
        if kind == "kvb_v":
            return T_F32, (cfg["kv_lora_rank"], cfg["v_head_dim"], HEADS), HEADS * cfg["kv_lora_rank"] * cfg["v_head_dim"] * 4
        if kind == "conv":
            return T_F32, (shp[2], 1, shp[0], 1), int(np.prod(shp)) * 4
        if kind == "sa":
            return T_F32, (1, shp[0]), shp[0] * 4          # bailingmoe3.cpp:89 create {1, n_head}
        if kind == "f32raw":
            return T_F32, (tuple(reversed(shp)) if len(shp) > 1 else (shp[0],)), int(np.prod(shp)) * 4
        K = shp[-1] * 8
        rows = int(np.prod(shp[:-1])) if len(shp) > 1 else 1
        return T_Q4_1, (K,) + tuple(reversed(shp[:-1])), rows * (K // QK41) * 20

    for dst, kind, src in table:
        tt_, dims, size = spec(kind, src)
        w.register(dst, tt_, dims, size)
    ds = w.finalize()
    audit = dict(tool=R3VER, gates=[], errors=[])
    t0 = time.time()
    for n, (dst, kind, src) in enumerate(table):
        if kind == "q41":
            W, S, Bs = q41_src(idx, src)
            rows = int(np.prod(W.shape[:-1])) if W.ndim > 1 else 1
            Wf, Sf, Bsf = W.reshape(rows, -1), S.reshape(rows, -1), Bs.reshape(rows, -1)
            blocks, exm = pack_q41_rows(mlx_q_bytes(Wf), Sf, Bsf, src)
            w.write(blocks.tobytes()); w.align_pad()
            a_side = deq_mlx_side(mlx_q_bytes(Wf[:1]), Sf[:1], Bsf[:1])
            same = sha16(a_side.tobytes()) == sha16(deq_gguf_side(blocks[:1]).tobytes())
            st = "OK" if same else ("EXEMPT" if any(r == 0 for r, _ in exm) else "FAIL")
            audit["gates"].append((dst, st))
            if st == "FAIL":
                audit["errors"].append(dst)
        elif kind == "kvb_k" or kind == "kvb_v":
            W, S, Bs = q41_src(idx, src)
            K = W.shape[-1] * 8
            q = mlx_q_bytes(W.reshape(4096, -1))
            v = deq_mlx_side(q, S.reshape(4096, -1), Bs.reshape(4096, -1))   # (4096,512)
            kv = v.reshape(HEADS, 256, K)
            kt = np.ascontiguousarray(kv[:, :128, :].transpose(0, 2, 1), np.float32)   # (16,512,128)
            vv = np.ascontiguousarray(kv[:, 128:, :], np.float32)                       # (16,128,512)
            payload = kt.tobytes() if kind == "kvb_k" else vv.tobytes()
            w.write(payload); w.align_pad()
        elif kind == "conv":
            x = idx.load(src)                          # [C,1,4]
            y = np.ascontiguousarray(x.reshape(1, x.shape[0], 1, x.shape[2]), np.float32)
            w.write(y.tobytes()); w.align_pad()
        elif kind == "sa":
            x = idx.load(src)
            v = np.exp(np.asarray(x, dtype=np.float32))
            v2 = np.array([math.exp(float(t)) for t in x.ravel()], np.float32)
            dv = float(np.abs(v - v2).max(initial=0.0))
            if dv > 1e-5:
                audit["errors"].append(f"{dst} exp dev {dv}")
            y = np.ascontiguousarray(v.reshape(-1, 1), np.float32)
            w.write(y.tobytes()); w.align_pad()         # ne {1,16} via reversed((16,1))
        else:  # f32raw
            x = idx.load(src)
            w.write(np.ascontiguousarray(x, np.float32).tobytes()); w.align_pad()
        if (n + 1) % 100 == 0:
            print(f"  {n+1}/{len(table)} {dst} {time.time()-t0:.0f}s", flush=True)
    w.close(); idx.close()
    ng = len(audit["gates"]); bad = [g for g in audit["gates"] if g[1] != "OK"]
    audit.update(n_tensors=len(table), pad_tokens=n_pad, elapsed_s=round(time.time() - t0, 1))
    json.dump(dict(audit=audit, size=os.path.getsize(gp),
                   verdict="R3-8b-artifact " + ("GREEN" if not bad and not audit["errors"] else "RED")),
              open(os.path.join(a.out, f"{stem}-audit.json"), "w"), ensure_ascii=False, indent=1)
    print(f"GGUF {gp} {os.path.getsize(gp)/1e9:.2f}GB tensors={len(table)} gate-mismatch={len(bad)} errors={audit['errors'][:5]}")
    print("sa dims check:", spec("sa", "model.layers.0.attention.A_log")[1])
    return 0 if not bad and not audit["errors"] else 1


if __name__ == "__main__":
    sys.exit(main() or 0)