Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
6.3 KB
Raw Blame
#!/usr/bin/env python3
"""Tiny Qwen3.8-27B-shaped vision-language fixture and its token-exact reference.

Qwen3_5ForConditionalGeneration at toy widths: the dense text model of the
qwen38-27b-dense geometry (make_qwen36_tiny.py) plus a small ViT of the tower
the whole family shares. transformers generates greedily from a prompt that
holds ONE image, and ref.json records the prompt ids, the generated ids and the
image patches, so `./qwen36 <cap> 8 ref.json` can be held to the same tokens
(#1757). A random model answers anything; token equality is the test.

The image grid is 4 x 8 patches on purpose: 2 x 4 tokens after the merge, so
the text after the image resumes at start + max(2, 4) = start + 4 while 8
positions went by. Every position after the image then depends on getting
Qwen3.5's 3D rope positions right, not just the first one.

  python3 tools/make_qwen36_vl_tiny.py --out qwen36_vl_tiny
"""
import argparse
import json
import sys
from pathlib import Path

import torch

sys.path.insert(0, str(Path(__file__).resolve().parent))
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tests"))
import make_qwen36_tiny  # noqa: E402

SEED = 20260927
IMAGE, VIDEO, START, END = 300, 301, 302, 303     # special ids inside the 320-token toy vocabulary


SPECIALS = {"<|image_pad|>": IMAGE, "<|video_pad|>": VIDEO, "<|vision_start|>": START,
            "<|vision_end|>": END, "<|im_start|>": 304, "<|im_end|>": 305, "<|endoftext|>": 319}


def write_tokenizer(out: Path):
    """One token per ASCII byte (the GPT-2 byte map tok.h builds) plus Qwen's
    special tokens as added tokens at their fixture ids, so a serve prompt written
    as text reaches the engine as exactly the oracle's ids."""
    from prefix_serve_harness import byte_level_vocab
    # ASCII bytes for ids 0..127 (the prompts only use a few of them); every
    # other id decodes to printable filler. A lone byte >= 0x80 is not valid UTF-8,
    # and the engine holds such bytes back from the stream, so ids that decode to
    # one would hide from a test that compares what the client receives.
    vocab = {k: v for k, v in byte_level_vocab(256).items() if v < 128}
    for i in range(128, 320):
        if i not in SPECIALS.values():
            vocab[f"t{i}"] = i
    added = [{"id": i, "content": t, "single_word": False, "lstrip": False, "rstrip": False,
              "normalized": False, "special": True} for t, i in sorted(SPECIALS.items(), key=lambda kv: kv[1])]
    (out / "tokenizer.json").write_text(json.dumps({
        "version": "1.0", "added_tokens": added,
        "model": {"type": "BPE", "vocab": vocab, "merges": []}}), encoding="utf-8")


def build(out: Path, grid_h=4, grid_w=8, max_new=16, seed=SEED):
    from transformers import Qwen3_5Config, Qwen3_5ForConditionalGeneration

    geo = make_qwen36_tiny.GEOMETRIES["qwen38-27b-dense"]
    hidden, layers = geo["hidden"], geo["n_layers"]
    text = dict(
        vocab_size=320, hidden_size=hidden, intermediate_size=geo["inter"],
        num_hidden_layers=layers, num_attention_heads=geo["q_heads"],
        num_key_value_heads=geo["kv_heads"], head_dim=32,
        max_position_embeddings=512, rms_norm_eps=1e-6, tie_word_embeddings=False,
        linear_conv_kernel_dim=4, linear_key_head_dim=8, linear_value_head_dim=8,
        linear_num_key_heads=geo["dn_key_heads"], linear_num_value_heads=geo["dn_value_heads"],
        layer_types=["full_attention" if i % 4 == 3 else "linear_attention" for i in range(layers)],
        hidden_act="silu", attention_bias=False, attention_dropout=0.0, use_cache=True,
        # the real checkpoint's rope: interleaved M-RoPE over a quarter of head_dim.
        # head_dim 32 gives four frequencies, so [2, 1, 1] puts at least one on
        # every axis (the real [11, 11, 10] spreads 32 the same way)
        rope_parameters={"rope_type": "default", "rope_theta": 10000.0,
                         "partial_rotary_factor": 0.25, "mrope_interleaved": True,
                         "mrope_section": [2, 1, 1]},
        partial_rotary_factor=0.25, pad_token_id=0, bos_token_id=1, eos_token_id=319,
    )
    vision = dict(depth=2, hidden_size=32, num_heads=4, intermediate_size=64, patch_size=16,
                  spatial_merge_size=2, temporal_patch_size=2, in_channels=3,
                  out_hidden_size=hidden, num_position_embeddings=64,
                  hidden_act="gelu_pytorch_tanh", deepstack_visual_indexes=[])
    config = Qwen3_5Config(text_config=text, vision_config=vision, image_token_id=IMAGE,
                           video_token_id=VIDEO, vision_start_token_id=START,
                           vision_end_token_id=END, tie_word_embeddings=False)
    torch.manual_seed(seed)
    model = Qwen3_5ForConditionalGeneration(config).eval()
    with torch.no_grad():                       # a tower whose output is not noise-small
        for name, parameter in model.named_parameters():
            if "visual" in name:
                parameter.copy_(torch.randn_like(parameter) * 0.5)
    out.mkdir(parents=True, exist_ok=True)
    model.save_pretrained(str(out))
    write_tokenizer(out)

    tokens = (grid_h // 2) * (grid_w // 2)
    prompt = [1, 2, 3, START] + [IMAGE] * tokens + [END, 4, 5, 6]
    generator = torch.Generator().manual_seed(seed + 1)
    features = 3 * 2 * 16 * 16
    patches = torch.randn(grid_h * grid_w, features, generator=generator)
    input_ids = torch.tensor([prompt])
    with torch.no_grad():
        full = model.generate(
            input_ids, pixel_values=patches, image_grid_thw=torch.tensor([[1, grid_h, grid_w]]),
            mm_token_type_ids=(input_ids == IMAGE).int(), max_new_tokens=max_new,
            do_sample=False, use_cache=True)[0].tolist()
    reference = {"prompt_ids": prompt, "full_ids": full, "mode": "full", "model": "qwen36_vl_tiny",
                 "image": {"grid_h": grid_h, "grid_w": grid_w,
                           "patches": [round(float(v), 7) for v in patches.reshape(-1)]}}
    (out / "ref.json").write_text(json.dumps(reference))
    print(f"wrote {out}: {tokens} image tokens, prompt {len(prompt)}, full {full[len(prompt):]}")


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("--out", type=Path, required=True)
    parser.add_argument("--max-new", type=int, default=16)
    args = parser.parse_args()
    build(args.out, max_new=args.max_new)
    return 0


if __name__ == "__main__":
    sys.exit(main())