Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
15.5 KB
Raw Blame
#!/usr/bin/env python3
"""Generate the tiny Laya checkpoint used by tests/test_laya_tiny.py.

numpy-only and deterministic: CI regenerates it from SEED and gets byte-identical
files on every platform. The reference probabilities in laya_tiny/ref.json are
NOT produced here: they come from the `laya` package itself (Convai Innovations'
reference implementation), run offline by tools/make_laya_ref.py, so the engine
and its oracle never share an author.

The fixture has the layout of the release (convaiinnovations/laya):

    rl_agent_config.json            head_layers, max_len, head_max_len, act_costs,
                                    temperature and temperature_by_options (the
                                    release's values, so choice:11+ is clamped)
    encoder/config.json             ModernBERT, transformers 5 field names
    tokenizer/tokenizer.json        byte-level BPE, NFC, GPT-2 regex, added tokens
    tokenizer/tokenizer_config.json
    model.safetensors               F16 like the release, temperature in F32

and a geometry that crosses every path of laya.c: a global layer every three
(0 and 3) with RoPE theta 160000 and sliding layers with theta 10000 and a
window of +-8 that every sequence outgrows; two decision-head layers with two
heads (hidden 128 = 2 x 64, the d // 64 rule); max_len 160 and head_max_len 64,
so long states are cut and many options are shrunk.

The tokenizer is trained here, by a small deterministic BPE over the corpus
below, and carries the added tokens of the release: the [CLS]/[SEP]/[MASK]
family, runs of spaces (matched before BPE), and the |||...||| placeholders.
"""

import argparse
import json
import re
import struct
import sys
from pathlib import Path

import numpy as np

SEED = 20261002

HIDDEN = 128
LAYERS = 4
HEADS = 4
INTER = 160
LOCAL_ATTENTION = 16            # the window is local_attention // 2 = 8 each side
MAX_POS = 512
VOCAB = 640                     # embedding rows; the tokenizer uses fewer
HEAD_LAYERS = 2
ACT_HIDDEN = 256
MERGES = 320

RL_AGENT_CONFIG = {
    "encoder": "answerdotai/ModernBERT-large",
    "head_layers": HEAD_LAYERS,
    "max_len": 160,
    "head_max_len": 64,
    "max_prefixes": 6,
    "act_costs": {"escalate": 0.5},
    "cost_wrong_act": 3.0,
    "amp_dtype": "bf16",
    "model_name": "laya-tiny",
    # the release's calibration: choice:11+ is 0.1006, which the reference
    # clamps to 0.5; score:6-10 and score:2 are absent and fall back to
    # temperature[1]
    "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606],
    "temperature_by_options": {
        "choice:3-5": 1.7601518630981445,
        "choice:6-10": 1.0000158548355103,
        "score:3-5": 1.2514300346374512,
        "noul:2": 1.983399510383606,
        "choice:11+": 0.10058280825614929,
        "choice:2": 1.9063563346862793,
    },
}

CORPUS = """
Hi, we were billed twice for March. Please refund the duplicate today or we will cancel our plan.
The application crashes every time I open the settings page. This is blocking our release.
Which department should handle this request? How urgent is this request?
Does the user threaten to cancel or leave? Does the user explicitly request a refund?
choice question: score question: noul question: level 0: level 1: level 2: level 3:
false: no, the statement does not hold. true: yes, the statement holds.
billing invoices payments refunds technical bugs outages system errors sales pricing new contracts
other everything else not urgent soon critical deadline or blocking issue
The customer wrote an email about the invoice, the payment and the subscription.
We would like to schedule a demo of the product for our team next week.
Our account was locked and we cannot log in since yesterday morning.
I love this product, it works great and the support team was very helpful.
This is the worst service I have ever used, nothing works and nobody answers.
The quick brown fox jumps over the lazy dog while the cat sleeps in the sun.
Numbers like 2026, 42 and 3.14 appear in tickets, as do dates like 2026-10-02.
{"subject": "Duplicate charge", "body": "Please fix this", "from": "[email protected]"}
The meeting notes say that the contract will be renewed if the price stays the same.
Security incident: a phishing email asked employees to reset their passwords.
"""

# GPT-2's map from bytes to printable code points (the ByteLevel alphabet)
def bytes_to_unicode():
    bs = (list(range(ord("!"), ord("~") + 1)) + list(range(ord("\xa1"), ord("\xac") + 1)) +
          list(range(ord("\xae"), ord("\xff") + 1)))
    cs = bs[:]
    n = 0
    for b in range(256):
        if b not in bs:
            bs.append(b)
            cs.append(256 + n)
            n += 1
    return dict(zip(bs, [chr(c) for c in cs]))


def train_bpe(corpus, merges):
    """A plain BPE: count adjacent pairs over the words, merge the most frequent,
    ties broken by the pair's text so every platform learns the same merges."""
    byte_map = bytes_to_unicode()
    words = {}
    for piece in re.findall(r" ?[A-Za-z]+| ?[0-9]+| ?[^\sA-Za-z0-9]+|\s+", corpus):
        symbols = tuple(byte_map[b] for b in piece.encode("utf-8"))
        words[symbols] = words.get(symbols, 0) + 1
    learned = []
    for _ in range(merges):
        counts = {}
        for symbols, freq in words.items():
            for pair in zip(symbols, symbols[1:]):
                counts[pair] = counts.get(pair, 0) + freq
        if not counts:
            break
        best = min(counts, key=lambda pair: (-counts[pair], pair))
        if counts[best] < 2:
            break
        learned.append(best)
        merged = {}
        for symbols, freq in words.items():
            out, i = [], 0
            while i < len(symbols):
                if i + 1 < len(symbols) and (symbols[i], symbols[i + 1]) == best:
                    out.append(symbols[i] + symbols[i + 1])
                    i += 2
                else:
                    out.append(symbols[i])
                    i += 1
            merged[tuple(out)] = merged.get(tuple(out), 0) + freq
        words = merged
    return byte_map, learned


def added(content, token_id, special, normalized, lstrip=False):
    return {"id": token_id, "content": content, "single_word": False, "lstrip": lstrip,
            "rstrip": False, "normalized": normalized, "special": special}


def build_tokenizer():
    byte_map, merges = train_bpe(CORPUS, MERGES)
    vocab = {"|||IP_ADDRESS|||": 0, "<|padding|>": 1}
    for b in range(256):
        vocab[byte_map[b]] = len(vocab)
    for left, right in merges:
        if left + right not in vocab:
            vocab[left + right] = len(vocab)
    runs = [" " * n for n in range(8, 1, -1)]          # longest first, as the release
    added_tokens = [added("|||IP_ADDRESS|||", 0, False, True), added("<|padding|>", 1, True, False)]
    for run in runs:
        vocab[run] = len(vocab)
        added_tokens.append(added(run, vocab[run], False, True))
    for name in ("|||EMAIL_ADDRESS|||", "|||PHONE_NUMBER|||"):
        vocab[name] = len(vocab)
        added_tokens.append(added(name, vocab[name], False, True))
    vocab["<|endoftext|>"] = len(vocab)
    added_tokens.append(added("<|endoftext|>", vocab["<|endoftext|>"], True, False))
    next_id = len(vocab)
    specials = {}
    for name in ("[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"):
        specials[name] = next_id
        added_tokens.append(added(name, next_id, True, False, lstrip=name == "[MASK]"))
        next_id += 1
    for i in range(2):
        added_tokens.append(added(f"[unused{i}]", next_id, False, True))
        next_id += 1
    if next_id > VOCAB:
        raise SystemExit(f"tokenizer needs {next_id} ids, the model has {VOCAB}")
    tokenizer = {
        "version": "1.0",
        "truncation": None,
        "padding": None,
        "added_tokens": added_tokens,
        "normalizer": {"type": "NFC"},
        "pre_tokenizer": {"type": "ByteLevel", "add_prefix_space": False, "trim_offsets": True,
                          "use_regex": True},
        "post_processor": {
            "type": "TemplateProcessing",
            "single": [{"SpecialToken": {"id": "[CLS]", "type_id": 0}},
                       {"Sequence": {"id": "A", "type_id": 0}},
                       {"SpecialToken": {"id": "[SEP]", "type_id": 0}}],
            "pair": [{"SpecialToken": {"id": "[CLS]", "type_id": 0}},
                     {"Sequence": {"id": "A", "type_id": 0}},
                     {"SpecialToken": {"id": "[SEP]", "type_id": 0}},
                     {"Sequence": {"id": "B", "type_id": 0}},
                     {"SpecialToken": {"id": "[SEP]", "type_id": 0}}],
            "special_tokens": {name: {"id": name, "ids": [token], "tokens": [name]}
                               for name, token in specials.items()},
        },
        "decoder": {"type": "ByteLevel", "add_prefix_space": False, "trim_offsets": True,
                    "use_regex": True},
        "model": {"type": "BPE", "dropout": None, "unk_token": None,
                  "continuing_subword_prefix": None, "end_of_word_suffix": None,
                  "fuse_unk": False, "byte_fallback": False, "ignore_merges": False,
                  "vocab": vocab, "merges": [[left, right] for left, right in merges]},
    }
    config = {"clean_up_tokenization_spaces": True, "cls_token": "[CLS]", "mask_token": "[MASK]",
              "model_input_names": ["input_ids", "attention_mask"], "model_max_length": MAX_POS,
              "pad_token": "[PAD]", "sep_token": "[SEP]",
              "tokenizer_class": "PreTrainedTokenizerFast", "unk_token": "[UNK]"}
    return tokenizer, config, specials


def encoder_config(specials):
    layer_types = ["full_attention" if i % 3 == 0 else "sliding_attention" for i in range(LAYERS)]
    return {
        "architectures": ["ModernBertForMaskedLM"],
        "attention_bias": False, "attention_dropout": 0.0,
        "bos_token_id": specials["[CLS]"], "cls_token_id": specials["[CLS]"],
        "eos_token_id": specials["[SEP]"], "sep_token_id": specials["[SEP]"],
        "pad_token_id": specials["[PAD]"],
        "classifier_activation": "gelu", "classifier_bias": False, "classifier_dropout": 0.0,
        "classifier_pooling": "mean", "decoder_bias": True, "deterministic_flash_attn": False,
        "dtype": "float32", "embedding_dropout": 0.0, "global_attn_every_n_layers": 3,
        "gradient_checkpointing": False, "hidden_activation": "gelu", "hidden_size": HIDDEN,
        "initializer_cutoff_factor": 2.0, "initializer_range": 0.02,
        "intermediate_size": INTER, "layer_norm_eps": 1e-05, "layer_types": layer_types,
        "local_attention": LOCAL_ATTENTION, "max_position_embeddings": MAX_POS,
        "mlp_bias": False, "mlp_dropout": 0.0, "model_type": "modernbert", "norm_bias": False,
        "norm_eps": 1e-05, "num_attention_heads": HEADS, "num_hidden_layers": LAYERS,
        "position_embedding_type": "absolute", "repad_logits_with_grad": False,
        "rope_parameters": {"full_attention": {"rope_theta": 160000.0, "rope_type": "default"},
                            "sliding_attention": {"rope_theta": 10000.0, "rope_type": "default"}},
        "sparse_pred_ignore_index": -100, "sparse_prediction": False,
        "tie_word_embeddings": True, "transformers_version": "5.0.0", "vocab_size": VOCAB,
    }


def build_tensors(rng):
    """Random weights at scales that keep every layer contributing and the logits
    of a question a few units apart, so the decisions are not near-ties."""
    d, I = HIDDEN, INTER
    tensors = {}

    def normal(shape, scale):
        return (rng.standard_normal(shape) * scale).astype(np.float32)

    def norm_weight(n):
        return (1.0 + 0.1 * rng.standard_normal(n)).astype(np.float32)

    def put(name, array):
        tensors[name] = array.astype(np.float16)

    put("encoder.embeddings.tok_embeddings.weight", normal((VOCAB, d), 1.0))
    put("encoder.embeddings.norm.weight", norm_weight(d))
    for layer in range(LAYERS):
        prefix = f"encoder.layers.{layer}."
        if layer:
            put(prefix + "attn_norm.weight", norm_weight(d))
        put(prefix + "attn.Wqkv.weight", normal((3 * d, d), 1.0 / np.sqrt(d)))
        put(prefix + "attn.Wo.weight", normal((d, d), 0.5 / np.sqrt(d)))
        put(prefix + "mlp_norm.weight", norm_weight(d))
        put(prefix + "mlp.Wi.weight", normal((2 * I, d), 1.0 / np.sqrt(d)))
        put(prefix + "mlp.Wo.weight", normal((d, I), 0.5 / np.sqrt(I)))
    put("encoder.final_norm.weight", norm_weight(d))
    for layer in range(HEAD_LAYERS):
        prefix = f"head.layers.{layer}."
        put(prefix + "self_attn.in_proj_weight", normal((3 * d, d), 1.0 / np.sqrt(d)))
        put(prefix + "self_attn.in_proj_bias", normal(3 * d, 0.05))
        put(prefix + "self_attn.out_proj.weight", normal((d, d), 0.5 / np.sqrt(d)))
        put(prefix + "self_attn.out_proj.bias", normal(d, 0.05))
        put(prefix + "linear1.weight", normal((4 * d, d), 1.0 / np.sqrt(d)))
        put(prefix + "linear1.bias", normal(4 * d, 0.05))
        put(prefix + "linear2.weight", normal((d, 4 * d), 0.5 / np.sqrt(4 * d)))
        put(prefix + "linear2.bias", normal(d, 0.05))
        put(prefix + "norm1.weight", norm_weight(d))
        put(prefix + "norm1.bias", normal(d, 0.1))
        put(prefix + "norm2.weight", norm_weight(d))
        put(prefix + "norm2.bias", normal(d, 0.1))
    put("type_emb.weight", normal((3, d), 0.5))
    put("scorer.0.weight", norm_weight(d))
    put("scorer.0.bias", normal(d, 0.1))
    put("scorer.1.weight", normal((d, d), 1.0 / np.sqrt(d)))
    put("scorer.1.bias", normal(d, 0.1))
    put("scorer.3.weight", normal((1, d), 3.0 / np.sqrt(d)))
    put("scorer.3.bias", normal(1, 0.1))
    put("act_head.0.weight", normal((ACT_HIDDEN, d + 4), 1.0 / np.sqrt(d + 4)))
    put("act_head.0.bias", normal(ACT_HIDDEN, 0.1))
    put("act_head.2.weight", normal((2, ACT_HIDDEN), 3.0 / np.sqrt(ACT_HIDDEN)))
    put("act_head.2.bias", normal(2, 0.1))
    tensors["temperature"] = np.ones(3, dtype=np.float32)
    return tensors


def write_safetensors(path, tensors):
    header, offset = {}, 0
    for name, array in tensors.items():
        dtype = {"float16": "F16", "float32": "F32"}[array.dtype.name]
        header[name] = {"dtype": dtype, "shape": list(array.shape),
                        "data_offsets": [offset, offset + array.nbytes]}
        offset += array.nbytes
    blob = json.dumps(header, separators=(",", ":")).encode()
    blob += b" " * ((-len(blob)) % 8)
    with open(path, "wb") as handle:
        handle.write(struct.pack("<Q", len(blob)))
        handle.write(blob)
        for array in tensors.values():
            handle.write(np.ascontiguousarray(array).view(np.uint8).tobytes())


def write_json(path, value):
    path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")


def main():
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("--output", default="./laya_tiny")
    args = parser.parse_args()
    out = Path(args.output)
    (out / "encoder").mkdir(parents=True, exist_ok=True)
    (out / "tokenizer").mkdir(parents=True, exist_ok=True)
    tokenizer, tokenizer_config, specials = build_tokenizer()
    write_json(out / "tokenizer" / "tokenizer.json", tokenizer)
    write_json(out / "tokenizer" / "tokenizer_config.json", tokenizer_config)
    write_json(out / "encoder" / "config.json", encoder_config(specials))
    write_json(out / "rl_agent_config.json", RL_AGENT_CONFIG)
    write_safetensors(out / "model.safetensors", build_tensors(np.random.default_rng(SEED)))
    print(f"wrote {out}: {len(tokenizer['model']['vocab'])} vocab entries, "
          f"{len(tokenizer['model']['merges'])} merges", file=sys.stderr)
    return 0


if __name__ == "__main__":
    sys.exit(main())