Skip to content

A computer, compiled into the weights of a language model. An exact CPU inside Qwen3.5-4B, with Gated DeltaNet state as RAM.

2.7 KB
Raw Blame
"""The Hugging Face export: loads through transformers and runs games token-exact."""

from functools import cache

import numpy as np
import pytest

torch = pytest.importorskip("torch")
transformers = pytest.importorskip("transformers")

from chipformer.chip8 import build
from chipformer.compile import compile_graph
from chipformer.dense import forward
from chipformer.hf_export import GAMES, example_game, export
from chipformer.trace import Machine


@pytest.fixture(scope="module")
def model(tmp_path_factory):
    out = tmp_path_factory.mktemp("hf")
    export(out)
    return transformers.AutoModelForCausalLM.from_pretrained(out, trust_remote_code=True,
                                                             dtype=torch.float64)


@cache
def dense_weights():
    return compile_graph(build())


def run_both(model, frames: int, sparse: bool):
    """The same input schedule through the transformer and the reference emulator."""
    g = example_game()
    rom = (GAMES / g["file"]).read_bytes()
    quirks = ",".join(g["quirks"])
    game = model.play(rom, quirks=quirks, ipf=g["ipf"], keep_tokens=True, sparse=sparse)
    ref = Machine(rom, quirks)
    keys = g["keys"]
    for f in range(frames):
        held = [keys["a"]] if f % 30 == 2 else [keys["left"]] if 5 <= f % 30 < 9 else \
            [keys["down"]] if f % 30 >= 20 else []
        game.frame(held)
        for k in range(16):
            ref.set_key(k, k in held)
        ref.tick()
        for _ in range(g["ipf"]):
            ref.step()
    return game, ref


def test_loads_as_causal_lm(model):
    assert type(model).__name__ == "ChipformerForCausalLM"
    assert next(model.parameters()).dtype == torch.float64
    assert sum(int((p != 0).sum()) for p in model.parameters()) == dense_weights()[1]["nonzero_params"]


def test_matches_numpy_dense_forward(model):
    w, config = dense_weights()
    tokens = Machine((GAMES / example_game()["file"]).read_bytes()).tokens[:400]
    _, want = forward(w, config, tokens)
    model.sparse(False)
    with torch.inference_mode():
        got = model(torch.tensor([tokens])).logits[0].numpy()
    np.testing.assert_allclose(got, want, rtol=1e-12, atol=1e-6)
    assert (got.argmax(1) == want.argmax(1)).all()


@pytest.mark.parametrize("sparse", [True, False])
def test_game_trace_is_exact(model, sparse):
    game, ref = run_both(model, 40 if sparse else 8, sparse)
    assert len(game.tokens) > len(ref.tokens[:1]) and game.tokens == ref.tokens
    assert game.cells == ref.emu.cells


def test_rollover_keeps_running(model):
    model.config.rollover_at = 2500
    try:
        game, ref = run_both(model, 60, True)
    finally:
        model.config.rollover_at = 250_000
    assert game.rollovers >= 1
    assert game.cells == ref.emu.cells