Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
3.2 KB
Raw Blame
"""GLM-5.3 prefill batching must not change a single bit.

A prefill block runs every dense matrix once over all of its tokens instead of
once per token. The kernels compute each (token, output) pair exactly as the
one-row call does, so the block size may change the speed and nothing else:
GLM53_PREFILL_CHUNK=1 (every matrix called one row at a time) and larger
chunks must print the same teacher-forcing tokens, greedy tokens and
last-position logits, byte for byte, at every dense precision.

Runs on whichever fixtures are set, as the GLM-5.3 CI job provides them:
GLM53_TINY (tools/make_glm53_tiny.py, resident f32 experts) and
COLI_GLM53_FIXTURE (the int4 streaming fixture, which exercises the routed
experts' batched CPU loop).
"""
import json
import os
import subprocess
import unittest
from pathlib import Path

C_DIR = Path(__file__).resolve().parents[1]
BINARY = next((C_DIR / name for name in ("glm53", "glm53.exe")
               if (C_DIR / name).exists()), None)
COMPARED = ("teacher_forcing", "last_logits", "greedy")


def fixtures():
    found = []
    for variable in ("GLM53_TINY", "COLI_GLM53_FIXTURE"):
        value = os.environ.get(variable)
        if value:
            found.append(Path(value))
    return found


def command(fixture):
    ref = json.loads((fixture / "ref.json").read_text())
    patches = fixture / "patches.f32"
    if patches.exists():
        grid = ref.get("grid", (0, 0))
        ids = ref["prompt"]
        extra = ["--patches", str(patches), "--grid", f"{grid[0]}x{grid[1]}"]
    else:
        ids = ref.get("prompt_ids") or ref["prompt"]
        extra = []
    return [str(BINARY), "--model", str(fixture), "--ids", ",".join(map(str, ids)),
            *extra, "--greedy", "4", "--logits"]


def run(fixture, bits, chunk):
    env = {**os.environ, "GLM53_BITS": str(bits), "GLM53_PREFILL_CHUNK": str(chunk)}
    for name in ("CLUSTER_WORKERS", "EXPERT_WORKER"):
        env.pop(name, None)
    result = subprocess.run(command(fixture), capture_output=True, text=True,
                            timeout=300, env=env)
    if result.returncode != 0:
        raise AssertionError(result.stderr)
    lines = {}
    for line in result.stdout.splitlines():
        if line.strip():
            lines[line.split()[0]] = line.split()[1:]
    return lines


@unittest.skipUnless(fixtures(), "neither GLM53_TINY nor COLI_GLM53_FIXTURE is set")
class Glm53PrefillBatchTest(unittest.TestCase):
    def test_chunk_size_never_changes_the_output(self):
        self.assertIsNotNone(BINARY, "a fixture is set but glm53 is not built")
        for fixture in fixtures():
            for bits in (32, 8, 4):
                one = run(fixture, bits, 1)
                for key in COMPARED:          # a refused run must not pass as "equal"
                    self.assertTrue(one.get(key), f"{fixture.name}: no {key} at chunk 1")
                for chunk in (3, 7, 128):
                    with self.subTest(fixture=fixture.name, bits=bits, chunk=chunk):
                        got = run(fixture, bits, chunk)
                        for key in COMPARED:
                            self.assertEqual(got.get(key), one[key],
                                             f"{key} at chunk {chunk} differs from chunk 1")


if __name__ == "__main__":
    unittest.main()