Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
23.6 KB
Raw Blame
#!/usr/bin/env python3
"""Speculative decoding with deeper verifies: MTP drafts up to 3 (qwen38) and
prompt-lookup drafts up to 5 (qwen38 and qwen36), against plain decoding.

The gate is the one docs/speculative.md states: with drafts, the tokens and the
last logits are byte-identical to the same run without them, whatever the drafts
were and wherever they were rejected. On a tiny fixture:

  reference  the fixture's prompt with a repeating tail (so the context has
             n-grams to find), decoded plainly for N_NEW tokens; that decode is
             the oracle every forced mode drafts from.
  mtp        (qwen38, a fixture with an MTP head) Q38_MTP_DRAFTS=1, 2, 3 in the
             normal mode and forced: every draft rejected (reject), every one
             right (accept), alternating verifies (mixed), the rejected row
             cycling over the verifies (cycle) and the rejection at each row
             (row1..rowD); the run's forwards, tokens and accepted/proposed
             drafts must be the count the loop gives. Q38_MTP_DRAFTS=0 (the gate
             picks the depth) in the normal and cycle modes.
  lookup     COLI_LOOKUP=1 with the gate off (COLI_SPEC_GATE=0) and forced
             proposals of the oracle's tokens: accept, mixed, cycle, row1..row5
             and a shallower COLI_LOOKUP_DRAFTS, counts checked; natural lookup
             with the gate on and off (off: the repeating prompt must give
             verifies, and accepted drafts), and on qwen38 combined with the MTP
             head at depth 3 and at the gate's depth.
  serve      (fixtures with a tokenizer) a session of turns with drafts against
             the same session without: a repeating prompt, its continuation by
             the reply (prefix reuse of the state a speculating turn left, which
             a turn cut off mid-verify must leave as plain decoding would), a
             pinned prompt and a turn restoring the pin, a sampled turn and a
             logprobs turn. Every DATA and ECHO frame must be the same.

usage: spec_drafts_harness.py --engine ./qwen38 --kind qwen38 --fixture qwen38_tiny_mtp [--ref ref.json]
       [--env K=V ...] [--mtp] [--serve] [--n-new 40] [--cap 2]
The extra environment applies to both arms (COLI_VULKAN=1 COLI_VK_CHAIN=1 runs the
whole gate in the dense chain: with drafts against without, on the device).
"""
import argparse
import json
import os
import re
import subprocess
import sys
import tempfile
import threading
from pathlib import Path

FAILS = []
SANITIZE = False   # --sanitize: every run's stderr is checked for an ASan or UBSan report
REQUIRE_KV_SPLIT = False
SAN_RE = re.compile(rb"ERROR: AddressSanitizer|runtime error:")
BASE_ENV = {"OMP_NUM_THREADS": "2", "COLI_NO_OMP_TUNE": "1", "COLI_CUDA": "0", "Q38_TRUNK_GPU": "0",
            "NOSTREAM": "1", "USAGE_SAVE": "0",
            # plain decoding is the reference, and both draft sources are on by default since
            # 1.13.0: every run starts from them off, and a case turns on what it checks
            "Q38_MTP": "0", "COLI_LOOKUP": "0"}
SPEC_KEYS = ("Q38_MTP", "Q38_MTP_DRAFTS", "Q38_MTP_FORCE", "Q38_MTP_DUMP", "COLI_LOOKUP", "COLI_LOOKUP_DRAFTS",
             "COLI_LOOKUP_FORCE", "COLI_SPEC_GATE", "DUMP", "SERVE", "N_NEW", "TOK")
MAX_ROWS = 6


def sanitizer_check(stderr, what):
    if SANITIZE and SAN_RE.search(stderr if isinstance(stderr, bytes) else stderr.encode()):
        text = stderr.decode(errors="replace") if isinstance(stderr, bytes) else stderr
        check(False, f"{what}: sanitizer report\n" + text[-4000:])


def check(ok, what):
    print(("  ok   " if ok else "  FAIL ") + what, flush=True)
    if not ok:
        FAILS.append(what)


class Engine:
    def __init__(self, path, kind, fixture, cap, extra):
        self.path, self.kind, self.fixture, self.cap, self.extra = Path(path).resolve(), kind, Path(fixture), cap, extra

    def env(self, spec):
        env = {k: v for k, v in os.environ.items() if k not in SPEC_KEYS}
        env.update(BASE_ENV)
        env.update(self.extra)
        env["SNAP"] = str(self.fixture)
        env.update(spec)
        return env

    def run(self, ref, spec, dump):
        env = self.env(dict(spec, DUMP=str(dump)))
        r = subprocess.run([str(self.path), str(self.cap), "8", str(ref)], env=env, capture_output=True,
                           timeout=1800)
        sanitizer_check(r.stderr, " ".join(f"{k}={v}" for k, v in spec.items()) or "plain")
        if REQUIRE_KV_SPLIT:
            trace = rb"\[VK\] " + self.kind.encode() + rb" chain: KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part"
            check(re.search(trace, r.stderr) is not None,
                  "split KV executed host attention: " + (" ".join(f"{k}={v}" for k, v in spec.items()) or "plain"))
        return r


SPEC_LINE = r"\[{kind} {src}\] run: [0-9.]+ tokens/forward \((\d+) forwards per (\d+) tokens\) \| acceptance [0-9.]+% \((\d+)/(\d+) drafts"


def counts(kind, src, stderr):
    """(forwards, tokens, accepted, drafts) of the run's line for a draft source."""
    m = re.search(SPEC_LINE.format(kind=kind, src=src), stderr.decode(errors="replace"))
    return tuple(int(m.group(i)) for i in (1, 2, 3, 4)) if m else None


def expected(n_new, depth, mode, row=0, room_cap=MAX_ROWS - 1):
    """What the speculation loop must count for a CLI run of n_new tokens with every
    verify carrying min(depth, more - 1) drafts, forced: the prompt's forward gives
    token 1, then one call per token fed; a pick equal to the next draft is answered
    from the verify, the first that differs ends it."""
    drafts = accepted = forwards = tokens = verifies = 0
    queue = []
    for s in range(n_new - 1):
        tokens += 1
        if queue:
            if queue.pop(0):
                accepted += 1
                continue
            queue = []
        more = n_new - 1 - s
        k = min(depth, more - 1, room_cap)
        if k >= 1:
            if mode == "reject":
                wrong = 1
            elif mode == "accept":
                wrong = 0
            elif mode == "mixed":
                wrong = 1 if verifies % 2 else 0
            elif mode == "cycle":
                wrong = verifies % (k + 1) + 1
            else:
                wrong = row
            queue = [wrong == 0 or j < wrong - 1 for j in range(k)]
            drafts += k
            verifies += 1
        forwards += 1
    return forwards, tokens, accepted, drafts


def engine_tokens(stdout):
    m = re.search(rb"^C engine :([^\n]*?)(?:Text|$)", stdout, re.M)
    return [int(t) for t in m.group(1).split()] if m else None


def check_device_loss(eng, ref, base_out, tmp):
    """A late failed chunk must replay embeddings, not earlier chunks' streams."""
    forced = ({"Q38_MTP": "1", "Q38_MTP_DRAFTS": "3", "Q38_MTP_FORCE": "cycle"}
              if eng.kind == "qwen38" else
              {"COLI_LOOKUP": "1", "COLI_LOOKUP_DRAFTS": "5", "COLI_SPEC_GATE": "0", "COLI_LOOKUP_FORCE": "cycle"})
    probe = eng.run(ref, forced, tmp / "loss-probe.f32")
    frames = re.search(rb"\[VK\] " + eng.kind.encode() + rb" chain: [0-9]+ forwards, ([0-9]+) frames", probe.stderr)
    if not frames or int(frames[1]) <= 15:
        check(False, "device-loss probe: no sufficiently long chain run")
        print(probe.stderr.decode(errors="replace")[-4000:])
        return
    check(probe.returncode == 0 and engine_tokens(probe.stdout) == engine_tokens(base_out),
          "device-loss probe: speculative tokens = plain")
    fault = int(frames[1]) - 15
    lost = eng.run(ref, dict(forced, COLI_VK_CHAIN_FAULT=str(fault)), tmp / "loss.f32")
    trace = lost.stderr.decode(errors="replace")
    released = re.search(r": ([1-9][0-9]*) dense matrices on the device only", trace)
    reloaded = re.search(r"\), ([1-9][0-9]*) read back from disk for the CPU", trace)
    rebuilt = re.search(r"rebuilding the state of ([1-9][0-9]*) positions", trace)
    check(bool(released and reloaded and rebuilt), "device loss: host matrices released, restored and recurrent state rebuilt")
    same = lost.returncode == 0 and engine_tokens(lost.stdout) == engine_tokens(base_out)
    check(same, f"device loss at frame {fault}: every token = plain decoding after CPU replay")
    if not same:
        print(lost.stdout.decode(errors="replace"))
        print(trace[-4000:])


def build_reference(eng, base_ref, n_new, tmp):
    """The fixture's prompt with a repeating tail, decoded plainly: the oracle."""
    data = json.loads(Path(base_ref).read_text(encoding="utf-8"))
    vocab = len(data.get("final_logits") or []) or 64
    prompt = list(data["prompt_ids"])
    tail = [t % vocab for t in (7, 11, 13, 17)]
    prompt += tail * 3 + tail[:2]
    probe = tmp / "probe.json"
    probe.write_text(json.dumps({"schema_version": 1, "prompt_ids": prompt, "full_ids": prompt + [0] * n_new}))
    r = eng.run(probe, {}, tmp / "probe.f32")
    gen = engine_tokens(r.stdout)
    if not gen or len(gen) != n_new:
        sys.exit("could not decode the reference:\n" + r.stdout.decode(errors="replace")
                 + r.stderr.decode(errors="replace")[-3000:])
    ref = tmp / "ref_spec.json"
    ref.write_text(json.dumps({"schema_version": 1, "prompt_ids": prompt, "full_ids": prompt + gen}))
    base = eng.run(ref, {}, tmp / "base.f32")
    if base.returncode:
        sys.exit("the plain decode does not reproduce itself:\n" + base.stdout.decode(errors="replace"))
    return ref, base.stdout, (tmp / "base.f32").read_bytes()


def gate(eng, ref, base_out, base_logits, tmp, name, spec, src=None, want=None, extra_check=None):
    dump = tmp / "spec.f32"
    if dump.exists():
        dump.unlink()
    r = eng.run(ref, spec, dump)
    same = r.returncode == 0 and r.stdout == base_out and dump.exists() and dump.read_bytes() == base_logits
    got = counts(eng.kind, src, r.stderr) if src else None
    if want is not None:
        same &= got == want
    elif src:
        same &= got is not None and got[2] <= got[3] and got[0] <= got[1]
    extra = extra_check(got, r.stderr) if extra_check else ""
    if extra is None:
        same = False
        extra = ""
    detail = f" {src} forwards/tokens/accepted/drafts {got}" if src else ""
    check(same, f"{name}: tokens and last logits = plain decoding{detail}{extra}")
    if not same:
        print(r.stdout.decode(errors="replace")[-1500:] + r.stderr.decode(errors="replace")[-2500:])
        if want is not None:
            print(f"       want {want}")
    return r


def check_mtp(eng, ref, base_out, base_logits, n_new, tmp, quick):
    print(f"mtp: {eng.fixture}")
    for depth in ((3,) if quick else (1, 2, 3)):
        modes = ["reject", "cycle", "row2"] if quick else \
            ["", "reject", "accept", "mixed", "cycle"] + [f"row{r}" for r in range(1, depth + 1)]
        for mode in modes:
            spec = {"Q38_MTP": "1", "Q38_MTP_DRAFTS": str(depth), "Q38_MTP_FORCE": mode}
            want = None
            if mode:
                m, row = (mode, 0) if not mode.startswith("row") else ("row", int(mode[3:]))
                want = expected(n_new, depth, m, row, 3)
            gate(eng, ref, base_out, base_logits, tmp, f"MTP depth {depth} {mode or 'normal'}", spec, "MTP", want)
    for mode in (() if quick else ("", "cycle")):
        gate(eng, ref, base_out, base_logits, tmp, f"MTP depth by the gate {mode or 'normal'}",
             {"Q38_MTP": "1", "Q38_MTP_DRAFTS": "auto", "Q38_MTP_FORCE": mode}, "MTP")


def check_lookup(eng, ref, base_out, base_logits, n_new, tmp, mtp, quick):
    print(f"lookup: {eng.fixture}")
    for drafts in ((5,) if quick else (5, 2)):
        modes = ["cycle", "row3"] if quick else ["accept", "mixed", "cycle"] + [f"row{r}" for r in range(1, drafts + 1)]
        for mode in modes:
            m, row = (mode, 0) if not mode.startswith("row") else ("row", int(mode[3:]))
            spec = {"COLI_LOOKUP": "1", "COLI_SPEC_GATE": "0", "COLI_LOOKUP_FORCE": mode,
                    "COLI_LOOKUP_DRAFTS": str(drafts)}
            gate(eng, ref, base_out, base_logits, tmp, f"lookup forced {mode}, {drafts} drafts", spec, "lookup",
                 expected(n_new, drafts, m, row, drafts))

    def proposed(got, _err):
        return f" (proposals on the repeating prompt: {got[3] if got else 0})" if got and got[3] > 0 else None

    gate(eng, ref, base_out, base_logits, tmp, "lookup, gate off", {"COLI_LOOKUP": "1", "COLI_SPEC_GATE": "0"},
         "lookup", extra_check=proposed)
    if quick:
        return
    gate(eng, ref, base_out, base_logits, tmp, "lookup, gate on", {"COLI_LOOKUP": "1"}, "lookup")
    if mtp:
        gate(eng, ref, base_out, base_logits, tmp, "MTP depth 3 + lookup",
             {"Q38_MTP": "1", "Q38_MTP_DRAFTS": "3", "COLI_LOOKUP": "1"}, "lookup")
        gate(eng, ref, base_out, base_logits, tmp, "MTP depth by the gate + lookup",
             {"Q38_MTP": "1", "Q38_MTP_DRAFTS": "auto", "COLI_LOOKUP": "1"}, "lookup")
        gate(eng, ref, base_out, base_logits, tmp, "MTP depth 2 + lookup, gate off",
             {"Q38_MTP": "1", "Q38_MTP_DRAFTS": "2", "COLI_LOOKUP": "1", "COLI_SPEC_GATE": "0"}, "lookup")


# ---------- serve ----------

READY = b"\x01\x01READY\x01\x01"


def serve_session(eng, spec, alphabet):
    """Turns sent one at a time. Returns (frames, stderr). A turn's prompt may be
    built from an earlier turn's reply (the transcript a chat client resends), so
    both arms run the same procedure and get the same prompts when they agree."""
    env = eng.env(dict(spec, SERVE="1", TOK=str(eng.fixture / "tokenizer.json"), COLI_PREFIX_LOG="1",
                       Q38_PREFIX_LOG="1", Q38_MAXT="128", Q36_MAXT="512"))
    p = subprocess.Popen([str(eng.path), str(eng.cap), "8"], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    err = []
    threading.Thread(target=lambda: [err.append(x) for x in iter(p.stderr.readline, b"")], daemon=True).start()
    frames, replies = [], {}
    a = alphabet
    rep = (a[:5] * 6)                      # a prompt built to repeat
    turns = [
        ("t1", lambda: rep, 24, ""),
        ("t2", lambda: rep + replies["t1"] + a[:3], 20, ""),             # the transcript, continued
        ("t3", lambda: rep + replies["t1"] + a[:3] + replies["t2"], 9, "pin=1"),
        ("t4", lambda: rep + replies["t1"] + a[:3] + replies["t2"] + a[5:8], 16, ""),
        ("t5", lambda: rep + a[1:4] * 3, 16, "logprobs=2"),
        ("t6", lambda: rep + a[1:4] * 3 + replies["t5"], 12, ""),
        ("t7", lambda: rep, 14, "", "0.9", "0.95"),
        ("t8", lambda: rep + replies["t1"] + a[:3] + replies["t2"] + a[2:4], 18, ""),
        ("t9", lambda: rep + replies["t1"] + a[:3] + replies["t2"] + a[2:4] + replies["t8"] + a[4:6], 10, ""),
    ]
    try:
        while True:
            line = p.stdout.readline()
            if not line:
                raise RuntimeError("engine exited before READY")
            if READY in line:
                break
        for t in turns:
            rid, build, max_tok, ext = t[:4]
            temp, top_p = (t[4], t[5]) if len(t) > 4 else ("0", "1")
            prompt = build()
            p.stdin.write(f"SUBMIT {rid} 0 {len(prompt)} {max_tok} {temp} {top_p}{' ' + ext if ext else ''}\n".encode()
                          + prompt + b"\n")
            p.stdin.flush()
            reply = []
            while True:
                line = p.stdout.readline()
                if not line:
                    raise RuntimeError("engine died")
                kind = line.split(b" ", 1)[0]
                if kind in (b"DATA", b"ECHO"):
                    body = p.stdout.read(int(line.split()[2]))
                    p.stdout.readline()
                    frames.append(line.rstrip() + b"|" + body)
                    if kind == b"DATA":
                        reply.append(body)
                elif kind == b"DONE":
                    frames.append(b" ".join(line.split()[:4]))
                    break
                elif kind == b"ERROR":
                    raise RuntimeError(f"engine error: {line!r}")
            replies[rid] = b"".join(reply)
        p.stdin.close()
        p.wait(timeout=120)
    except Exception as exc:   # noqa: BLE001 -- a failed gate, reported
        p.kill()
        p.wait()
        frames.append(f"harness: {exc!r}".encode())
    text = b"".join(err).decode(errors="replace")
    sanitizer_check(text, "serve " + (" ".join(f"{k}={v}" for k, v in spec.items()) or "plain"))
    return frames, text


def check_serve(eng, mtp, quick):
    print(f"serve: {eng.fixture}")
    sys.path.insert(0, str(Path(__file__).resolve().parent))
    from prefix_serve_harness import ensure_byte_tokenizer
    ensure_byte_tokenizer(eng.fixture)
    alphabet = b"0123456789" if eng.kind == "qwen38" else b"abcdefghij"
    base, berr = serve_session(eng, {}, alphabet)
    breused = reuse_count(berr)
    check(len([f for f in base if f.startswith(b"DONE")]) == 9, f"plain session: 9 turns ({len(base)} frames)")
    # (name, environment, what the session must have done for the gate to mean something)
    arms = [("lookup, gate off", {"COLI_LOOKUP": "1", "COLI_SPEC_GATE": "0"}, "verified"),
            ("lookup, gate on", {"COLI_LOOKUP": "1"}, "")]
    if mtp:
        arms += [("MTP depth 3 + lookup, gate off", {"Q38_MTP": "1", "Q38_MTP_DRAFTS": "3", "COLI_LOOKUP": "1",
                                                     "COLI_SPEC_GATE": "0"}, "cut"),
                 ("MTP depth by the gate", {"Q38_MTP": "1", "Q38_MTP_DRAFTS": "auto"}, ""),
                 ("MTP depth 2, every draft rejected", {"Q38_MTP": "1", "Q38_MTP_DRAFTS": "2",
                                                        "Q38_MTP_FORCE": "reject"}, "")]
    if quick:
        arms = arms[:1] + arms[2:3]
    if eng.kind == "qwen36":
        check_serve_eos_mid_verify(eng, base, alphabet)
    for name, spec, need in arms:
        frames, err = serve_session(eng, spec, alphabet)
        verifies = sum(int(x) for x in re.findall(r"acceptance [0-9.]+% \(\d+/\d+ drafts in (\d+) verifies\)", err))
        mtp_drafts = sum(int(x) for x in re.findall(r"\[qwen38 MTP\] turn \S+: .*?\(\d+/(\d+) drafts\)", err))
        cut = len(re.findall(r"ended mid-verify", err))
        reused = reuse_count(err)
        meant = reused > 0 and reused == breused and (need != "verified" or verifies > 0) and (need != "cut" or cut > 0)
        check(frames == base and meant,
              f"{name}: every frame = plain ({verifies} lookup verifies, {mtp_drafts} MTP drafts, {cut} turns "
              f"ended mid-verify, {reused} turns reused a prefix, plain {breused})")
        if frames != base:
            for x, y in zip(base, frames):
                if x != y:
                    print(f"       first difference:\n         plain {x[:160]!r}\n         spec  {y[:160]!r}")
                    break
            print(err[-3000:])


def reuse_count(err):
    """Turns that reused a prefix: live state (qwen36) or a snapshot (qwen38). A
    wrong state after a speculating turn can hide behind a reuse that is refused
    (the record no longer matches, so the turn prefills from scratch and answers
    right): the sessions with drafts must reuse as often as the one without."""
    return len(re.findall(r"\[PREFIX\] reusing|\[qwen38 prefix\] request=\S+ reused=[1-9]", err))


def check_serve_eos_mid_verify(eng, base_frames, alphabet):
    """A turn that ends inside a verify (the sampled token is an end of sequence while
    drafts are pending) must leave the state of the tokens fed, which the next turn
    reuses live (qwen36 keeps the state and reuses it when the transcript extends it).
    The end-of-sequence id is picked from the first turn's reply: the first of its
    tokens that, made the end of sequence (Q36_EOS), ends a turn mid-verify."""
    reply = b"".join(f.split(b"|", 1)[1] for f in base_frames if f.startswith(b"DATA "))
    spec = {"COLI_LOOKUP": "1", "COLI_SPEC_GATE": "0"}
    tried = []
    for b in dict.fromkeys(reply):
        if b >= 128 or len(tried) >= 16:
            continue
        tried.append(b)
        frames, err = serve_session(eng, dict(spec, Q36_EOS=str(b)), alphabet)
        # a turn that ended mid-verify, and the next turn reusing its live state
        lines = [x for x in err.splitlines() if x.startswith("[PREFIX]") or "lookup] turn" in x]
        if not any("ended mid-verify" in x and nxt.startswith("[PREFIX] reusing")
                   for x, nxt in zip(lines, lines[1:])):
            continue
        plain, perr = serve_session(eng, {"Q36_EOS": str(b)}, alphabet)
        reused, preused = reuse_count(err), reuse_count(perr)
        check(frames == plain and reused > 0 and reused == preused,
              f"lookup, end of sequence {b} inside a verify, the next turn reusing the state it left: every "
              f"frame = plain, {reused} turns reused the live state (plain: {preused})")
        if frames != plain:
            for x, y in zip(plain, frames):
                if x != y:
                    print(f"       first difference:\n         plain {x[:160]!r}\n         spec  {y[:160]!r}")
                    break
        return
    check(False, f"no end-of-sequence id among {tried} ends a turn inside a verify before a turn that reuses it")


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--engine", required=True)
    ap.add_argument("--kind", required=True, choices=("qwen38", "qwen36"))
    ap.add_argument("--fixture", required=True)
    ap.add_argument("--ref", help="the fixture's reference (default: <fixture>/ref.json, ref_full.json for qwen36)")
    ap.add_argument("--cap", type=int, default=2)
    ap.add_argument("--n-new", type=int, default=40)
    ap.add_argument("--env", action="append", default=[], help="KEY=VALUE for both arms")
    ap.add_argument("--mtp", action="store_true", help="the fixture has an MTP head (qwen38)")
    ap.add_argument("--serve", action="store_true", help="the serve session too (needs a tokenizer)")
    ap.add_argument("--sanitize", action="store_true", help="fail on an ASan or UBSan report in any run's stderr")
    ap.add_argument("--require-kv-split", action="store_true", help="require every CLI run to execute attention over host KV rows")
    ap.add_argument("--device-loss", action="store_true", help="only the late device-loss gate with host copies released")
    ap.add_argument("--quick", action="store_true",
                    help="a subset (sanitizer runs): depth 3 rejected at row 1, at row 2 and cycling; lookup cycling "
                         "and at row 3; one serve arm per source")
    a = ap.parse_args()
    global SANITIZE, REQUIRE_KV_SPLIT
    SANITIZE = a.sanitize
    REQUIRE_KV_SPLIT = a.require_kv_split
    extra = dict(kv.split("=", 1) for kv in a.env)
    fixture = Path(a.fixture)
    ref = a.ref or str(fixture / ("ref_full.json" if a.kind == "qwen36" else "ref.json"))
    eng = Engine(a.engine, a.kind, fixture, a.cap, extra)
    with tempfile.TemporaryDirectory() as td:
        tmp = Path(td)
        if "COLI_USAGE" not in extra:
            eng.extra["COLI_USAGE"] = str(tmp / "usage")
        oracle, base_out, base_logits = build_reference(eng, ref, a.n_new, tmp)
        print(f"reference: {a.n_new} tokens decoded plainly{(' with ' + ' '.join(a.env)) if a.env else ''}")
        if a.device_loss:
            check_device_loss(eng, oracle, base_out, tmp)
        else:
            if a.mtp:
                check_mtp(eng, oracle, base_out, base_logits, a.n_new, tmp, a.quick)
            check_lookup(eng, oracle, base_out, base_logits, a.n_new, tmp, a.mtp, a.quick)
            if a.serve:
                check_serve(eng, a.mtp, a.quick)
    if FAILS:
        print(f"spec drafts: {len(FAILS)} check(s) failed")
        return 1
    print("spec drafts: device-loss replay preserves plain decoding's tokens" if a.device_loss else
          "spec drafts: every configuration gives plain decoding's tokens and last logits")
    return 0


if __name__ == "__main__":
    sys.exit(main())