Official Inference Code of EAGLE-4
# Copyright 2026 Eagle AI Corp.
# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available
"""Serving distribution gates must reject missing or incomparable evidence, pass a correct sampler whose logits
move by bf16 rounding between runs, and fail a biased one."""
import copy
import importlib.util
import json
import math
from pathlib import Path
import random
import subprocess
import sys
from types import SimpleNamespace
import pytest
@pytest.fixture
def sampling_check():
path = Path(__file__).parents[1] / "tools" / "sampling_check.py"
spec = importlib.util.spec_from_file_location("sampling_check", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def artifact():
return {
"api": "sglang",
"max_new_tokens": 3,
"samples": 20,
"settings": {
"unrestricted": {
"params": {"temperature": 1.0, "top_p": 1.0, "top_k": -1, "min_p": 0.0},
"tokens": [[[1, 2, 3]] * 20 for _ in range(4)],
"seconds": 1.0,
}
},
"seeds": {},
}
def compare(module, tmp_path, ref, candidate, scores=None, deterministic=False):
reference = tmp_path / "reference.json"
output = tmp_path / "candidate.json"
reference.write_text(json.dumps(ref))
output.write_text(json.dumps(candidate))
score_path = None
if scores is not None:
score_path = tmp_path / "scores.json"
score_path.write_text(json.dumps(scores))
args = dict(reference=reference, candidate=output, alpha=0.01, scores=score_path, deterministic=deterministic)
return module.compare(SimpleNamespace(**args))
def test_unrestricted_setting_exercises_temperature_one(sampling_check):
assert sampling_check.SETTINGS["unrestricted"] == {
"temperature": 1.0, "top_p": 1.0, "top_k": -1, "min_p": 0.0, "repetition_penalty": 1.0
}
def test_every_setting_names_each_filter(sampling_check):
# A server fills an unnamed filter from the model's generation_config (Qwen3.8: top-k 20, top-p 0.95).
for params in sampling_check.SETTINGS.values():
assert params.keys() == sampling_check.NEUTRAL.keys()
assert not sampling_check.truncates(sampling_check.SETTINGS["temperature"])
assert not sampling_check.truncates(sampling_check.SETTINGS["penalty"])
def test_complete_distribution_evidence_passes(sampling_check, tmp_path):
ref = artifact()
assert compare(sampling_check, tmp_path, ref, copy.deepcopy(ref)) == 0
@pytest.mark.parametrize(
"fault", ["empty", "missing", "refused", "parameters", "prompts", "samples", "tokens", "unnamed", "api"]
)
def test_incomplete_distribution_evidence_fails(sampling_check, tmp_path, fault):
ref = artifact()
candidate = copy.deepcopy(ref)
setting = candidate["settings"]["unrestricted"]
if fault == "empty":
ref["settings"].clear()
candidate["settings"].clear()
elif fault == "missing":
candidate["settings"].clear()
elif fault == "refused":
setting.clear()
setting["refused"] = "unsupported"
elif fault == "api":
candidate["api"] = "vllm"
elif fault == "parameters":
setting["params"]["temperature"] = 0.0
elif fault == "prompts":
setting["tokens"].pop()
elif fault == "samples":
setting["tokens"][0].pop()
elif fault == "tokens":
setting["tokens"][0][0] = [1]
elif fault == "unnamed":
for doc in (ref, candidate):
del doc["settings"]["unrestricted"]["params"]["top_p"]
assert compare(sampling_check, tmp_path, ref, candidate) == 1
def test_changed_distribution_fails(sampling_check, tmp_path):
ref = artifact()
candidate = copy.deepcopy(ref)
candidate["settings"]["unrestricted"]["tokens"] = [[[4, 5, 6]] * 20 for _ in range(4)]
assert compare(sampling_check, tmp_path, ref, candidate) == 1
@pytest.mark.parametrize("deterministic", [False, True])
def test_seeds_are_judged_only_on_a_deterministic_server(sampling_check, tmp_path, deterministic):
ref = artifact()
ref["seeds"] = {str(s): [s, 1, 2] for s in range(32)}
ref["seeds_reproduced"] = 32
candidate = copy.deepcopy(ref)
candidate["seeds_reproduced"] = 31
assert compare(sampling_check, tmp_path, ref, candidate, deterministic=deterministic) == int(deterministic)
def test_compare_runs_without_the_http_client(tmp_path):
# Only `run` talks HTTP (requests); the public test env has no requests.
path = Path(__file__).parents[1] / "tools" / "sampling_check.py"
ref = tmp_path / "ref.json"
ref.write_text(json.dumps(artifact()))
code = (
"import importlib.util, sys, types; sys.modules['requests'] = None; "
f"spec = importlib.util.spec_from_file_location('sampling_check', {str(path)!r}); "
"m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); "
f"sys.exit(m.compare(types.SimpleNamespace(reference={str(ref)!r}, candidate={str(ref)!r}, alpha=0.01, "
"scores=None, deterministic=False)))"
)
out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
assert out.returncode == 0, out.stderr
@pytest.mark.parametrize("reply", [
{"output_ids": [5, 6], "meta_info": {"spec_verify_ct": 1}},
[{"output_ids": [5, 6], "meta_info": {"spec_verify_ct": 1}}],
[{"meta_info": {"output_ids": [5, 6], "spec_verify_ct": 1}}],
])
def test_tokenspeed_generate_names_the_model_and_unwraps_the_reply(sampling_check, monkeypatch, reply):
# TokenSpeed's /generate routes by served model name and answers with a one-element list.
sent = {}
def post(url, json, timeout):
sent.update(url=url, body=json)
return SimpleNamespace(raise_for_status=lambda: None, json=lambda: reply)
monkeypatch.setitem(sys.modules, "requests", SimpleNamespace(post=post))
ids, meta = sampling_check.generate("tokenspeed", "http://h", "served", [1, 2], {"temperature": 1.0}, 2)
assert (ids, meta) == ([5, 6], {"spec_verify_ct": 1})
assert sent["url"] == "http://h/generate" and sent["body"]["model"] == "served"
assert sent["body"]["sampling_params"] == {"max_new_tokens": 2, "ignore_eos": True, "temperature": 1.0}
# A synthetic target: 40 tokens whose fp32 logits fall 0.45 (3.6 bf16 ulps at [16, 32)) per rank from 24, except that
# ranks 8 and 9 sit 0.04 apart, a near tie right at a top-8 cut. A server start moves every logit by up to a quarter
# bf16 ulp and the rank-8 one a further ulp up (odd starts) or down (even starts), so odd starts keep rank 8 and even
# starts rank 9, as a real Qwen3.8 top-p cut did between two no-spec vLLM starts (a token drawn 0 of 1428 times by one
# start and 115 of 1397 times by the other).
VOCAB, SCORED, SAMPLES = 40, 24, 2000
PROMPTS = [[101], [102], [103], [104]]
TOP_K = {"temperature": 1.0, "top_k": 8, "top_p": 1.0, "min_p": 0.0, "repetition_penalty": 1.0}
TEMPERATURE = {"temperature": 0.7, "top_k": -1, "top_p": 1.0, "min_p": 0.0, "repetition_penalty": 1.0}
def fp32_logits(context):
"""{token: logit} of a context."""
ranks = [24.0 - 0.45 * r for r in range(VOCAB)]
ranks[8] = ranks[7] - 0.04
order = list(range(VOCAB))
random.Random(f"order {context}").shuffle(order)
return {token: logit for token, logit in zip(order, ranks)}
def served_logits(context, start):
"""The logits a server start computes: fp32 plus rounding (see above), within the fp32 arbiter's 2 ulps a pair."""
noise = random.Random(f"start {start} {context}")
logits = fp32_logits(context)
eighth = sorted(logits, key=logits.get, reverse=True)[7]
served = {}
for token, value in logits.items():
ulp = 2.0 ** (math.frexp(abs(value))[1] - 8)
served[token] = value + noise.uniform(-0.25, 0.25) * ulp + (ulp if start % 2 else -ulp) * (token == eighth)
return served
def processed(logits, params):
"""The sampled distribution: temperature, then the top-k cut (no other filter is used here)."""
order = sorted(logits, key=logits.get, reverse=True)
if params["top_k"] > 0:
order = order[: params["top_k"]]
weights = [math.exp((logits[t] - logits[order[0]]) / params["temperature"]) for t in order]
return {t: w / sum(weights) for t, w in zip(order, weights)}
def draw(probs, rng):
u, total = rng.random(), 0.0
for token, p in probs.items():
total += p
if u < total:
return token
return token
def biased_chain_verify(p, q, rng):
"""One draft token y ~ q, accepted with min(1, p_y / q_y); on rejection the classic mistake of subtracting only
y's q mass from p (tests/test_sampled_tree.py's mutation) instead of drawing from norm(max(p - q, 0))."""
y = draw(q, rng)
if rng.random() * q[y] < p.get(y, 0.0):
return y
residual = dict(p)
residual[y] = max(p.get(y, 0.0) - q[y], 0.0)
total = sum(residual.values())
return draw({t: v / total for t, v in residual.items()}, rng)
def sampled_run(params, start, sampler=None, seed=0):
"""A run artifact: SAMPLES completions of 3 tokens per prompt, each token drawn by ``sampler(logits, rng)``
from the start's logits (by default the processed distribution)."""
rng = random.Random(f"draws {start} {seed}")
sampler = sampler or (lambda logits, rng: draw(processed(logits, params), rng))
tokens = []
for prompt in PROMPTS:
rows = []
for _ in range(SAMPLES):
context = list(prompt)
for _ in range(3):
context.append(sampler(served_logits(tuple(context), start), rng))
rows.append(context[len(prompt) :])
tokens.append(rows)
return {
"api": "vllm",
"max_new_tokens": 3,
"samples": SAMPLES,
"prompt_ids": PROMPTS,
"settings": {"setting": {"params": dict(params), "tokens": tokens, "seconds": 1.0}},
"seeds": {},
}
def fp32_scores(module, reference):
"""What ``score`` writes for the reference: the top SCORED fp32 logits of every judged context."""
contexts = {}
for _, i, prefix in module.judged_contexts(reference):
ids = reference["prompt_ids"][i] + list(prefix)
logits = fp32_logits(tuple(ids))
top = sorted(logits, key=logits.get, reverse=True)[:SCORED]
contexts[module.context_key(ids)] = {"vocab": VOCAB, "top": [[logits[t], t] for t in top]}
return {"tolerance_ulps": 2.0, "contexts": contexts}
def test_cut_bounds_leave_near_ties_out(sampling_check):
top = [[24.0 - 0.45 * r, r] for r in range(SCORED)]
top[8][0] = top[7][0] - 0.04
kept, possible = sampling_check.cut_bounds(top, VOCAB, TOP_K, 2.0)
assert kept == set(range(7)) and possible == set(range(9))
# A clear cut (3.6 ulps) is no near tie.
kept, possible = sampling_check.cut_bounds(top, VOCAB, dict(TOP_K, top_k=5), 2.0)
assert kept == possible == set(range(5))
# top-p 0.6: shares 0.57 / 0.21 / 0.21, so whichever of the near-tied second and third ranks first is kept.
flat = [[21.0, 0], [20.0, 1], [19.98, 2], [17.0, 3]] + [[10.0 - r, 4 + r] for r in range(SCORED - 4)]
kept, possible = sampling_check.cut_bounds(flat, VOCAB, dict(TOP_K, top_k=-1, top_p=0.6), 2.0)
assert kept == {0} and possible == {0, 1, 2}
# min-p 0.05 at temperature 1: a token log(20) = 3.0 below the top, near the floor, is a near tie.
edge = [[20.0, 0], [20.0 - math.log(20.0) + 0.01, 1], [15.0, 2]] + [[10.0 - r, 3 + r] for r in range(SCORED - 3)]
kept, possible = sampling_check.cut_bounds(edge, VOCAB, dict(TOP_K, top_k=-1, min_p=0.05), 2.0)
assert kept == {0} and possible == {0, 1}
with pytest.raises(ValueError, match="past the 8 scored"):
sampling_check.cut_bounds(top[:8], VOCAB, TOP_K, 2.0)
with pytest.raises(ValueError, match="not modelled"):
sampling_check.cut_bounds(top, VOCAB, dict(TOP_K, repetition_penalty=1.5), 2.0)
def test_score_keeps_as_many_tokens_as_bound_the_cut(sampling_check):
# min-p 0.05 on a flat distribution keeps every token within log(20) = 3.0 of the top: 300 here.
pairs = [[20.0 - 0.01 * r, r] for r in range(1000)]
min_p = dict(TOP_K, top_k=-1, min_p=0.05)
top = sampling_check.bounded_top(pairs, 5000, [TOP_K, min_p], 2.0)
assert len(top) == 1000 # 64 and 256 do not bound it
kept, possible = sampling_check.cut_bounds(top, 5000, min_p, 2.0)
# Near-ties: within one ulp (0.125) either side of the top and of the floor, i.e. 25 ranks around rank 300.
assert kept == set(range(275)) and possible == set(range(325))
assert len(sampling_check.bounded_top(pairs, 5000, [TOP_K], 2.0)) == 64
@pytest.mark.parametrize("api, expected", [("sglang", 0), ("vllm", 1)])
def test_joint_and_sequential_cuts_use_the_served_engine(sampling_check, tmp_path, api, expected):
# Joint top-k=2/top-p=.5 keeps token 1 (.4 mass before it). Top-k-first
# excludes it because .4 / (.4 + .3) exceeds .5. No rounding is involved.
ref = artifact()
ref["api"] = api
ref["prompt_ids"] = PROMPTS
setting = ref["settings"]["unrestricted"]
setting["params"].update(top_k=2, top_p=0.5)
setting["tokens"] = [[[1, 1, 1]] * ref["samples"] for _ in PROMPTS]
top = [[20.0 + math.log(p), t] for t, p in enumerate([0.4, 0.3, 0.2, 0.1])]
scores = {"tolerance_ulps": 0.0, "contexts": {
sampling_check.context_key(prompt + [1] * depth): {"vocab": 4, "top": top}
for prompt in PROMPTS for depth in range(3)
}}
assert compare(sampling_check, tmp_path, ref, copy.deepcopy(ref), scores) == expected
# Even in joint mode, a token beyond top-k remains an error.
bad = copy.deepcopy(ref)
bad["settings"]["unrestricted"]["tokens"][0][0] = [3, 1, 1]
assert compare(sampling_check, tmp_path, ref, bad, scores) == 1
@pytest.mark.parametrize("joint", [False, True])
def test_cut_bounds_contain_supported_draws_under_rounding(sampling_check, joint):
rng = random.Random(178)
for _ in range(100):
logits = sorted([rng.uniform(16.0, 24.0) for _ in range(40)], reverse=True)
params = dict(TOP_K, top_k=8, top_p=0.7)
kept, possible = sampling_check.cut_bounds(
[[value, token] for token, value in enumerate(logits)], 40, params, 2.0, joint=joint
)
moved = [value + rng.uniform(-0.125, 0.125) for value in logits]
order = sorted(range(40), key=moved.__getitem__, reverse=True)
weights = {token: math.exp(moved[token] - max(moved)) for token in order}
total = sum(weights[token] for token in (order if joint else order[:8]))
above = 0.0
support = set()
for token in order[:8]:
if above / total < params["top_p"]:
support.add(token)
above += weights[token]
assert kept <= support <= possible
def test_self_control_passes_through_bf16_cut_flips(sampling_check, tmp_path):
ref, cand = sampled_run(TOP_K, start=1), sampled_run(TOP_K, start=2)
# The flips are real: per draw position, the two starts' supports differ, which failed the old per-token test.
first = [[row[0] for row in doc["settings"]["setting"]["tokens"][0]] for doc in (ref, cand)]
assert set(first[0]) != set(first[1])
assert sampling_check.homogeneity(*first)[0] < 1e-6
assert compare(sampling_check, tmp_path, ref, cand, fp32_scores(sampling_check, ref)) == 0
def test_truncated_setting_needs_fp32_scores(sampling_check, tmp_path):
ref = sampled_run(TOP_K, start=1)
assert compare(sampling_check, tmp_path, ref, copy.deepcopy(ref)) == 1
def test_sampler_that_ignores_the_cut_fails(sampling_check, tmp_path):
ref = sampled_run(TOP_K, start=1)
untruncated = dict(TOP_K, top_k=-1)
cand = sampled_run(TOP_K, start=2, sampler=lambda logits, rng: draw(processed(logits, untruncated), rng))
assert compare(sampling_check, tmp_path, ref, cand, fp32_scores(sampling_check, ref)) == 1
@pytest.mark.parametrize("biased", [False, True])
def test_biased_verify_residual_fails(sampling_check, tmp_path, biased):
"""An under-confident draft (the target at temperature 2): a correct chain verify keeps the target's
distribution, the mutated residual moves 6% of it (TV 0.063) off the top token."""
def verify(logits, rng):
p = processed(logits, TOP_K)
q = processed(logits, dict(TOP_K, temperature=2.0))
if biased:
return biased_chain_verify(p, q, rng)
y = draw(q, rng)
if rng.random() * q[y] < p[y]:
return y
residual = {t: max(p[t] - q.get(t, 0.0), 0.0) for t in p}
total = sum(residual.values())
return draw({t: v / total for t, v in residual.items()}, rng)
ref = sampled_run(TOP_K, start=1)
cand = sampled_run(TOP_K, start=2, sampler=verify)
assert compare(sampling_check, tmp_path, ref, cand, fp32_scores(sampling_check, ref)) == int(biased)
def test_ignored_temperature_fails(sampling_check, tmp_path):
ref = sampled_run(TEMPERATURE, start=1)
assert compare(sampling_check, tmp_path, ref, sampled_run(TEMPERATURE, start=2)) == 0
hot = dict(TEMPERATURE, temperature=1.0)
cand = sampled_run(TEMPERATURE, start=2, sampler=lambda logits, rng: draw(processed(logits, hot), rng))
assert compare(sampling_check, tmp_path, ref, cand) == 1