Official Inference Code of EAGLE-4
fix(pkg): stop sampling_check failing a no-spec server against itself
Seyfal Sultanov5bae2f883345
diff --git a/tests/test_sampling_check.py b/tests/test_sampling_check.pyindex 93e2cde..de09c22 100644--- a/tests/test_sampling_check.py+++ b/tests/test_sampling_check.py@@ -1,12 +1,15 @@# Copyright 2026 Eagle AI Corp.# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available-"""Serving distribution gates must reject missing or incomparable evidence."""+"""Serving distribution gates must reject missing or incomparable evidence, pass a correct sampler whose logits+move by bf16 rounding between runs, and fail a biased one."""import copyimport importlib.utilimport json+import mathfrom pathlib import Path+import randomimport subprocessimport sysfrom types import SimpleNamespace@@ -31,7 +34,7 @@ def artifact():"samples": 20,"settings": {"unrestricted": {- "params": {"temperature": 1.0, "top_p": 1.0, "top_k": -1},+ "params": {"temperature": 1.0, "top_p": 1.0, "top_k": -1, "min_p": 0.0},"tokens": [[[1, 2, 3]] * 20 for _ in range(4)],"seconds": 1.0,}@@ -40,16 +43,31 @@ def artifact():}-def compare(module, tmp_path, ref, candidate):+def compare(module, tmp_path, ref, candidate, scores=None, deterministic=False):reference = tmp_path / "reference.json"output = tmp_path / "candidate.json"reference.write_text(json.dumps(ref))output.write_text(json.dumps(candidate))- return module.compare(SimpleNamespace(reference=reference, candidate=output, alpha=0.01))+ score_path = None+ if scores is not None:+ score_path = tmp_path / "scores.json"+ score_path.write_text(json.dumps(scores))+ args = dict(reference=reference, candidate=output, alpha=0.01, scores=score_path, deterministic=deterministic)+ return module.compare(SimpleNamespace(**args))def test_unrestricted_setting_exercises_temperature_one(sampling_check):- assert sampling_check.SETTINGS["unrestricted"] == {"temperature": 1.0, "top_p": 1.0, "top_k": -1}+ assert sampling_check.SETTINGS["unrestricted"] == {+ "temperature": 1.0, "top_p": 1.0, "top_k": -1, "min_p": 0.0, "repetition_penalty": 1.0+ }+++def test_every_setting_names_each_filter(sampling_check):+ # A server fills an unnamed filter from the model's generation_config (Qwen3.8: top-k 20, top-p 0.95).+ for params in sampling_check.SETTINGS.values():+ assert params.keys() == sampling_check.NEUTRAL.keys()+ assert not sampling_check.truncates(sampling_check.SETTINGS["temperature"])+ assert not sampling_check.truncates(sampling_check.SETTINGS["penalty"])def test_complete_distribution_evidence_passes(sampling_check, tmp_path):@@ -57,7 +75,9 @@ def test_complete_distribution_evidence_passes(sampling_check, tmp_path):assert compare(sampling_check, tmp_path, ref, copy.deepcopy(ref)) == 0[email protected]("fault", ["empty", "missing", "refused", "parameters", "prompts", "samples", "tokens"])+ "fault", ["empty", "missing", "refused", "parameters", "prompts", "samples", "tokens", "unnamed"]+)def test_incomplete_distribution_evidence_fails(sampling_check, tmp_path, fault):ref = artifact()candidate = copy.deepcopy(ref)@@ -78,6 +98,9 @@ def test_incomplete_distribution_evidence_fails(sampling_check, tmp_path, fault)setting["tokens"][0].pop()elif fault == "tokens":setting["tokens"][0][0] = [1]+ elif fault == "unnamed":+ for doc in (ref, candidate):+ del doc["settings"]["unrestricted"]["params"]["top_p"]assert compare(sampling_check, tmp_path, ref, candidate) == 1@@ -88,6 +111,16 @@ def test_changed_distribution_fails(sampling_check, tmp_path):assert compare(sampling_check, tmp_path, ref, candidate) == 1[email protected]("deterministic", [False, True])+def test_seeds_are_judged_only_on_a_deterministic_server(sampling_check, tmp_path, deterministic):+ ref = artifact()+ ref["seeds"] = {str(s): [s, 1, 2] for s in range(32)}+ ref["seeds_reproduced"] = 32+ candidate = copy.deepcopy(ref)+ candidate["seeds_reproduced"] = 31+ assert compare(sampling_check, tmp_path, ref, candidate, deterministic=deterministic) == int(deterministic)++def test_compare_runs_without_the_http_client(tmp_path):# Only `run` talks HTTP (requests); the public test env has no requests.path = Path(__file__).parents[1] / "tools" / "sampling_check.py"@@ -97,7 +130,190 @@ def test_compare_runs_without_the_http_client(tmp_path):"import importlib.util, sys, types; sys.modules['requests'] = None; "f"spec = importlib.util.spec_from_file_location('sampling_check', {str(path)!r}); ""m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); "- f"sys.exit(m.compare(types.SimpleNamespace(reference={str(ref)!r}, candidate={str(ref)!r}, alpha=0.01)))"+ f"sys.exit(m.compare(types.SimpleNamespace(reference={str(ref)!r}, candidate={str(ref)!r}, alpha=0.01, "+ "scores=None, deterministic=False)))")out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)assert out.returncode == 0, out.stderr+++# A synthetic target: 40 tokens whose fp32 logits fall 0.45 (3.6 bf16 ulps at [16, 32)) per rank from 24, except that+# ranks 8 and 9 sit 0.04 apart, a near tie right at a top-8 cut. A server start moves every logit by up to a quarter+# bf16 ulp and the rank-8 one a further ulp up (odd starts) or down (even starts), so odd starts keep rank 8 and even+# starts rank 9, as a real Qwen3.8 top-p cut did between two no-spec vLLM starts (a token drawn 0 of 1428 times by one+# start and 115 of 1397 times by the other).+VOCAB, SCORED, SAMPLES = 40, 24, 2000+PROMPTS = [[101], [102], [103], [104]]+TOP_K = {"temperature": 1.0, "top_k": 8, "top_p": 1.0, "min_p": 0.0, "repetition_penalty": 1.0}+TEMPERATURE = {"temperature": 0.7, "top_k": -1, "top_p": 1.0, "min_p": 0.0, "repetition_penalty": 1.0}+++def fp32_logits(context):+ """{token: logit} of a context."""+ ranks = [24.0 - 0.45 * r for r in range(VOCAB)]+ ranks[8] = ranks[7] - 0.04+ order = list(range(VOCAB))+ random.Random(f"order {context}").shuffle(order)+ return {token: logit for token, logit in zip(order, ranks)}+++def served_logits(context, start):+ """The logits a server start computes: fp32 plus rounding (see above), within the fp32 arbiter's 2 ulps a pair."""+ noise = random.Random(f"start {start} {context}")+ logits = fp32_logits(context)+ eighth = sorted(logits, key=logits.get, reverse=True)[7]+ served = {}+ for token, value in logits.items():+ ulp = 2.0 ** (math.frexp(abs(value))[1] - 8)+ served[token] = value + noise.uniform(-0.25, 0.25) * ulp + (ulp if start % 2 else -ulp) * (token == eighth)+ return served+++def processed(logits, params):+ """The sampled distribution: temperature, then the top-k cut (no other filter is used here)."""+ order = sorted(logits, key=logits.get, reverse=True)+ if params["top_k"] > 0:+ order = order[: params["top_k"]]+ weights = [math.exp((logits[t] - logits[order[0]]) / params["temperature"]) for t in order]+ return {t: w / sum(weights) for t, w in zip(order, weights)}+++def draw(probs, rng):+ u, total = rng.random(), 0.0+ for token, p in probs.items():+ total += p+ if u < total:+ return token+ return token+++def biased_chain_verify(p, q, rng):+ """One draft token y ~ q, accepted with min(1, p_y / q_y); on rejection the classic mistake of subtracting only+ y's q mass from p (tests/test_sampled_tree.py's mutation) instead of drawing from norm(max(p - q, 0))."""+ y = draw(q, rng)+ if rng.random() * q[y] < p.get(y, 0.0):+ return y+ residual = dict(p)+ residual[y] = max(p.get(y, 0.0) - q[y], 0.0)+ total = sum(residual.values())+ return draw({t: v / total for t, v in residual.items()}, rng)+++def sampled_run(params, start, sampler=None, seed=0):+ """A run artifact: SAMPLES completions of 3 tokens per prompt, each token drawn by ``sampler(logits, rng)``+ from the start's logits (by default the processed distribution)."""+ rng = random.Random(f"draws {start} {seed}")+ sampler = sampler or (lambda logits, rng: draw(processed(logits, params), rng))+ tokens = []+ for prompt in PROMPTS:+ rows = []+ for _ in range(SAMPLES):+ context = list(prompt)+ for _ in range(3):+ context.append(sampler(served_logits(tuple(context), start), rng))+ rows.append(context[len(prompt) :])+ tokens.append(rows)+ return {+ "api": "vllm",+ "max_new_tokens": 3,+ "samples": SAMPLES,+ "prompt_ids": PROMPTS,+ "settings": {"setting": {"params": dict(params), "tokens": tokens, "seconds": 1.0}},+ "seeds": {},+ }+++def fp32_scores(module, reference):+ """What ``score`` writes for the reference: the top SCORED fp32 logits of every judged context."""+ contexts = {}+ for _, i, prefix in module.judged_contexts(reference):+ ids = reference["prompt_ids"][i] + list(prefix)+ logits = fp32_logits(tuple(ids))+ top = sorted(logits, key=logits.get, reverse=True)[:SCORED]+ contexts[module.context_key(ids)] = {"vocab": VOCAB, "top": [[logits[t], t] for t in top]}+ return {"tolerance_ulps": 2.0, "contexts": contexts}+++def test_cut_bounds_leave_near_ties_out(sampling_check):+ top = [[24.0 - 0.45 * r, r] for r in range(SCORED)]+ top[8][0] = top[7][0] - 0.04+ kept, possible = sampling_check.cut_bounds(top, VOCAB, TOP_K, 2.0)+ assert kept == set(range(7)) and possible == set(range(9))+ # A clear cut (3.6 ulps) is no near tie.+ kept, possible = sampling_check.cut_bounds(top, VOCAB, dict(TOP_K, top_k=5), 2.0)+ assert kept == possible == set(range(5))+ # top-p 0.6: shares 0.57 / 0.21 / 0.21, so whichever of the near-tied second and third ranks first is kept.+ flat = [[21.0, 0], [20.0, 1], [19.98, 2], [17.0, 3]] + [[10.0 - r, 4 + r] for r in range(SCORED - 4)]+ kept, possible = sampling_check.cut_bounds(flat, VOCAB, dict(TOP_K, top_k=-1, top_p=0.6), 2.0)+ assert kept == {0} and possible == {0, 1, 2}+ # min-p 0.05 at temperature 1: a token log(20) = 3.0 below the top, near the floor, is a near tie.+ edge = [[20.0, 0], [20.0 - math.log(20.0) + 0.01, 1], [15.0, 2]] + [[10.0 - r, 3 + r] for r in range(SCORED - 3)]+ kept, possible = sampling_check.cut_bounds(edge, VOCAB, dict(TOP_K, top_k=-1, min_p=0.05), 2.0)+ assert kept == {0} and possible == {0, 1}+ with pytest.raises(ValueError, match="past the 8 scored"):+ sampling_check.cut_bounds(top[:8], VOCAB, TOP_K, 2.0)+ with pytest.raises(ValueError, match="not modelled"):+ sampling_check.cut_bounds(top, VOCAB, dict(TOP_K, repetition_penalty=1.5), 2.0)+++def test_score_keeps_as_many_tokens_as_bound_the_cut(sampling_check):+ # min-p 0.05 on a flat distribution keeps every token within log(20) = 3.0 of the top: 300 here.+ pairs = [[20.0 - 0.01 * r, r] for r in range(1000)]+ min_p = dict(TOP_K, top_k=-1, min_p=0.05)+ top = sampling_check.bounded_top(pairs, 5000, [TOP_K, min_p], 2.0)+ assert len(top) == 1000 # 64 and 256 do not bound it+ kept, possible = sampling_check.cut_bounds(top, 5000, min_p, 2.0)+ # Near-ties: within one ulp (0.125) either side of the top and of the floor, i.e. 25 ranks around rank 300.+ assert kept == set(range(275)) and possible == set(range(325))+ assert len(sampling_check.bounded_top(pairs, 5000, [TOP_K], 2.0)) == 64+++def test_self_control_passes_through_bf16_cut_flips(sampling_check, tmp_path):+ ref, cand = sampled_run(TOP_K, start=1), sampled_run(TOP_K, start=2)+ # The flips are real: per draw position, the two starts' supports differ, which failed the old per-token test.+ first = [[row[0] for row in doc["settings"]["setting"]["tokens"][0]] for doc in (ref, cand)]+ assert set(first[0]) != set(first[1])+ assert sampling_check.homogeneity(*first)[0] < 1e-6+ assert compare(sampling_check, tmp_path, ref, cand, fp32_scores(sampling_check, ref)) == 0+++def test_truncated_setting_needs_fp32_scores(sampling_check, tmp_path):+ ref = sampled_run(TOP_K, start=1)+ assert compare(sampling_check, tmp_path, ref, copy.deepcopy(ref)) == 1+++def test_sampler_that_ignores_the_cut_fails(sampling_check, tmp_path):+ ref = sampled_run(TOP_K, start=1)+ untruncated = dict(TOP_K, top_k=-1)+ cand = sampled_run(TOP_K, start=2, sampler=lambda logits, rng: draw(processed(logits, untruncated), rng))+ assert compare(sampling_check, tmp_path, ref, cand, fp32_scores(sampling_check, ref)) == 1++[email protected]("biased", [False, True])+def test_biased_verify_residual_fails(sampling_check, tmp_path, biased):+ """An under-confident draft (the target at temperature 2): a correct chain verify keeps the target's+ distribution, the mutated residual moves 6% of it (TV 0.063) off the top token."""++ def verify(logits, rng):+ p = processed(logits, TOP_K)+ q = processed(logits, dict(TOP_K, temperature=2.0))+ if biased:+ return biased_chain_verify(p, q, rng)+ y = draw(q, rng)+ if rng.random() * q[y] < p[y]:+ return y+ residual = {t: max(p[t] - q.get(t, 0.0), 0.0) for t in p}+ total = sum(residual.values())+ return draw({t: v / total for t, v in residual.items()}, rng)++ ref = sampled_run(TOP_K, start=1)+ cand = sampled_run(TOP_K, start=2, sampler=verify)+ assert compare(sampling_check, tmp_path, ref, cand, fp32_scores(sampling_check, ref)) == int(biased)+++def test_ignored_temperature_fails(sampling_check, tmp_path):+ ref = sampled_run(TEMPERATURE, start=1)+ assert compare(sampling_check, tmp_path, ref, sampled_run(TEMPERATURE, start=2)) == 0+ hot = dict(TEMPERATURE, temperature=1.0)+ cand = sampled_run(TEMPERATURE, start=2, sampler=lambda logits, rng: draw(processed(logits, hot), rng))+ assert compare(sampling_check, tmp_path, ref, cand) == 1diff --git a/tools/sampling_check.py b/tools/sampling_check.pyindex 84bf353..7ac8ba3 100644--- a/tools/sampling_check.py+++ b/tools/sampling_check.py@@ -7,22 +7,36 @@``run`` sends ``--samples`` short completions of each of a few open-ended prompts to a running server under eachsampling setting (temperature, top-k with top-p, min-p, a repetition penalty), ``--concurrency`` requests at atime so they share decode rounds, and saves the output token ids. The first token comes from the prefill sampler;-every later one from a speculative verify round.+every later one from a speculative verify round. Every setting names temperature, top-k, top-p, min-p and the+repetition penalty: a server fills unnamed ones from the model's ``generation_config`` (Qwen3.8 sets top-k 20 and+top-p 0.95), which would truncate a setting meant to be untruncated.``compare`` tests, per setting and prompt, that the speculative server's tokens come from the same distribution as-the reference's: a two-sample chi-square test of homogeneity on the joint distribution of the first three tokens and-on the marginal of every position (cells seen fewer than 10 times in both runs together are pooled), at a-Bonferroni-corrected level ``--alpha / tests``. It also prints the total variation distance of each test. A server-that ignores a sampling parameter (for example, verifies greedily at temperature > 0) fails it by a wide margin.-The first token comes from the prefill, so a speculative server whose prefill runs other attention kernels can-differ there where a top-k or top-p cut falls between near-tied tokens; tokens 2 and 3 are therefore also tested-given the reference's most common prefix ("verify draws given the modal prefix"), which isolates the verify rounds.-Run the no-spec server twice to see the engine's own run-to-run level.+the reference's, with two-sample chi-square tests of homogeneity (cells seen fewer than 10 times in both runs+together are pooled) at a Bonferroni-corrected level ``--alpha / tests``, and prints the total variation distance+of each test. A server that ignores a sampling parameter (for example, verifies greedily at temperature > 0) fails+it by a wide margin.++An untruncated setting is tested on the joint distribution of the first three tokens, on the marginal of every+position, and on tokens 2 and 3 given the reference's most common prefix ("verify draws given the modal prefix",+which isolates the verify rounds).++A truncated setting (top-k, top-p or min-p) is tested only at fixed contexts: token 1 given the prompt, and tokens+2 and 3 given the reference's most common prefix. A truncation cut is discontinuous in the logits: a token whose+logit sits within bf16 rounding of the cut is kept by one server start or batch shape and dropped by another, and+its whole probability moves with it, so two correct no-spec servers differ there. ``score`` re-forwards each of+these contexts with the unmodified target in float32 (``tools/fp32_arbiter.py``'s reference model, on a GPU) and+``compare --scores`` splits the vocabulary with the fp32 arbiter's tolerance (each logit may move by one bf16 ulp+at its magnitude, so a pair's gap by two): tokens every such logit keeps, near-ties at the cut, and tokens no such+logit keeps. Truncation renormalizes the kept tokens proportionally, so draws among the always-kept tokens follow+the same distribution whichever near-ties a run kept; the test runs on those draws. A draw of a token no tolerated+logit keeps fails the setting outright (a sampler that ignores the cut).With ``--seeds S`` (``run``) every one of S seeds is sent twice, in batches of a different composition, attemperature 0.7 for ``--seed-tokens`` tokens; ``compare`` (and ``run`` itself) report how many seeds reproduced-their text, and ``compare`` how many seeded texts equal the reference server's for the same seed. sglang honors-seeds only with ``--enable-deterministic-inference``.+their text, and ``compare`` how many seeded texts equal the reference server's for the same seed. A seed reproduces+across batches only on a batch-invariant server (sglang with ``--enable-deterministic-inference``), so ``compare``+requires every seed to reproduce only with ``--deterministic``.With ``--speed-data`` (a SPEED-Bench jsonl) ``run`` also decodes the first user turn of ``--speed-prompts``conversations one at a time (batch size 1, chat template, thinking off) at temperature 0 and at temperature 1, and@@ -30,21 +44,27 @@ conversations one at a time (batch size 1, chat template, thinking off) at tempepython tools/sampling_check.py run --api vllm --port P --tokenizer <model dir> --out nospec.json --seeds 32python tools/sampling_check.py run --api vllm --port P --tokenizer <model dir> --out spec.json --seeds 32- python tools/sampling_check.py compare nospec.json spec.json+ python tools/sampling_check.py score --model <model dir> nospec.json --out scores.json+ python tools/sampling_check.py compare nospec.json spec.json --scores scores.json-Exit code 1 (``compare``) when a test rejects at the corrected level, or a seed did not reproduce.+Exit code 1 (``compare``) when a test rejects at the corrected level, a draw falls outside every tolerated cut, or+(``--deterministic``) a seed did not reproduce."""from __future__ import annotationsimport argparse+import bisectimport collectionsimport concurrent.futures+import importlib.util+import itertoolsimport jsonimport mathimport sysimport time-from typing import Dict, List, Optional, Sequence, Tuple+from pathlib import Path+from typing import Dict, List, Optional, Sequence, Set, TuplePROMPTS = ("Here is a list of ten random animals:",@@ -52,14 +72,16 @@ PROMPTS = ("My favorite word is","The weather today is",)+# Every setting names each filter: a server fills unnamed ones from the model's generation_config.+NEUTRAL = {"temperature": 1.0, "top_k": -1, "top_p": 1.0, "min_p": 0.0, "repetition_penalty": 1.0}SETTINGS: Dict[str, dict] = {- "unrestricted": {"temperature": 1.0, "top_p": 1.0, "top_k": -1},- "temperature": {"temperature": 0.7},- "top_k_top_p": {"temperature": 1.0, "top_k": 8, "top_p": 0.7},- "min_p": {"temperature": 1.0, "min_p": 0.05},- "penalty": {"temperature": 0.8, "repetition_penalty": 1.5},+ "unrestricted": dict(NEUTRAL),+ "temperature": {**NEUTRAL, "temperature": 0.7},+ "top_k_top_p": {**NEUTRAL, "top_k": 8, "top_p": 0.7},+ "min_p": {**NEUTRAL, "min_p": 0.05},+ "penalty": {**NEUTRAL, "temperature": 0.8, "repetition_penalty": 1.5},}-SEED_SETTING = {"temperature": 0.7}+SEED_SETTING = SETTINGS["temperature"]JOINT = 3 # tokens in the joint testPOOL = 10 # cells seen fewer times than this in both runs together are pooled@@ -219,6 +241,8 @@ def homogeneity(ref: Sequence, test: Sequence) -> Tuple[float, float, int]:"""Two-sample chi-square test of homogeneity: (p value, total variation distance, cells after pooling)."""a, b = collections.Counter(ref), collections.Counter(test)na, nb = len(ref), len(test)+ if not na or not nb:+ return 1.0, 0.0, 0tv = 0.5 * sum(abs(a[c] / na - b[c] / nb) for c in set(a) | set(b))cells: List[Tuple[int, int]] = []rest = [0, 0]@@ -241,6 +265,158 @@ def homogeneity(ref: Sequence, test: Sequence) -> Tuple[float, float, int]:return chi2_sf(stat, len(cells) - 1), tv, len(cells)+TRUNCATION = ("top_k", "top_p", "min_p")+++def truncates(params: dict) -> bool:+ """Whether the setting cuts the vocabulary (top-k, top-p or min-p)."""+ return params["top_k"] not in (-1, 0) or params["top_p"] < 1.0 or params["min_p"] > 0.0+++def _sibling(name: str):+ """Another tool of this directory, imported by path (tools/ is not a package)."""+ if name not in sys.modules:+ spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(f"{name}.py"))+ assert spec is not None and spec.loader is not None+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module # its dataclasses resolve their annotations through sys.modules+ spec.loader.exec_module(module)+ return sys.modules[name]+++def cut_bounds(top: Sequence[Sequence[float]], vocab: int, params: dict, tolerance_ulps: float) -> Tuple[Set, Set]:+ """(tokens every tolerated logit keeps, tokens some tolerated logit keeps) under the setting's cut.++ ``top``: fp32 ``[logit, token]`` pairs, best first; the ``vocab - len(top)`` other tokens are at most the last+ logit. Each logit may move by ``tolerance_ulps / 2`` bf16 ulps at its magnitude, so the gap of a pair moves by up+ to ``tolerance_ulps`` (the fp32 arbiter's rule). The cut is top-k, then top-p on the renormalized top-k+ distribution at the setting's temperature, or min-p alone (vLLM's and sglang's order)."""+ ulp = _sibling("fp32_arbiter").bf16_ulp+ temperature, k, top_p, min_p = params["temperature"], params["top_k"], params["top_p"], params["min_p"]+ k = None if k in (-1, 0) else k+ if min_p > 0.0 and (k is not None or top_p < 1.0):+ raise ValueError("min-p together with top-k or top-p is not modelled")+ if params.get("repetition_penalty", 1.0) != 1.0:+ raise ValueError("a repetition penalty together with a cut is not modelled")+ tokens = [int(t) for _, t in top]+ lo = [v - tolerance_ulps / 2 * ulp(abs(v)) for v, _ in top]+ hi = [v + tolerance_ulps / 2 * ulp(abs(v)) for v, _ in top]+ # The unscored tokens as one entry of multiplicity ``rest``, anywhere at or below the last scored logit.+ rest = vocab - len(top)+ lo.append(-math.inf)+ hi.append(hi[-1])+ count = [1] * len(tokens) + [rest]+ n = len(count)+ shift = max(hi)++ def mass(x: float) -> float:+ return math.exp((x - shift) / temperature)++ def counter(bounds: List[float]):+ """Tokens whose bound is above x, by bisection (the scored set can run to thousands of tokens)."""+ pairs = sorted(zip(bounds, count))+ keys = [b for b, _ in pairs]+ tail = list(itertools.accumulate([c for _, c in pairs][::-1]))[::-1] + [0]+ return lambda x: tail[bisect.bisect_right(keys, x)]++ above_hi, above_lo = counter(hi), counter(lo)++ # In the top k for every tolerated logit (fewer than k others can reach it) / for some (fewer than k others are+ # surely above it). Only these tokens enter the top-p sums.+ in_k = [k is None or above_hi(lo[i]) - count[i] < k for i in range(n)]+ maybe_k = [count[i] > 0 and (k is None or above_lo(hi[i]) < k) for i in range(n)]+ pool = [j for j in range(n) if maybe_k[j]]+ by_hi = sorted(range(n), key=lambda j: hi[j], reverse=True)[:2] # the two largest bounds of each side, for min-p+ by_lo = sorted(range(n), key=lambda j: lo[j], reverse=True)[:2]+ kept, possible = [], []+ for i in range(n):+ sure, maybe = in_k[i], maybe_k[i]+ if top_p < 1.0 and maybe:+ # Kept when the renormalized mass of the tokens above it is below top_p: bound that share both ways.+ others = [j for j in pool if j != i]+ above_max = sum(count[j] * mass(hi[j]) for j in others if hi[j] > lo[i])+ below_min = mass(lo[i]) + sum(count[j] * mass(lo[j]) for j in others if hi[j] < lo[i] and in_k[j])+ sure = sure and above_max / (above_max + below_min) < top_p+ above_min = sum(count[j] * mass(lo[j]) for j in others if lo[j] > hi[i] and in_k[j])+ rest_max = count[i] * mass(hi[i]) + sum(count[j] * mass(hi[j]) for j in others if lo[j] <= hi[i])+ maybe = above_min / (above_min + rest_max) < top_p+ if min_p > 0.0:+ # Kept when its probability is at least min_p times the largest.+ floor = temperature * math.log(min_p)+ others_hi = hi[by_hi[1] if by_hi[0] == i else by_hi[0]]+ others_lo = lo[by_lo[1] if by_lo[0] == i else by_lo[0]]+ sure = sure and lo[i] - max(lo[i], others_hi) >= floor+ maybe = maybe and hi[i] - max(hi[i], others_lo) >= floor+ kept.append(sure and i < len(tokens))+ possible.append(maybe)+ if possible[-1]:+ raise ValueError(f"the cut may keep tokens past the {len(tokens)} scored ones")+ return ({t for t, keep in zip(tokens, kept) if keep}, {t for t, keep in zip(tokens, possible) if keep})+++def modal_prefix(tokens: Sequence[Sequence[int]], depth: int) -> Tuple[int, ...]:+ """The most common first ``depth`` tokens (the first seen among equally common ones)."""+ return collections.Counter(tuple(t[:depth]) for t in tokens).most_common(1)[0][0]+++def judged_contexts(artifact: dict):+ """(setting, prompt, prefix) of every context a truncated setting of this reference is judged at: the prompt,+ then the reference's most common prefix of each length below ``JOINT``."""+ for name, entry in artifact["settings"].items():+ if "refused" in entry or not truncates(entry["params"]):+ continue+ for i, tokens in enumerate(entry["tokens"]):+ for depth in range(JOINT):+ yield name, i, modal_prefix(tokens, depth)+++def context_key(ids: Sequence[int]) -> str:+ return ",".join(map(str, ids))+++def bounded_top(pairs: Sequence[Sequence[float]], vocab: int, settings: Sequence[dict], tolerance_ulps: float) -> list:+ """The shortest head of ``pairs`` (fp32 ``[logit, token]``, best first) that bounds every setting's cut."""+ size = min(64, len(pairs))+ while True:+ try:+ for params in settings:+ cut_bounds(pairs[:size], vocab, params, tolerance_ulps)+ return list(pairs[:size])+ except ValueError:+ if size == len(pairs):+ raise+ size = min(4 * size, len(pairs))+++def score(args) -> int:+ """fp32 logits of every judged context of the references: as many of the best as bound each setting's cut,+ and the vocabulary size."""+ import torch++ fp32 = _sibling("fp32_arbiter")+ judged: Dict[str, Tuple[list, list]] = {} # context key: (ids, params of the settings judged there)+ for path in args.references:+ with open(path, encoding="utf-8") as f:+ artifact = json.load(f)+ for name, i, prefix in judged_contexts(artifact):+ ids = list(artifact["prompt_ids"][i]) + list(prefix)+ judged.setdefault(context_key(ids), (ids, []))[1].append(artifact["settings"][name]["params"])+ model, seconds = fp32.load_reference_model(args.model)+ print(f"fp32 reference loaded in {seconds:.0f} s; {len(judged)} contexts", flush=True)+ contexts: Dict[str, dict] = {}+ for key, (ids, settings) in judged.items():+ batch = torch.tensor([ids], dtype=torch.long, device="cuda")+ with torch.inference_mode():+ out = model(input_ids=batch, attention_mask=torch.ones_like(batch), use_cache=False, logits_to_keep=1)+ logits = out.logits[0, -1].double()+ values, indices = torch.sort(logits, descending=True)+ pairs = [[v, int(t)] for v, t in zip(values.tolist(), indices.tolist())]+ top = bounded_top(pairs, logits.numel(), settings, fp32.TOLERANCE_ULPS)+ contexts[key] = {"vocab": logits.numel(), "top": top}+ print(f"context {key}: {len(top)} tokens scored", flush=True)+ save(args.out, {"model": args.model, "tolerance_ulps": fp32.TOLERANCE_ULPS, "contexts": contexts})+ return 0+def validate_pair(ref: dict, cand: dict) -> None:"""Reject partial evidence before a statistical test can silently omit it."""if not ref["settings"] or ref["settings"].keys() != cand["settings"].keys():@@ -256,6 +432,9 @@ def validate_pair(ref: dict, cand: dict) -> None:raise ValueError(f"{name}: refused setting is not distribution evidence")if r["params"] != c["params"]:raise ValueError(f"{name}: sampling parameters differ")+ unnamed = [key for key in ("temperature", *TRUNCATION) if key not in r["params"]]+ if unnamed:+ raise ValueError(f"{name}: {', '.join(unnamed)} left to the server's defaults (the model's may truncate)")if len(r["tokens"]) != len(PROMPTS) or len(c["tokens"]) != len(PROMPTS):raise ValueError(f"{name}: missing prompt results")for r_tokens, c_tokens in zip(r["tokens"], c["tokens"]):@@ -268,6 +447,57 @@ def validate_pair(ref: dict, cand: dict) -> None:raise ValueError(f"{name}: incomplete output token sequence")+def mixed_tests(name: str, prompt: int, r_tokens: list, c_tokens: list) -> list:+ """An untruncated setting's tests: the joint first tokens, every position's marginal, and tokens 2 and 3 given+ the reference's most common prefix (the verify round's own draws, free of any difference in the prefill's)."""+ tests = []+ r_joint = [tuple(t[:JOINT]) for t in r_tokens]+ c_joint = [tuple(t[:JOINT]) for t in c_tokens]+ tests.append((name, prompt, f"joint 1-{JOINT}", *homogeneity(r_joint, c_joint)))+ for pos in range(min(len(r_tokens[0]), len(c_tokens[0]))):+ marginals = [t[pos] for t in r_tokens], [t[pos] for t in c_tokens]+ tests.append((name, prompt, f"token {pos + 1}", *homogeneity(*marginals)))+ for pos in (1, 2):+ prefix = modal_prefix(r_tokens, pos)+ r_next = [t[pos] for t in r_tokens if tuple(t[:pos]) == prefix]+ c_next = [t[pos] for t in c_tokens if tuple(t[:pos]) == prefix]+ if r_next and c_next:+ tests.append((name, prompt, f"token {pos + 1} | modal prefix", *homogeneity(r_next, c_next)))+ return tests+++def cut_tests(+ name: str,+ params: dict,+ prompt: int,+ prompt_ids: list,+ r_tokens: list,+ c_tokens: list,+ scores: dict,+ tally: List[int],+ outside: list,+) -> list:+ """A truncated setting's tests at its judged contexts, on the draws among the tokens every tolerated logit keeps.+ Counts near-tie and outside draws into ``tally`` and lists the outside tokens in ``outside``."""+ tests = []+ for depth in range(JOINT):+ prefix = modal_prefix(r_tokens, depth)+ context = scores["contexts"].get(context_key(list(prompt_ids) + list(prefix)))+ if context is None:+ raise ValueError(f"{name}: prompt {prompt} prefix {list(prefix)} was not scored")+ kept, possible = cut_bounds(context["top"], context["vocab"], params, scores["tolerance_ulps"])+ draws = [[t[depth] for t in tokens if tuple(t[:depth]) == prefix] for tokens in (r_tokens, c_tokens)]+ what = "token 1" if depth == 0 else f"token {depth + 1} | modal prefix"+ tests.append((name, prompt, what, *homogeneity(*[[t for t in d if t in kept] for d in draws])))+ for slot, d in enumerate(draws):+ tally[slot] += sum(t in possible and t not in kept for t in d)+ tally[2 + slot] += sum(t not in possible for t in d)+ stray = [sorted({t for t in d if t not in possible}) for d in draws]+ if stray[0] or stray[1]:+ outside.append((name, prompt, what, *stray))+ return tests++def compare(args) -> int:with open(args.reference, encoding="utf-8") as f:ref = json.load(f)@@ -278,25 +508,28 @@ def compare(args) -> int:except (KeyError, TypeError, ValueError) as error:print(f"INVALID distribution evidence: {error}")return 1+ scores = None+ if args.scores:+ with open(args.scores, encoding="utf-8") as f:+ scores = json.load(f)tests = [] # (setting, prompt, what, p, tv, cells)- for name, entry in cand["settings"].items():- for i, (r_tokens, c_tokens) in enumerate(zip(ref["settings"][name]["tokens"], entry["tokens"])):- k = min(len(r_tokens[0]), len(c_tokens[0]))- r_joint = [tuple(t[:JOINT]) for t in r_tokens]- c_joint = [tuple(t[:JOINT]) for t in c_tokens]- tests.append((name, i, f"joint 1-{JOINT}", *homogeneity(r_joint, c_joint)))- for pos in range(k):- tests.append(- (name, i, f"token {pos + 1}", *homogeneity([t[pos] for t in r_tokens], [t[pos] for t in c_tokens]))- )- # Tokens 2 and 3 given the reference's most common prefix: the verify round's own draws, free of any- # difference in the prefill's sampling (other attention kernels move near-tied top-k / top-p cuts).- for pos in (1, 2):- prefix = collections.Counter(tuple(t[:pos]) for t in r_tokens).most_common(1)[0][0]- r_next = [t[pos] for t in r_tokens if tuple(t[:pos]) == prefix]- c_next = [t[pos] for t in c_tokens if tuple(t[:pos]) == prefix]- if r_next and c_next:- tests.append((name, i, f"token {pos + 1} | modal prefix", *homogeneity(r_next, c_next)))+ cuts: Dict[str, List[int]] = {} # setting: draws [ref near-tie, candidate near-tie, ref outside, candidate outside]+ outside: list = [] # (setting, prompt, what, reference tokens, candidate tokens) outside every tolerated cut+ try:+ for name, entry in cand["settings"].items():+ params = entry["params"]+ if truncates(params) and scores is None:+ raise ValueError(f"{name} truncates: judge it with --scores (see score)")+ for i, (r_tokens, c_tokens) in enumerate(zip(ref["settings"][name]["tokens"], entry["tokens"])):+ if truncates(params):+ tally = cuts.setdefault(name, [0, 0, 0, 0])+ prompt_ids = ref["prompt_ids"][i]+ tests += cut_tests(name, params, i, prompt_ids, r_tokens, c_tokens, scores, tally, outside)+ else:+ tests += mixed_tests(name, i, r_tokens, c_tokens)+ except (KeyError, ValueError) as error: # unscored contexts, or a cut the scores cannot bound+ print(f"INVALID distribution evidence: {error}")+ return 1level = args.alpha / max(1, len(tests))failed = Falsefor name in cand["settings"]:@@ -304,17 +537,25 @@ def compare(args) -> int:if not rows:continueworst = min(rows, key=lambda t: t[3])- joint_tv = max(t[4] for t in rows if t[2].startswith("joint"))+ joint = [t[4] for t in rows if t[2].startswith("joint")]bad = [t for t in rows if t[3] < level]+ stray = [t for t in outside if t[0] == name]conditional = [t for t in rows if "modal prefix" in t[2]]cond_bad = [t for t in conditional if t[3] < level]- failed |= bool(bad)+ failed |= bool(bad) or bool(stray)extra = cand["settings"][name].get("mean_tokens_per_verify") or cand["settings"][name].get("spec_counters", "")print(- f"{name:12s} {'FAIL' if bad else 'PASS'}: {len(rows)} tests, {len(bad)} below {level:.1e}; min p "- f"{worst[3]:.2e} ({worst[2]}, prompt {worst[1]}, TV {worst[4]:.3f}); max joint TV {joint_tv:.3f}; "+ f"{name:12s} {'FAIL' if bad or stray else 'PASS'}: {len(rows)} tests, {len(bad)} below {level:.1e}; min p "+ f"{worst[3]:.2e} ({worst[2]}, prompt {worst[1]}, TV {worst[4]:.3f})"+ f"{f'; max joint TV {max(joint):.3f}' if joint else ''}; "f"{cand['settings'][name]['seconds']} s (ref {ref['settings'][name]['seconds']} s) {extra}")+ if name in cuts:+ tally = cuts[name]+ print(+ f" fp32 cut ({scores['tolerance_ulps']:g} bf16 ulps): near-tie draws left out {tally[0]} ref / "+ f"{tally[1]} candidate; draws outside every tolerated cut {tally[2]} ref / {tally[3]} candidate"+ )cond_worst = min(conditional, key=lambda t: t[3]) if conditional else Noneif cond_worst:print(@@ -323,6 +564,8 @@ def compare(args) -> int:)for t in sorted(bad, key=lambda t: t[3])[:5]:print(f" {t[2]}, prompt {t[1]}: p {t[3]:.2e}, TV {t[4]:.3f}, {t[5]} cells")+ for t in stray[:5]:+ print(f" {t[2]}, prompt {t[1]}: tokens outside every tolerated cut: ref {t[3]}, candidate {t[4]}")for temperature, entry in cand.get("speed", {}).items():ref_entry = ref.get("speed", {}).get(temperature, {})line = f"speed at temperature {temperature}: {entry['tok_s']} tok/s (reference {ref_entry.get('tok_s')})"@@ -339,11 +582,14 @@ def compare(args) -> int:for s, ids in seeds.items()if s in ref.get("seeds", {})]+ # A seed reproduces its text in another batch only on a batch-invariant server (vLLM's default is not: its+ # no-spec server reproduced 24 to 32 of 32 by chance), so only --deterministic makes it a gate.print(f"seeds: {reproduced}/{len(seeds)} reproduced their text; {same_as_ref}/{len(seeds)} equal the "f"reference's text for the same seed (mean common prefix {sum(prefix) / max(1, len(prefix)):.1f} tokens)"+ f"{'' if args.deterministic else '; not judged (no --deterministic)'}")- failed |= reproduced != len(seeds)+ failed |= args.deterministic and reproduced != len(seeds)return 1 if failed else 0@@ -364,12 +610,18 @@ def main(argv: Optional[Sequence[str]] = None) -> int:r.add_argument("--speed-data", help="SPEED-Bench jsonl: also time batch-size-1 decoding")r.add_argument("--speed-prompts", type=int, default=8)r.add_argument("--speed-tokens", type=int, default=512)+ s = sub.add_parser("score", help="fp32 logits of the truncated settings' judged contexts (a GPU and the model)")+ s.add_argument("--model", required=True)+ s.add_argument("references", nargs="+", help="reference run outputs (their modal prefixes are judged)")+ s.add_argument("--out", required=True)c = sub.add_parser("compare")c.add_argument("reference")c.add_argument("candidate")c.add_argument("--alpha", type=float, default=0.01, help="family-wise level, Bonferroni-corrected")+ c.add_argument("--scores", help="score's output for the reference: required by truncated settings")+ c.add_argument("--deterministic", action="store_true", help="the servers are batch-invariant: seeds must reproduce")args = parser.parse_args(argv)- return run(args) if args.cmd == "run" else compare(args)+ return {"run": run, "score": score, "compare": compare}[args.cmd](args)if __name__ == "__main__":