Skip to content

Official Inference Code of EAGLE-4

fix(pkg): stop sampling_check failing a no-spec server against itself

Seyfal Sultanov5bae2f883345
diff --git a/tests/test_sampling_check.py b/tests/test_sampling_check.py
index 93e2cde..de09c22 100644
--- a/tests/test_sampling_check.py
+++ b/tests/test_sampling_check.py
@@ -1,12 +1,15 @@
# Copyright 2026 Eagle AI Corp.
# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available
-"""Serving distribution gates must reject missing or incomparable evidence."""
+"""Serving distribution gates must reject missing or incomparable evidence, pass a correct sampler whose logits
+move by bf16 rounding between runs, and fail a biased one."""
import copy
import importlib.util
import json
+import math
from pathlib import Path
+import random
import subprocess
import sys
from types import SimpleNamespace
@@ -31,7 +34,7 @@ def artifact():
"samples": 20,
"settings": {
"unrestricted": {
- "params": {"temperature": 1.0, "top_p": 1.0, "top_k": -1},
+ "params": {"temperature": 1.0, "top_p": 1.0, "top_k": -1, "min_p": 0.0},
"tokens": [[[1, 2, 3]] * 20 for _ in range(4)],
"seconds": 1.0,
}
@@ -40,16 +43,31 @@ def artifact():
}
-def compare(module, tmp_path, ref, candidate):
+def compare(module, tmp_path, ref, candidate, scores=None, deterministic=False):
reference = tmp_path / "reference.json"
output = tmp_path / "candidate.json"
reference.write_text(json.dumps(ref))
output.write_text(json.dumps(candidate))
- return module.compare(SimpleNamespace(reference=reference, candidate=output, alpha=0.01))
+ score_path = None
+ if scores is not None:
+ score_path = tmp_path / "scores.json"
+ score_path.write_text(json.dumps(scores))
+ args = dict(reference=reference, candidate=output, alpha=0.01, scores=score_path, deterministic=deterministic)
+ return module.compare(SimpleNamespace(**args))
def test_unrestricted_setting_exercises_temperature_one(sampling_check):
- assert sampling_check.SETTINGS["unrestricted"] == {"temperature": 1.0, "top_p": 1.0, "top_k": -1}
+ assert sampling_check.SETTINGS["unrestricted"] == {
+ "temperature": 1.0, "top_p": 1.0, "top_k": -1, "min_p": 0.0, "repetition_penalty": 1.0
+ }
+
+
+def test_every_setting_names_each_filter(sampling_check):
+ # A server fills an unnamed filter from the model's generation_config (Qwen3.8: top-k 20, top-p 0.95).
+ for params in sampling_check.SETTINGS.values():
+ assert params.keys() == sampling_check.NEUTRAL.keys()
+ assert not sampling_check.truncates(sampling_check.SETTINGS["temperature"])
+ assert not sampling_check.truncates(sampling_check.SETTINGS["penalty"])
def test_complete_distribution_evidence_passes(sampling_check, tmp_path):
@@ -57,7 +75,9 @@ def test_complete_distribution_evidence_passes(sampling_check, tmp_path):
assert compare(sampling_check, tmp_path, ref, copy.deepcopy(ref)) == 0
[email protected]("fault", ["empty", "missing", "refused", "parameters", "prompts", "samples", "tokens"])
+ "fault", ["empty", "missing", "refused", "parameters", "prompts", "samples", "tokens", "unnamed"]
+)
def test_incomplete_distribution_evidence_fails(sampling_check, tmp_path, fault):
ref = artifact()
candidate = copy.deepcopy(ref)
@@ -78,6 +98,9 @@ def test_incomplete_distribution_evidence_fails(sampling_check, tmp_path, fault)
setting["tokens"][0].pop()
elif fault == "tokens":
setting["tokens"][0][0] = [1]
+ elif fault == "unnamed":
+ for doc in (ref, candidate):
+ del doc["settings"]["unrestricted"]["params"]["top_p"]
assert compare(sampling_check, tmp_path, ref, candidate) == 1
@@ -88,6 +111,16 @@ def test_changed_distribution_fails(sampling_check, tmp_path):
assert compare(sampling_check, tmp_path, ref, candidate) == 1
[email protected]("deterministic", [False, True])
+def test_seeds_are_judged_only_on_a_deterministic_server(sampling_check, tmp_path, deterministic):
+ ref = artifact()
+ ref["seeds"] = {str(s): [s, 1, 2] for s in range(32)}
+ ref["seeds_reproduced"] = 32
+ candidate = copy.deepcopy(ref)
+ candidate["seeds_reproduced"] = 31
+ assert compare(sampling_check, tmp_path, ref, candidate, deterministic=deterministic) == int(deterministic)
+
+
def test_compare_runs_without_the_http_client(tmp_path):
# Only `run` talks HTTP (requests); the public test env has no requests.
path = Path(__file__).parents[1] / "tools" / "sampling_check.py"
@@ -97,7 +130,190 @@ def test_compare_runs_without_the_http_client(tmp_path):
"import importlib.util, sys, types; sys.modules['requests'] = None; "
f"spec = importlib.util.spec_from_file_location('sampling_check', {str(path)!r}); "
"m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); "
- f"sys.exit(m.compare(types.SimpleNamespace(reference={str(ref)!r}, candidate={str(ref)!r}, alpha=0.01)))"
+ f"sys.exit(m.compare(types.SimpleNamespace(reference={str(ref)!r}, candidate={str(ref)!r}, alpha=0.01, "
+ "scores=None, deterministic=False)))"
)
out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
assert out.returncode == 0, out.stderr
+
+
+# A synthetic target: 40 tokens whose fp32 logits fall 0.45 (3.6 bf16 ulps at [16, 32)) per rank from 24, except that
+# ranks 8 and 9 sit 0.04 apart, a near tie right at a top-8 cut. A server start moves every logit by up to a quarter
+# bf16 ulp and the rank-8 one a further ulp up (odd starts) or down (even starts), so odd starts keep rank 8 and even
+# starts rank 9, as a real Qwen3.8 top-p cut did between two no-spec vLLM starts (a token drawn 0 of 1428 times by one
+# start and 115 of 1397 times by the other).
+VOCAB, SCORED, SAMPLES = 40, 24, 2000
+PROMPTS = [[101], [102], [103], [104]]
+TOP_K = {"temperature": 1.0, "top_k": 8, "top_p": 1.0, "min_p": 0.0, "repetition_penalty": 1.0}
+TEMPERATURE = {"temperature": 0.7, "top_k": -1, "top_p": 1.0, "min_p": 0.0, "repetition_penalty": 1.0}
+
+
+def fp32_logits(context):
+ """{token: logit} of a context."""
+ ranks = [24.0 - 0.45 * r for r in range(VOCAB)]
+ ranks[8] = ranks[7] - 0.04
+ order = list(range(VOCAB))
+ random.Random(f"order {context}").shuffle(order)
+ return {token: logit for token, logit in zip(order, ranks)}
+
+
+def served_logits(context, start):
+ """The logits a server start computes: fp32 plus rounding (see above), within the fp32 arbiter's 2 ulps a pair."""
+ noise = random.Random(f"start {start} {context}")
+ logits = fp32_logits(context)
+ eighth = sorted(logits, key=logits.get, reverse=True)[7]
+ served = {}
+ for token, value in logits.items():
+ ulp = 2.0 ** (math.frexp(abs(value))[1] - 8)
+ served[token] = value + noise.uniform(-0.25, 0.25) * ulp + (ulp if start % 2 else -ulp) * (token == eighth)
+ return served
+
+
+def processed(logits, params):
+ """The sampled distribution: temperature, then the top-k cut (no other filter is used here)."""
+ order = sorted(logits, key=logits.get, reverse=True)
+ if params["top_k"] > 0:
+ order = order[: params["top_k"]]
+ weights = [math.exp((logits[t] - logits[order[0]]) / params["temperature"]) for t in order]
+ return {t: w / sum(weights) for t, w in zip(order, weights)}
+
+
+def draw(probs, rng):
+ u, total = rng.random(), 0.0
+ for token, p in probs.items():
+ total += p
+ if u < total:
+ return token
+ return token
+
+
+def biased_chain_verify(p, q, rng):
+ """One draft token y ~ q, accepted with min(1, p_y / q_y); on rejection the classic mistake of subtracting only
+ y's q mass from p (tests/test_sampled_tree.py's mutation) instead of drawing from norm(max(p - q, 0))."""
+ y = draw(q, rng)
+ if rng.random() * q[y] < p.get(y, 0.0):
+ return y
+ residual = dict(p)
+ residual[y] = max(p.get(y, 0.0) - q[y], 0.0)
+ total = sum(residual.values())
+ return draw({t: v / total for t, v in residual.items()}, rng)
+
+
+def sampled_run(params, start, sampler=None, seed=0):
+ """A run artifact: SAMPLES completions of 3 tokens per prompt, each token drawn by ``sampler(logits, rng)``
+ from the start's logits (by default the processed distribution)."""
+ rng = random.Random(f"draws {start} {seed}")
+ sampler = sampler or (lambda logits, rng: draw(processed(logits, params), rng))
+ tokens = []
+ for prompt in PROMPTS:
+ rows = []
+ for _ in range(SAMPLES):
+ context = list(prompt)
+ for _ in range(3):
+ context.append(sampler(served_logits(tuple(context), start), rng))
+ rows.append(context[len(prompt) :])
+ tokens.append(rows)
+ return {
+ "api": "vllm",
+ "max_new_tokens": 3,
+ "samples": SAMPLES,
+ "prompt_ids": PROMPTS,
+ "settings": {"setting": {"params": dict(params), "tokens": tokens, "seconds": 1.0}},
+ "seeds": {},
+ }
+
+
+def fp32_scores(module, reference):
+ """What ``score`` writes for the reference: the top SCORED fp32 logits of every judged context."""
+ contexts = {}
+ for _, i, prefix in module.judged_contexts(reference):
+ ids = reference["prompt_ids"][i] + list(prefix)
+ logits = fp32_logits(tuple(ids))
+ top = sorted(logits, key=logits.get, reverse=True)[:SCORED]
+ contexts[module.context_key(ids)] = {"vocab": VOCAB, "top": [[logits[t], t] for t in top]}
+ return {"tolerance_ulps": 2.0, "contexts": contexts}
+
+
+def test_cut_bounds_leave_near_ties_out(sampling_check):
+ top = [[24.0 - 0.45 * r, r] for r in range(SCORED)]
+ top[8][0] = top[7][0] - 0.04
+ kept, possible = sampling_check.cut_bounds(top, VOCAB, TOP_K, 2.0)
+ assert kept == set(range(7)) and possible == set(range(9))
+ # A clear cut (3.6 ulps) is no near tie.
+ kept, possible = sampling_check.cut_bounds(top, VOCAB, dict(TOP_K, top_k=5), 2.0)
+ assert kept == possible == set(range(5))
+ # top-p 0.6: shares 0.57 / 0.21 / 0.21, so whichever of the near-tied second and third ranks first is kept.
+ flat = [[21.0, 0], [20.0, 1], [19.98, 2], [17.0, 3]] + [[10.0 - r, 4 + r] for r in range(SCORED - 4)]
+ kept, possible = sampling_check.cut_bounds(flat, VOCAB, dict(TOP_K, top_k=-1, top_p=0.6), 2.0)
+ assert kept == {0} and possible == {0, 1, 2}
+ # min-p 0.05 at temperature 1: a token log(20) = 3.0 below the top, near the floor, is a near tie.
+ edge = [[20.0, 0], [20.0 - math.log(20.0) + 0.01, 1], [15.0, 2]] + [[10.0 - r, 3 + r] for r in range(SCORED - 3)]
+ kept, possible = sampling_check.cut_bounds(edge, VOCAB, dict(TOP_K, top_k=-1, min_p=0.05), 2.0)
+ assert kept == {0} and possible == {0, 1}
+ with pytest.raises(ValueError, match="past the 8 scored"):
+ sampling_check.cut_bounds(top[:8], VOCAB, TOP_K, 2.0)
+ with pytest.raises(ValueError, match="not modelled"):
+ sampling_check.cut_bounds(top, VOCAB, dict(TOP_K, repetition_penalty=1.5), 2.0)
+
+
+def test_score_keeps_as_many_tokens_as_bound_the_cut(sampling_check):
+ # min-p 0.05 on a flat distribution keeps every token within log(20) = 3.0 of the top: 300 here.
+ pairs = [[20.0 - 0.01 * r, r] for r in range(1000)]
+ min_p = dict(TOP_K, top_k=-1, min_p=0.05)
+ top = sampling_check.bounded_top(pairs, 5000, [TOP_K, min_p], 2.0)
+ assert len(top) == 1000 # 64 and 256 do not bound it
+ kept, possible = sampling_check.cut_bounds(top, 5000, min_p, 2.0)
+ # Near-ties: within one ulp (0.125) either side of the top and of the floor, i.e. 25 ranks around rank 300.
+ assert kept == set(range(275)) and possible == set(range(325))
+ assert len(sampling_check.bounded_top(pairs, 5000, [TOP_K], 2.0)) == 64
+
+
+def test_self_control_passes_through_bf16_cut_flips(sampling_check, tmp_path):
+ ref, cand = sampled_run(TOP_K, start=1), sampled_run(TOP_K, start=2)
+ # The flips are real: per draw position, the two starts' supports differ, which failed the old per-token test.
+ first = [[row[0] for row in doc["settings"]["setting"]["tokens"][0]] for doc in (ref, cand)]
+ assert set(first[0]) != set(first[1])
+ assert sampling_check.homogeneity(*first)[0] < 1e-6
+ assert compare(sampling_check, tmp_path, ref, cand, fp32_scores(sampling_check, ref)) == 0
+
+
+def test_truncated_setting_needs_fp32_scores(sampling_check, tmp_path):
+ ref = sampled_run(TOP_K, start=1)
+ assert compare(sampling_check, tmp_path, ref, copy.deepcopy(ref)) == 1
+
+
+def test_sampler_that_ignores_the_cut_fails(sampling_check, tmp_path):
+ ref = sampled_run(TOP_K, start=1)
+ untruncated = dict(TOP_K, top_k=-1)
+ cand = sampled_run(TOP_K, start=2, sampler=lambda logits, rng: draw(processed(logits, untruncated), rng))
+ assert compare(sampling_check, tmp_path, ref, cand, fp32_scores(sampling_check, ref)) == 1
+
+
[email protected]("biased", [False, True])
+def test_biased_verify_residual_fails(sampling_check, tmp_path, biased):
+ """An under-confident draft (the target at temperature 2): a correct chain verify keeps the target's
+ distribution, the mutated residual moves 6% of it (TV 0.063) off the top token."""
+
+ def verify(logits, rng):
+ p = processed(logits, TOP_K)
+ q = processed(logits, dict(TOP_K, temperature=2.0))
+ if biased:
+ return biased_chain_verify(p, q, rng)
+ y = draw(q, rng)
+ if rng.random() * q[y] < p[y]:
+ return y
+ residual = {t: max(p[t] - q.get(t, 0.0), 0.0) for t in p}
+ total = sum(residual.values())
+ return draw({t: v / total for t, v in residual.items()}, rng)
+
+ ref = sampled_run(TOP_K, start=1)
+ cand = sampled_run(TOP_K, start=2, sampler=verify)
+ assert compare(sampling_check, tmp_path, ref, cand, fp32_scores(sampling_check, ref)) == int(biased)
+
+
+def test_ignored_temperature_fails(sampling_check, tmp_path):
+ ref = sampled_run(TEMPERATURE, start=1)
+ assert compare(sampling_check, tmp_path, ref, sampled_run(TEMPERATURE, start=2)) == 0
+ hot = dict(TEMPERATURE, temperature=1.0)
+ cand = sampled_run(TEMPERATURE, start=2, sampler=lambda logits, rng: draw(processed(logits, hot), rng))
+ assert compare(sampling_check, tmp_path, ref, cand) == 1
diff --git a/tools/sampling_check.py b/tools/sampling_check.py
index 84bf353..7ac8ba3 100644
--- a/tools/sampling_check.py
+++ b/tools/sampling_check.py
@@ -7,22 +7,36 @@
``run`` sends ``--samples`` short completions of each of a few open-ended prompts to a running server under each
sampling setting (temperature, top-k with top-p, min-p, a repetition penalty), ``--concurrency`` requests at a
time so they share decode rounds, and saves the output token ids. The first token comes from the prefill sampler;
-every later one from a speculative verify round.
+every later one from a speculative verify round. Every setting names temperature, top-k, top-p, min-p and the
+repetition penalty: a server fills unnamed ones from the model's ``generation_config`` (Qwen3.8 sets top-k 20 and
+top-p 0.95), which would truncate a setting meant to be untruncated.
``compare`` tests, per setting and prompt, that the speculative server's tokens come from the same distribution as
-the reference's: a two-sample chi-square test of homogeneity on the joint distribution of the first three tokens and
-on the marginal of every position (cells seen fewer than 10 times in both runs together are pooled), at a
-Bonferroni-corrected level ``--alpha / tests``. It also prints the total variation distance of each test. A server
-that ignores a sampling parameter (for example, verifies greedily at temperature > 0) fails it by a wide margin.
-The first token comes from the prefill, so a speculative server whose prefill runs other attention kernels can
-differ there where a top-k or top-p cut falls between near-tied tokens; tokens 2 and 3 are therefore also tested
-given the reference's most common prefix ("verify draws given the modal prefix"), which isolates the verify rounds.
-Run the no-spec server twice to see the engine's own run-to-run level.
+the reference's, with two-sample chi-square tests of homogeneity (cells seen fewer than 10 times in both runs
+together are pooled) at a Bonferroni-corrected level ``--alpha / tests``, and prints the total variation distance
+of each test. A server that ignores a sampling parameter (for example, verifies greedily at temperature > 0) fails
+it by a wide margin.
+
+An untruncated setting is tested on the joint distribution of the first three tokens, on the marginal of every
+position, and on tokens 2 and 3 given the reference's most common prefix ("verify draws given the modal prefix",
+which isolates the verify rounds).
+
+A truncated setting (top-k, top-p or min-p) is tested only at fixed contexts: token 1 given the prompt, and tokens
+2 and 3 given the reference's most common prefix. A truncation cut is discontinuous in the logits: a token whose
+logit sits within bf16 rounding of the cut is kept by one server start or batch shape and dropped by another, and
+its whole probability moves with it, so two correct no-spec servers differ there. ``score`` re-forwards each of
+these contexts with the unmodified target in float32 (``tools/fp32_arbiter.py``'s reference model, on a GPU) and
+``compare --scores`` splits the vocabulary with the fp32 arbiter's tolerance (each logit may move by one bf16 ulp
+at its magnitude, so a pair's gap by two): tokens every such logit keeps, near-ties at the cut, and tokens no such
+logit keeps. Truncation renormalizes the kept tokens proportionally, so draws among the always-kept tokens follow
+the same distribution whichever near-ties a run kept; the test runs on those draws. A draw of a token no tolerated
+logit keeps fails the setting outright (a sampler that ignores the cut).
With ``--seeds S`` (``run``) every one of S seeds is sent twice, in batches of a different composition, at
temperature 0.7 for ``--seed-tokens`` tokens; ``compare`` (and ``run`` itself) report how many seeds reproduced
-their text, and ``compare`` how many seeded texts equal the reference server's for the same seed. sglang honors
-seeds only with ``--enable-deterministic-inference``.
+their text, and ``compare`` how many seeded texts equal the reference server's for the same seed. A seed reproduces
+across batches only on a batch-invariant server (sglang with ``--enable-deterministic-inference``), so ``compare``
+requires every seed to reproduce only with ``--deterministic``.
With ``--speed-data`` (a SPEED-Bench jsonl) ``run`` also decodes the first user turn of ``--speed-prompts``
conversations one at a time (batch size 1, chat template, thinking off) at temperature 0 and at temperature 1, and
@@ -30,21 +44,27 @@ conversations one at a time (batch size 1, chat template, thinking off) at tempe
python tools/sampling_check.py run --api vllm --port P --tokenizer <model dir> --out nospec.json --seeds 32
python tools/sampling_check.py run --api vllm --port P --tokenizer <model dir> --out spec.json --seeds 32
- python tools/sampling_check.py compare nospec.json spec.json
+ python tools/sampling_check.py score --model <model dir> nospec.json --out scores.json
+ python tools/sampling_check.py compare nospec.json spec.json --scores scores.json
-Exit code 1 (``compare``) when a test rejects at the corrected level, or a seed did not reproduce.
+Exit code 1 (``compare``) when a test rejects at the corrected level, a draw falls outside every tolerated cut, or
+(``--deterministic``) a seed did not reproduce.
"""
from __future__ import annotations
import argparse
+import bisect
import collections
import concurrent.futures
+import importlib.util
+import itertools
import json
import math
import sys
import time
-from typing import Dict, List, Optional, Sequence, Tuple
+from pathlib import Path
+from typing import Dict, List, Optional, Sequence, Set, Tuple
PROMPTS = (
"Here is a list of ten random animals:",
@@ -52,14 +72,16 @@ PROMPTS = (
"My favorite word is",
"The weather today is",
)
+# Every setting names each filter: a server fills unnamed ones from the model's generation_config.
+NEUTRAL = {"temperature": 1.0, "top_k": -1, "top_p": 1.0, "min_p": 0.0, "repetition_penalty": 1.0}
SETTINGS: Dict[str, dict] = {
- "unrestricted": {"temperature": 1.0, "top_p": 1.0, "top_k": -1},
- "temperature": {"temperature": 0.7},
- "top_k_top_p": {"temperature": 1.0, "top_k": 8, "top_p": 0.7},
- "min_p": {"temperature": 1.0, "min_p": 0.05},
- "penalty": {"temperature": 0.8, "repetition_penalty": 1.5},
+ "unrestricted": dict(NEUTRAL),
+ "temperature": {**NEUTRAL, "temperature": 0.7},
+ "top_k_top_p": {**NEUTRAL, "top_k": 8, "top_p": 0.7},
+ "min_p": {**NEUTRAL, "min_p": 0.05},
+ "penalty": {**NEUTRAL, "temperature": 0.8, "repetition_penalty": 1.5},
}
-SEED_SETTING = {"temperature": 0.7}
+SEED_SETTING = SETTINGS["temperature"]
JOINT = 3 # tokens in the joint test
POOL = 10 # cells seen fewer times than this in both runs together are pooled
@@ -219,6 +241,8 @@ def homogeneity(ref: Sequence, test: Sequence) -> Tuple[float, float, int]:
"""Two-sample chi-square test of homogeneity: (p value, total variation distance, cells after pooling)."""
a, b = collections.Counter(ref), collections.Counter(test)
na, nb = len(ref), len(test)
+ if not na or not nb:
+ return 1.0, 0.0, 0
tv = 0.5 * sum(abs(a[c] / na - b[c] / nb) for c in set(a) | set(b))
cells: List[Tuple[int, int]] = []
rest = [0, 0]
@@ -241,6 +265,158 @@ def homogeneity(ref: Sequence, test: Sequence) -> Tuple[float, float, int]:
return chi2_sf(stat, len(cells) - 1), tv, len(cells)
+TRUNCATION = ("top_k", "top_p", "min_p")
+
+
+def truncates(params: dict) -> bool:
+ """Whether the setting cuts the vocabulary (top-k, top-p or min-p)."""
+ return params["top_k"] not in (-1, 0) or params["top_p"] < 1.0 or params["min_p"] > 0.0
+
+
+def _sibling(name: str):
+ """Another tool of this directory, imported by path (tools/ is not a package)."""
+ if name not in sys.modules:
+ spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(f"{name}.py"))
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module # its dataclasses resolve their annotations through sys.modules
+ spec.loader.exec_module(module)
+ return sys.modules[name]
+
+
+def cut_bounds(top: Sequence[Sequence[float]], vocab: int, params: dict, tolerance_ulps: float) -> Tuple[Set, Set]:
+ """(tokens every tolerated logit keeps, tokens some tolerated logit keeps) under the setting's cut.
+
+ ``top``: fp32 ``[logit, token]`` pairs, best first; the ``vocab - len(top)`` other tokens are at most the last
+ logit. Each logit may move by ``tolerance_ulps / 2`` bf16 ulps at its magnitude, so the gap of a pair moves by up
+ to ``tolerance_ulps`` (the fp32 arbiter's rule). The cut is top-k, then top-p on the renormalized top-k
+ distribution at the setting's temperature, or min-p alone (vLLM's and sglang's order)."""
+ ulp = _sibling("fp32_arbiter").bf16_ulp
+ temperature, k, top_p, min_p = params["temperature"], params["top_k"], params["top_p"], params["min_p"]
+ k = None if k in (-1, 0) else k
+ if min_p > 0.0 and (k is not None or top_p < 1.0):
+ raise ValueError("min-p together with top-k or top-p is not modelled")
+ if params.get("repetition_penalty", 1.0) != 1.0:
+ raise ValueError("a repetition penalty together with a cut is not modelled")
+ tokens = [int(t) for _, t in top]
+ lo = [v - tolerance_ulps / 2 * ulp(abs(v)) for v, _ in top]
+ hi = [v + tolerance_ulps / 2 * ulp(abs(v)) for v, _ in top]
+ # The unscored tokens as one entry of multiplicity ``rest``, anywhere at or below the last scored logit.
+ rest = vocab - len(top)
+ lo.append(-math.inf)
+ hi.append(hi[-1])
+ count = [1] * len(tokens) + [rest]
+ n = len(count)
+ shift = max(hi)
+
+ def mass(x: float) -> float:
+ return math.exp((x - shift) / temperature)
+
+ def counter(bounds: List[float]):
+ """Tokens whose bound is above x, by bisection (the scored set can run to thousands of tokens)."""
+ pairs = sorted(zip(bounds, count))
+ keys = [b for b, _ in pairs]
+ tail = list(itertools.accumulate([c for _, c in pairs][::-1]))[::-1] + [0]
+ return lambda x: tail[bisect.bisect_right(keys, x)]
+
+ above_hi, above_lo = counter(hi), counter(lo)
+
+ # In the top k for every tolerated logit (fewer than k others can reach it) / for some (fewer than k others are
+ # surely above it). Only these tokens enter the top-p sums.
+ in_k = [k is None or above_hi(lo[i]) - count[i] < k for i in range(n)]
+ maybe_k = [count[i] > 0 and (k is None or above_lo(hi[i]) < k) for i in range(n)]
+ pool = [j for j in range(n) if maybe_k[j]]
+ by_hi = sorted(range(n), key=lambda j: hi[j], reverse=True)[:2] # the two largest bounds of each side, for min-p
+ by_lo = sorted(range(n), key=lambda j: lo[j], reverse=True)[:2]
+ kept, possible = [], []
+ for i in range(n):
+ sure, maybe = in_k[i], maybe_k[i]
+ if top_p < 1.0 and maybe:
+ # Kept when the renormalized mass of the tokens above it is below top_p: bound that share both ways.
+ others = [j for j in pool if j != i]
+ above_max = sum(count[j] * mass(hi[j]) for j in others if hi[j] > lo[i])
+ below_min = mass(lo[i]) + sum(count[j] * mass(lo[j]) for j in others if hi[j] < lo[i] and in_k[j])
+ sure = sure and above_max / (above_max + below_min) < top_p
+ above_min = sum(count[j] * mass(lo[j]) for j in others if lo[j] > hi[i] and in_k[j])
+ rest_max = count[i] * mass(hi[i]) + sum(count[j] * mass(hi[j]) for j in others if lo[j] <= hi[i])
+ maybe = above_min / (above_min + rest_max) < top_p
+ if min_p > 0.0:
+ # Kept when its probability is at least min_p times the largest.
+ floor = temperature * math.log(min_p)
+ others_hi = hi[by_hi[1] if by_hi[0] == i else by_hi[0]]
+ others_lo = lo[by_lo[1] if by_lo[0] == i else by_lo[0]]
+ sure = sure and lo[i] - max(lo[i], others_hi) >= floor
+ maybe = maybe and hi[i] - max(hi[i], others_lo) >= floor
+ kept.append(sure and i < len(tokens))
+ possible.append(maybe)
+ if possible[-1]:
+ raise ValueError(f"the cut may keep tokens past the {len(tokens)} scored ones")
+ return ({t for t, keep in zip(tokens, kept) if keep}, {t for t, keep in zip(tokens, possible) if keep})
+
+
+def modal_prefix(tokens: Sequence[Sequence[int]], depth: int) -> Tuple[int, ...]:
+ """The most common first ``depth`` tokens (the first seen among equally common ones)."""
+ return collections.Counter(tuple(t[:depth]) for t in tokens).most_common(1)[0][0]
+
+
+def judged_contexts(artifact: dict):
+ """(setting, prompt, prefix) of every context a truncated setting of this reference is judged at: the prompt,
+ then the reference's most common prefix of each length below ``JOINT``."""
+ for name, entry in artifact["settings"].items():
+ if "refused" in entry or not truncates(entry["params"]):
+ continue
+ for i, tokens in enumerate(entry["tokens"]):
+ for depth in range(JOINT):
+ yield name, i, modal_prefix(tokens, depth)
+
+
+def context_key(ids: Sequence[int]) -> str:
+ return ",".join(map(str, ids))
+
+
+def bounded_top(pairs: Sequence[Sequence[float]], vocab: int, settings: Sequence[dict], tolerance_ulps: float) -> list:
+ """The shortest head of ``pairs`` (fp32 ``[logit, token]``, best first) that bounds every setting's cut."""
+ size = min(64, len(pairs))
+ while True:
+ try:
+ for params in settings:
+ cut_bounds(pairs[:size], vocab, params, tolerance_ulps)
+ return list(pairs[:size])
+ except ValueError:
+ if size == len(pairs):
+ raise
+ size = min(4 * size, len(pairs))
+
+
+def score(args) -> int:
+ """fp32 logits of every judged context of the references: as many of the best as bound each setting's cut,
+ and the vocabulary size."""
+ import torch
+
+ fp32 = _sibling("fp32_arbiter")
+ judged: Dict[str, Tuple[list, list]] = {} # context key: (ids, params of the settings judged there)
+ for path in args.references:
+ with open(path, encoding="utf-8") as f:
+ artifact = json.load(f)
+ for name, i, prefix in judged_contexts(artifact):
+ ids = list(artifact["prompt_ids"][i]) + list(prefix)
+ judged.setdefault(context_key(ids), (ids, []))[1].append(artifact["settings"][name]["params"])
+ model, seconds = fp32.load_reference_model(args.model)
+ print(f"fp32 reference loaded in {seconds:.0f} s; {len(judged)} contexts", flush=True)
+ contexts: Dict[str, dict] = {}
+ for key, (ids, settings) in judged.items():
+ batch = torch.tensor([ids], dtype=torch.long, device="cuda")
+ with torch.inference_mode():
+ out = model(input_ids=batch, attention_mask=torch.ones_like(batch), use_cache=False, logits_to_keep=1)
+ logits = out.logits[0, -1].double()
+ values, indices = torch.sort(logits, descending=True)
+ pairs = [[v, int(t)] for v, t in zip(values.tolist(), indices.tolist())]
+ top = bounded_top(pairs, logits.numel(), settings, fp32.TOLERANCE_ULPS)
+ contexts[key] = {"vocab": logits.numel(), "top": top}
+ print(f"context {key}: {len(top)} tokens scored", flush=True)
+ save(args.out, {"model": args.model, "tolerance_ulps": fp32.TOLERANCE_ULPS, "contexts": contexts})
+ return 0
+
def validate_pair(ref: dict, cand: dict) -> None:
"""Reject partial evidence before a statistical test can silently omit it."""
if not ref["settings"] or ref["settings"].keys() != cand["settings"].keys():
@@ -256,6 +432,9 @@ def validate_pair(ref: dict, cand: dict) -> None:
raise ValueError(f"{name}: refused setting is not distribution evidence")
if r["params"] != c["params"]:
raise ValueError(f"{name}: sampling parameters differ")
+ unnamed = [key for key in ("temperature", *TRUNCATION) if key not in r["params"]]
+ if unnamed:
+ raise ValueError(f"{name}: {', '.join(unnamed)} left to the server's defaults (the model's may truncate)")
if len(r["tokens"]) != len(PROMPTS) or len(c["tokens"]) != len(PROMPTS):
raise ValueError(f"{name}: missing prompt results")
for r_tokens, c_tokens in zip(r["tokens"], c["tokens"]):
@@ -268,6 +447,57 @@ def validate_pair(ref: dict, cand: dict) -> None:
raise ValueError(f"{name}: incomplete output token sequence")
+def mixed_tests(name: str, prompt: int, r_tokens: list, c_tokens: list) -> list:
+ """An untruncated setting's tests: the joint first tokens, every position's marginal, and tokens 2 and 3 given
+ the reference's most common prefix (the verify round's own draws, free of any difference in the prefill's)."""
+ tests = []
+ r_joint = [tuple(t[:JOINT]) for t in r_tokens]
+ c_joint = [tuple(t[:JOINT]) for t in c_tokens]
+ tests.append((name, prompt, f"joint 1-{JOINT}", *homogeneity(r_joint, c_joint)))
+ for pos in range(min(len(r_tokens[0]), len(c_tokens[0]))):
+ marginals = [t[pos] for t in r_tokens], [t[pos] for t in c_tokens]
+ tests.append((name, prompt, f"token {pos + 1}", *homogeneity(*marginals)))
+ for pos in (1, 2):
+ prefix = modal_prefix(r_tokens, pos)
+ r_next = [t[pos] for t in r_tokens if tuple(t[:pos]) == prefix]
+ c_next = [t[pos] for t in c_tokens if tuple(t[:pos]) == prefix]
+ if r_next and c_next:
+ tests.append((name, prompt, f"token {pos + 1} | modal prefix", *homogeneity(r_next, c_next)))
+ return tests
+
+
+def cut_tests(
+ name: str,
+ params: dict,
+ prompt: int,
+ prompt_ids: list,
+ r_tokens: list,
+ c_tokens: list,
+ scores: dict,
+ tally: List[int],
+ outside: list,
+) -> list:
+ """A truncated setting's tests at its judged contexts, on the draws among the tokens every tolerated logit keeps.
+ Counts near-tie and outside draws into ``tally`` and lists the outside tokens in ``outside``."""
+ tests = []
+ for depth in range(JOINT):
+ prefix = modal_prefix(r_tokens, depth)
+ context = scores["contexts"].get(context_key(list(prompt_ids) + list(prefix)))
+ if context is None:
+ raise ValueError(f"{name}: prompt {prompt} prefix {list(prefix)} was not scored")
+ kept, possible = cut_bounds(context["top"], context["vocab"], params, scores["tolerance_ulps"])
+ draws = [[t[depth] for t in tokens if tuple(t[:depth]) == prefix] for tokens in (r_tokens, c_tokens)]
+ what = "token 1" if depth == 0 else f"token {depth + 1} | modal prefix"
+ tests.append((name, prompt, what, *homogeneity(*[[t for t in d if t in kept] for d in draws])))
+ for slot, d in enumerate(draws):
+ tally[slot] += sum(t in possible and t not in kept for t in d)
+ tally[2 + slot] += sum(t not in possible for t in d)
+ stray = [sorted({t for t in d if t not in possible}) for d in draws]
+ if stray[0] or stray[1]:
+ outside.append((name, prompt, what, *stray))
+ return tests
+
+
def compare(args) -> int:
with open(args.reference, encoding="utf-8") as f:
ref = json.load(f)
@@ -278,25 +508,28 @@ def compare(args) -> int:
except (KeyError, TypeError, ValueError) as error:
print(f"INVALID distribution evidence: {error}")
return 1
+ scores = None
+ if args.scores:
+ with open(args.scores, encoding="utf-8") as f:
+ scores = json.load(f)
tests = [] # (setting, prompt, what, p, tv, cells)
- for name, entry in cand["settings"].items():
- for i, (r_tokens, c_tokens) in enumerate(zip(ref["settings"][name]["tokens"], entry["tokens"])):
- k = min(len(r_tokens[0]), len(c_tokens[0]))
- r_joint = [tuple(t[:JOINT]) for t in r_tokens]
- c_joint = [tuple(t[:JOINT]) for t in c_tokens]
- tests.append((name, i, f"joint 1-{JOINT}", *homogeneity(r_joint, c_joint)))
- for pos in range(k):
- tests.append(
- (name, i, f"token {pos + 1}", *homogeneity([t[pos] for t in r_tokens], [t[pos] for t in c_tokens]))
- )
- # Tokens 2 and 3 given the reference's most common prefix: the verify round's own draws, free of any
- # difference in the prefill's sampling (other attention kernels move near-tied top-k / top-p cuts).
- for pos in (1, 2):
- prefix = collections.Counter(tuple(t[:pos]) for t in r_tokens).most_common(1)[0][0]
- r_next = [t[pos] for t in r_tokens if tuple(t[:pos]) == prefix]
- c_next = [t[pos] for t in c_tokens if tuple(t[:pos]) == prefix]
- if r_next and c_next:
- tests.append((name, i, f"token {pos + 1} | modal prefix", *homogeneity(r_next, c_next)))
+ cuts: Dict[str, List[int]] = {} # setting: draws [ref near-tie, candidate near-tie, ref outside, candidate outside]
+ outside: list = [] # (setting, prompt, what, reference tokens, candidate tokens) outside every tolerated cut
+ try:
+ for name, entry in cand["settings"].items():
+ params = entry["params"]
+ if truncates(params) and scores is None:
+ raise ValueError(f"{name} truncates: judge it with --scores (see score)")
+ for i, (r_tokens, c_tokens) in enumerate(zip(ref["settings"][name]["tokens"], entry["tokens"])):
+ if truncates(params):
+ tally = cuts.setdefault(name, [0, 0, 0, 0])
+ prompt_ids = ref["prompt_ids"][i]
+ tests += cut_tests(name, params, i, prompt_ids, r_tokens, c_tokens, scores, tally, outside)
+ else:
+ tests += mixed_tests(name, i, r_tokens, c_tokens)
+ except (KeyError, ValueError) as error: # unscored contexts, or a cut the scores cannot bound
+ print(f"INVALID distribution evidence: {error}")
+ return 1
level = args.alpha / max(1, len(tests))
failed = False
for name in cand["settings"]:
@@ -304,17 +537,25 @@ def compare(args) -> int:
if not rows:
continue
worst = min(rows, key=lambda t: t[3])
- joint_tv = max(t[4] for t in rows if t[2].startswith("joint"))
+ joint = [t[4] for t in rows if t[2].startswith("joint")]
bad = [t for t in rows if t[3] < level]
+ stray = [t for t in outside if t[0] == name]
conditional = [t for t in rows if "modal prefix" in t[2]]
cond_bad = [t for t in conditional if t[3] < level]
- failed |= bool(bad)
+ failed |= bool(bad) or bool(stray)
extra = cand["settings"][name].get("mean_tokens_per_verify") or cand["settings"][name].get("spec_counters", "")
print(
- f"{name:12s} {'FAIL' if bad else 'PASS'}: {len(rows)} tests, {len(bad)} below {level:.1e}; min p "
- f"{worst[3]:.2e} ({worst[2]}, prompt {worst[1]}, TV {worst[4]:.3f}); max joint TV {joint_tv:.3f}; "
+ f"{name:12s} {'FAIL' if bad or stray else 'PASS'}: {len(rows)} tests, {len(bad)} below {level:.1e}; min p "
+ f"{worst[3]:.2e} ({worst[2]}, prompt {worst[1]}, TV {worst[4]:.3f})"
+ f"{f'; max joint TV {max(joint):.3f}' if joint else ''}; "
f"{cand['settings'][name]['seconds']} s (ref {ref['settings'][name]['seconds']} s) {extra}"
)
+ if name in cuts:
+ tally = cuts[name]
+ print(
+ f" fp32 cut ({scores['tolerance_ulps']:g} bf16 ulps): near-tie draws left out {tally[0]} ref / "
+ f"{tally[1]} candidate; draws outside every tolerated cut {tally[2]} ref / {tally[3]} candidate"
+ )
cond_worst = min(conditional, key=lambda t: t[3]) if conditional else None
if cond_worst:
print(
@@ -323,6 +564,8 @@ def compare(args) -> int:
)
for t in sorted(bad, key=lambda t: t[3])[:5]:
print(f" {t[2]}, prompt {t[1]}: p {t[3]:.2e}, TV {t[4]:.3f}, {t[5]} cells")
+ for t in stray[:5]:
+ print(f" {t[2]}, prompt {t[1]}: tokens outside every tolerated cut: ref {t[3]}, candidate {t[4]}")
for temperature, entry in cand.get("speed", {}).items():
ref_entry = ref.get("speed", {}).get(temperature, {})
line = f"speed at temperature {temperature}: {entry['tok_s']} tok/s (reference {ref_entry.get('tok_s')})"
@@ -339,11 +582,14 @@ def compare(args) -> int:
for s, ids in seeds.items()
if s in ref.get("seeds", {})
]
+ # A seed reproduces its text in another batch only on a batch-invariant server (vLLM's default is not: its
+ # no-spec server reproduced 24 to 32 of 32 by chance), so only --deterministic makes it a gate.
print(
f"seeds: {reproduced}/{len(seeds)} reproduced their text; {same_as_ref}/{len(seeds)} equal the "
f"reference's text for the same seed (mean common prefix {sum(prefix) / max(1, len(prefix)):.1f} tokens)"
+ f"{'' if args.deterministic else '; not judged (no --deterministic)'}"
)
- failed |= reproduced != len(seeds)
+ failed |= args.deterministic and reproduced != len(seeds)
return 1 if failed else 0
@@ -364,12 +610,18 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
r.add_argument("--speed-data", help="SPEED-Bench jsonl: also time batch-size-1 decoding")
r.add_argument("--speed-prompts", type=int, default=8)
r.add_argument("--speed-tokens", type=int, default=512)
+ s = sub.add_parser("score", help="fp32 logits of the truncated settings' judged contexts (a GPU and the model)")
+ s.add_argument("--model", required=True)
+ s.add_argument("references", nargs="+", help="reference run outputs (their modal prefixes are judged)")
+ s.add_argument("--out", required=True)
c = sub.add_parser("compare")
c.add_argument("reference")
c.add_argument("candidate")
c.add_argument("--alpha", type=float, default=0.01, help="family-wise level, Bonferroni-corrected")
+ c.add_argument("--scores", help="score's output for the reference: required by truncated settings")
+ c.add_argument("--deterministic", action="store_true", help="the servers are batch-invariant: seeds must reproduce")
args = parser.parse_args(argv)
- return run(args) if args.cmd == "run" else compare(args)
+ return {"run": run, "score": score, "compare": compare}[args.cmd](args)
if __name__ == "__main__":