Skip to content

Official Inference Code of EAGLE-4

feat(pkg): run sampling_check against TokenSpeed servers

Seyfal Sultanovfc55e030b07d
diff --git a/docs/faq.md b/docs/faq.md
index 37cc9fd..c9f23df 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -56,7 +56,8 @@ own code for those parts.
## Does it support sampling, logprobs or images?
SGLang and vLLM support sampled requests with the engine-specific restrictions below.
-`tools/sampling_check.py` compares a speculative server with a no-spec server. Sampling smoke tests alone
+`tools/sampling_check.py` compares a speculative server with a no-spec server (`--api sglang`, `vllm` or
+`tokenspeed`). Sampling smoke tests alone
do not establish distribution equivalence for every model and workload.
- vLLM: seeded sampling uses the target sampler, but an identical seed is not a blanket guarantee of the same
diff --git a/tests/test_sampling_check.py b/tests/test_sampling_check.py
index de09c22..9d044ae 100644
--- a/tests/test_sampling_check.py
+++ b/tests/test_sampling_check.py
@@ -137,6 +137,26 @@ def test_compare_runs_without_the_http_client(tmp_path):
assert out.returncode == 0, out.stderr
+ {"output_ids": [5, 6], "meta_info": {"spec_verify_ct": 1}},
+ [{"output_ids": [5, 6], "meta_info": {"spec_verify_ct": 1}}],
+ [{"meta_info": {"output_ids": [5, 6], "spec_verify_ct": 1}}],
+])
+def test_tokenspeed_generate_names_the_model_and_unwraps_the_reply(sampling_check, monkeypatch, reply):
+ # TokenSpeed's /generate routes by served model name and answers with a one-element list.
+ sent = {}
+
+ def post(url, json, timeout):
+ sent.update(url=url, body=json)
+ return SimpleNamespace(raise_for_status=lambda: None, json=lambda: reply)
+
+ monkeypatch.setitem(sys.modules, "requests", SimpleNamespace(post=post))
+ ids, meta = sampling_check.generate("tokenspeed", "http://h", "served", [1, 2], {"temperature": 1.0}, 2)
+ assert (ids, meta) == ([5, 6], {"spec_verify_ct": 1})
+ assert sent["url"] == "http://h/generate" and sent["body"]["model"] == "served"
+ assert sent["body"]["sampling_params"] == {"max_new_tokens": 2, "ignore_eos": True, "temperature": 1.0}
+
+
# A synthetic target: 40 tokens whose fp32 logits fall 0.45 (3.6 bf16 ulps at [16, 32)) per rank from 24, except that
# ranks 8 and 9 sit 0.04 apart, a near tie right at a top-8 cut. A server start moves every logit by up to a quarter
# bf16 ulp and the rank-8 one a further ulp up (odd starts) or down (even starts), so odd starts keep rank 8 and even
diff --git a/tools/sampling_check.py b/tools/sampling_check.py
index 7ac8ba3..18f53b9 100644
--- a/tools/sampling_check.py
+++ b/tools/sampling_check.py
@@ -47,6 +47,9 @@ conversations one at a time (batch size 1, chat template, thinking off) at tempe
python tools/sampling_check.py score --model <model dir> nospec.json --out scores.json
python tools/sampling_check.py compare nospec.json spec.json --scores scores.json
+``--api`` names the server's HTTP API: ``sglang`` and ``tokenspeed`` (``/generate``, TokenSpeed with the served
+model's name) or ``vllm`` (``/v1/completions``).
+
Exit code 1 (``compare``) when a test rejects at the corrected level, a draw falls outside every tolerated cut, or
(``--deterministic``) a seed did not reproduce.
"""
@@ -89,14 +92,20 @@ POOL = 10 # cells seen fewer times than this in both runs together are pooled
def generate(api: str, base: str, model: Optional[str], prompt: List[int], params: dict, max_new: int, seed=None):
"""One completion's output token ids (and sglang's ``spec_*`` meta info)."""
import requests # only the HTTP client needs it; compare runs in test envs without it
- if api == "sglang":
+ if api in ("sglang", "tokenspeed"):
sp = {"max_new_tokens": max_new, "ignore_eos": True, **params}
if seed is not None:
sp["sampling_seed"] = seed
- r = requests.post(f"{base}/generate", json={"input_ids": prompt, "sampling_params": sp}, timeout=3600)
+ body = {"input_ids": prompt, "sampling_params": sp}
+ if api == "tokenspeed": # TokenSpeed's /generate routes by model name and answers with a one-element list
+ body["model"] = model
+ r = requests.post(f"{base}/generate", json=body, timeout=3600)
r.raise_for_status()
out = r.json()
- return out["output_ids"], {k: v for k, v in out["meta_info"].items() if k.startswith("spec_")}
+ out = out[0] if isinstance(out, list) else out
+ meta = out.get("meta_info") or {}
+ ids = out["output_ids"] if "output_ids" in out else meta["output_ids"]
+ return ids, {k: v for k, v in meta.items() if k.startswith("spec_")}
body = {"model": model, "prompt": prompt, "max_tokens": max_new, "ignore_eos": True, "return_token_ids": True}
body.update(params)
if seed is not None:
@@ -122,7 +131,7 @@ def run(args) -> int:
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer)
base = f"http://127.0.0.1:{args.port}"
- model = requests.get(f"{base}/v1/models", timeout=60).json()["data"][0]["id"] if args.api == "vllm" else None
+ model = requests.get(f"{base}/v1/models", timeout=60).json()["data"][0]["id"] if args.api != "sglang" else None
prompts = [tokenizer.encode(p, add_special_tokens=False) for p in PROMPTS]
result: dict = {
"api": args.api,
@@ -597,7 +606,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = parser.add_subparsers(dest="cmd", required=True)
r = sub.add_parser("run")
- r.add_argument("--api", choices=("sglang", "vllm"), required=True)
+ r.add_argument("--api", choices=("sglang", "vllm", "tokenspeed"), required=True)
r.add_argument("--port", type=int, required=True)
r.add_argument("--tokenizer", required=True)
r.add_argument("--out", required=True)