Official Inference Code of EAGLE-4
feat(pkg): run sampling_check against TokenSpeed servers
Seyfal Sultanovfc55e030b07d
diff --git a/docs/faq.md b/docs/faq.mdindex 37cc9fd..c9f23df 100644--- a/docs/faq.md+++ b/docs/faq.md@@ -56,7 +56,8 @@ own code for those parts.## Does it support sampling, logprobs or images?SGLang and vLLM support sampled requests with the engine-specific restrictions below.-`tools/sampling_check.py` compares a speculative server with a no-spec server. Sampling smoke tests alone+`tools/sampling_check.py` compares a speculative server with a no-spec server (`--api sglang`, `vllm` or+`tokenspeed`). Sampling smoke tests alonedo not establish distribution equivalence for every model and workload.- vLLM: seeded sampling uses the target sampler, but an identical seed is not a blanket guarantee of the samediff --git a/tests/test_sampling_check.py b/tests/test_sampling_check.pyindex de09c22..9d044ae 100644--- a/tests/test_sampling_check.py+++ b/tests/test_sampling_check.py@@ -137,6 +137,26 @@ def test_compare_runs_without_the_http_client(tmp_path):assert out.returncode == 0, out.stderr[email protected]("reply", [+ {"output_ids": [5, 6], "meta_info": {"spec_verify_ct": 1}},+ [{"output_ids": [5, 6], "meta_info": {"spec_verify_ct": 1}}],+ [{"meta_info": {"output_ids": [5, 6], "spec_verify_ct": 1}}],+])+def test_tokenspeed_generate_names_the_model_and_unwraps_the_reply(sampling_check, monkeypatch, reply):+ # TokenSpeed's /generate routes by served model name and answers with a one-element list.+ sent = {}++ def post(url, json, timeout):+ sent.update(url=url, body=json)+ return SimpleNamespace(raise_for_status=lambda: None, json=lambda: reply)++ monkeypatch.setitem(sys.modules, "requests", SimpleNamespace(post=post))+ ids, meta = sampling_check.generate("tokenspeed", "http://h", "served", [1, 2], {"temperature": 1.0}, 2)+ assert (ids, meta) == ([5, 6], {"spec_verify_ct": 1})+ assert sent["url"] == "http://h/generate" and sent["body"]["model"] == "served"+ assert sent["body"]["sampling_params"] == {"max_new_tokens": 2, "ignore_eos": True, "temperature": 1.0}++# A synthetic target: 40 tokens whose fp32 logits fall 0.45 (3.6 bf16 ulps at [16, 32)) per rank from 24, except that# ranks 8 and 9 sit 0.04 apart, a near tie right at a top-8 cut. A server start moves every logit by up to a quarter# bf16 ulp and the rank-8 one a further ulp up (odd starts) or down (even starts), so odd starts keep rank 8 and evendiff --git a/tools/sampling_check.py b/tools/sampling_check.pyindex 7ac8ba3..18f53b9 100644--- a/tools/sampling_check.py+++ b/tools/sampling_check.py@@ -47,6 +47,9 @@ conversations one at a time (batch size 1, chat template, thinking off) at tempepython tools/sampling_check.py score --model <model dir> nospec.json --out scores.jsonpython tools/sampling_check.py compare nospec.json spec.json --scores scores.json+``--api`` names the server's HTTP API: ``sglang`` and ``tokenspeed`` (``/generate``, TokenSpeed with the served+model's name) or ``vllm`` (``/v1/completions``).+Exit code 1 (``compare``) when a test rejects at the corrected level, a draw falls outside every tolerated cut, or(``--deterministic``) a seed did not reproduce."""@@ -89,14 +92,20 @@ POOL = 10 # cells seen fewer times than this in both runs together are pooleddef generate(api: str, base: str, model: Optional[str], prompt: List[int], params: dict, max_new: int, seed=None):"""One completion's output token ids (and sglang's ``spec_*`` meta info)."""import requests # only the HTTP client needs it; compare runs in test envs without it- if api == "sglang":+ if api in ("sglang", "tokenspeed"):sp = {"max_new_tokens": max_new, "ignore_eos": True, **params}if seed is not None:sp["sampling_seed"] = seed- r = requests.post(f"{base}/generate", json={"input_ids": prompt, "sampling_params": sp}, timeout=3600)+ body = {"input_ids": prompt, "sampling_params": sp}+ if api == "tokenspeed": # TokenSpeed's /generate routes by model name and answers with a one-element list+ body["model"] = model+ r = requests.post(f"{base}/generate", json=body, timeout=3600)r.raise_for_status()out = r.json()- return out["output_ids"], {k: v for k, v in out["meta_info"].items() if k.startswith("spec_")}+ out = out[0] if isinstance(out, list) else out+ meta = out.get("meta_info") or {}+ ids = out["output_ids"] if "output_ids" in out else meta["output_ids"]+ return ids, {k: v for k, v in meta.items() if k.startswith("spec_")}body = {"model": model, "prompt": prompt, "max_tokens": max_new, "ignore_eos": True, "return_token_ids": True}body.update(params)if seed is not None:@@ -122,7 +131,7 @@ def run(args) -> int:tokenizer = AutoTokenizer.from_pretrained(args.tokenizer)base = f"http://127.0.0.1:{args.port}"- model = requests.get(f"{base}/v1/models", timeout=60).json()["data"][0]["id"] if args.api == "vllm" else None+ model = requests.get(f"{base}/v1/models", timeout=60).json()["data"][0]["id"] if args.api != "sglang" else Noneprompts = [tokenizer.encode(p, add_special_tokens=False) for p in PROMPTS]result: dict = {"api": args.api,@@ -597,7 +606,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int:parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)sub = parser.add_subparsers(dest="cmd", required=True)r = sub.add_parser("run")- r.add_argument("--api", choices=("sglang", "vllm"), required=True)+ r.add_argument("--api", choices=("sglang", "vllm", "tokenspeed"), required=True)r.add_argument("--port", type=int, required=True)r.add_argument("--tokenizer", required=True)r.add_argument("--out", required=True)