Official Inference Code of EAGLE-4
fix(vllm): refuse chain widths and tree rounds measured on another setup
Seyfal Sultanove9badddc1996
diff --git a/eagle_harness/core/chain_width.py b/eagle_harness/core/chain_width.pyindex a7ff31d..993c1ed 100644--- a/eagle_harness/core/chain_width.py+++ b/eagle_harness/core/chain_width.py@@ -46,6 +46,7 @@ from typing import Dict, List, Mapping, Optional, Sequence, Tuplefrom eagle_harness.core.cost_table import BatchCostTable, CostCurve, load_cost_tablesfrom eagle_harness.core.limits import nearest_cost_batch_size+from eagle_harness.core.live_costs import CostKey, check_keyPROFILE_FILE = os.path.join("accept", "chain.json")ROUNDS_FILE = os.path.join("rounds", "chain.json")@@ -216,13 +217,21 @@ def save_round_times(data_path: str, rows: Sequence[Mapping[str, float]]) -> strdef chain_widths(- data_path: str, max_width: int, max_batch_size: int, *, allow_no_spec: bool = False+ data_path: str,+ max_width: int,+ max_batch_size: int,+ *,+ allow_no_spec: bool = False,+ key: Optional[CostKey] = None,) -> Optional[List[Tuple[int, int, int]]]:"""The chain widths per batch size range under ``data_path``: from the served round times when measured, else- from the cost tables; both need the acceptance profile. None when either part is missing."""+ from the cost tables; both need the acceptance profile. None when either part is missing. With the server's+ ``key``, a directory measured on another setup is refused (:func:`~eagle_harness.core.live_costs.check_key`)."""profile = load_accept_profile(data_path)if profile is None:return None+ if key is not None:+ check_key(data_path, key)rounds = load_round_times(data_path)if rounds is not None:return served_width_schedule(rounds, profile, max_width, max_batch_size, allow_no_spec=allow_no_spec)@@ -271,13 +280,18 @@ def _check_tree_row(row: Mapping[str, float]) -> None:)-def load_tree_rounds(data_path: str) -> Optional[Dict[int, Dict[int, Tuple[float, float]]]]:- """``{batch size: {width: (round us, accept)}}`` from ``<data_path>/rounds/tree.json``, None if not measured."""+def load_tree_rounds(+ data_path: str, key: Optional[CostKey] = None+) -> Optional[Dict[int, Dict[int, Tuple[float, float]]]]:+ """``{batch size: {width: (round us, accept)}}`` from ``<data_path>/rounds/tree.json``, None if not measured.+ With the server's ``key``, rounds measured on another setup are refused (as by :func:`chain_widths`)."""try:with open(os.path.join(data_path, TREE_ROUNDS_FILE), encoding="utf-8") as f:rows = json.load(f)["rounds"]except FileNotFoundError:return None+ if key is not None:+ check_key(data_path, key)rounds: Dict[int, Dict[int, Tuple[float, float]]] = {}for row in rows:_check_tree_row(row)diff --git a/eagle_harness/vllm/config.py b/eagle_harness/vllm/config.pyindex d7a6f36..5f7d360 100644--- a/eagle_harness/vllm/config.py+++ b/eagle_harness/vllm/config.py@@ -54,6 +54,7 @@ from eagle_harness.core.limits import (req_to_token_span,verify_buckets,)+from eagle_harness.core.live_costs import CostKeyMETHOD_NAME = "eagle_harness"# Keys of ``method_config`` that are not HarnessConfig fields.@@ -148,11 +149,17 @@ def verify_width_schedule(def _verify_widths(- intensity: ComputeIntensity, max_verify_width: int, data_path: str | None, *, allow_unpriced: bool+ intensity: ComputeIntensity,+ max_verify_width: int,+ data_path: str | None,+ *,+ allow_unpriced: bool,+ key: CostKey | None = None,) -> List[Tuple[int, int, int]]:limit = max_batch_size(intensity)- widths = derived_chain_widths(data_path, max_verify_width, limit) if intensity is ComputeIntensity.CHAIN else None- if widths is None and intensity is ComputeIntensity.CHAIN and not allow_unpriced:+ chain = intensity is ComputeIntensity.CHAIN+ widths = derived_chain_widths(data_path, max_verify_width, limit, key) if chain else None+ if widths is None and chain and not allow_unpriced:raise ValueError(f"eagle_harness: no priced chain schedule with an acceptance profile under {data_path} "f"({PROFILE_FILE}); calibrate the chain or explicitly set max_verify_width for a fixed unpriced chain."@@ -174,12 +181,14 @@ def _draft_token_schedule(widths: List[Tuple[int, int, int]], limit: int) -> Lisreturn ranges-def derived_chain_widths(data_path: str | None, max_verify_width: int, limit: int) -> List[Tuple[int, int, int]] | None:+def derived_chain_widths(+ data_path: str | None, max_verify_width: int, limit: int, key: CostKey | None = None+) -> List[Tuple[int, int, int]] | None:"""The chain widths per batch size range priced by the served round times (else the cost tables) and the acceptanceprofile under ``data_path`` (:func:`eagle_harness.core.chain_width.chain_widths`); None when missing."""if not data_path:return None- widths = chain_widths(data_path, max_verify_width, limit, allow_no_spec=True)+ widths = chain_widths(data_path, max_verify_width, limit, allow_no_spec=True, key=key)if widths is not None:logger.info("eagle_harness: chain widths per batch size from %s: %s", data_path, widths)@@ -277,10 +286,10 @@ def native_decode_ranges(spec_config: Any) -> List[Tuple[int, int]]:return _native_ranges(widths)-def measured_tree_schedule(parsed: VllmMethodConfig) -> List[Tuple[int, int, int]] | None:+def measured_tree_schedule(parsed: VllmMethodConfig, key: CostKey | None = None) -> List[Tuple[int, int, int]] | None:"""The static tree's draft-count schedule from served rounds under ``data_path`` (None: one width throughout)."""data_path = parsed.harness.data_path- rounds = load_tree_rounds(data_path) if data_path else None+ rounds = load_tree_rounds(data_path, key) if data_path else Noneif not rounds:return Nonelimit = max_batch_size(parsed.harness.intensity)@@ -291,6 +300,18 @@ def measured_tree_schedule(parsed: VllmMethodConfig) -> List[Tuple[int, int, intreturn _draft_token_schedule(widths, limit)+def check_data_key(spec_config: Any, key: CostKey) -> None:+ """Refuse a ``data_path`` measured on another setup: re-read what :func:`configure_speculative_config` sized the+ schedule from, with this server's ``key``. Configure runs in the API process, which cannot name the GPU, so every+ worker checks before it serves."""+ parsed = parse_method_config(spec_config.method_config)+ harness = parsed.harness+ if harness.mode.uses_cost_model:+ _verify_widths(harness.intensity, parsed.max_verify_width, harness.data_path, allow_unpriced=True, key=key)+ elif parsed.node_top_k > 1:+ measured_tree_schedule(parsed, key)++def num_lookahead_tokens(spec_config: Any) -> int:"""``SpeculativeMethodSpec.num_lookahead_tokens`` (sglang hook H5's role)."""return parse_method_config(spec_config.method_config).draft_kv_lookaheaddiff --git a/eagle_harness/vllm/speculator.py b/eagle_harness/vllm/speculator.pyindex f933108..c7ad693 100644--- a/eagle_harness/vllm/speculator.py+++ b/eagle_harness/vllm/speculator.py@@ -42,7 +42,13 @@ from vllm.v1.worker.gpu.spec_decode.eagle.speculator import EagleSpeculatorfrom vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import SpeculatorCudaGraphManagerfrom eagle_harness.vllm.batch_policy import BatchPolicy, host_speculator_hooks-from eagle_harness.vllm.config import DERIVED, VllmMethodConfig, native_decode_ranges, parse_method_config+from eagle_harness.vllm.config import (+ DERIVED,+ VllmMethodConfig,+ check_data_key,+ native_decode_ranges,+ parse_method_config,+)from eagle_harness.vllm.depth_stop import derived_cost_tablesfrom eagle_harness.vllm.gdn_tree_verify import LinearStateCommit, check_tree_verifiablefrom eagle_harness.vllm.round_commit import KvCompactor, chainize@@ -88,22 +94,23 @@ class HarnessSpeculator(EagleSpeculator):)def load_model(self, target_model: Any) -> None:+ key = None+ if self.method_config.harness.data_path:+ from eagle_harness.vllm.calibrate import cost_key++ vc = self.vllm_config+ key = cost_key(+ vc.model_config.model,+ vc.speculative_config.draft_model_config.model,+ vc.parallel_config.tensor_parallel_size,+ vc.model_config.dtype,+ )+ check_data_key(self.speculative_config, key)super().load_model(target_model)if self.tree_mode:(self.draft_layer_name,) = self.draft_attn_layer_nameslayer = self.vllm_config.compilation_config.static_forward_context[self.draft_layer_name]gain = self.method_config.depth_stop_gain- key = None- if gain == DERIVED and self.method_config.harness.data_path:- from eagle_harness.vllm.calibrate import cost_key-- vc = self.vllm_config- key = cost_key(- vc.model_config.model,- vc.speculative_config.draft_model_config.model,- vc.parallel_config.tensor_parallel_size,- vc.model_config.dtype,- )self.drafter = TreeDrafter(self.model,layer,diff --git a/tests/test_live_costs.py b/tests/test_live_costs.pyindex 33575e2..00b4e12 100644--- a/tests/test_live_costs.py+++ b/tests/test_live_costs.py@@ -136,6 +136,44 @@ def test_offline_tables_are_refused_on_another_setup(tmp_path, caplog):assert json.load(open(tmp_path / "key.json"))["gpu"] == "NVIDIA B300"+def test_every_calibration_reader_refuses_another_setup(tmp_path):+ """A directory measured on B300/TP4 is refused by an H200/TP8 server on each path that reads it, as the cost+ tables are: the served chain widths, the vLLM tree rounds, and the vLLM worker's re-read of both."""+ from types import SimpleNamespace++ from eagle_harness.core.chain_width import (+ chain_widths,+ load_tree_rounds,+ merge_tree_rounds,+ save_accept_profile,+ save_round_times,+ )+ from eagle_harness.vllm.config import check_data_key++ data_path = str(tmp_path)+ b300 = CostKey("NVIDIA B300", 4, "qwen:abc", "eagle4:def", "vllm 0.11.1", "torch.bfloat16")+ h200 = CostKey("NVIDIA H200", 8, "qwen:abc", "eagle4:def", "vllm 0.11.1", "torch.bfloat16")+ write_key(data_path, b300)+ save_accept_profile(data_path, 100, [90, 80, 70])+ rows = [{"batch_size": 1, "width": w, "round_us": us, "count": 10} for w, us in ((1, 900.0), (4, 1000.0))]+ save_round_times(data_path, rows)+ merge_tree_rounds(data_path, [{"batch_size": 64, "width": 16, "round_us": 48000.0, "accept": 4.0, "count": 10}])+ chain = SimpleNamespace(method_config={"intensity": "chain_intensity", "data_path": data_path})+ tree = SimpleNamespace(method_config={"mode": "static", "max_verify_width": 16, "data_path": data_path})+ assert chain_widths(data_path, 4, 8, key=b300) == [(1, 8, 4)]+ assert load_tree_rounds(data_path, b300) == {64: {16: (48000.0, 4.0)}}+ check_data_key(chain, b300)+ check_data_key(tree, b300)+ for read in (+ lambda: chain_widths(data_path, 4, 8, key=h200),+ lambda: load_tree_rounds(data_path, h200),+ lambda: check_data_key(chain, h200),+ lambda: check_data_key(tree, h200),+ ):+ with pytest.raises(ValueError, match="another setup.*'gpu_count': \\(4, 8\\)"):+ read()++def test_offline_tables_are_validated_and_keyed_at_load(tmp_path):"""A table the cost model could not decide from fails at load with the directory, not in a decode round."""from eagle_harness.core.config import HarnessConfig, build_cost_model