Skip to content

Official Inference Code of EAGLE-4

fix(vllm): refuse chain widths and tree rounds measured on another setup

Seyfal Sultanove9badddc1996
diff --git a/eagle_harness/core/chain_width.py b/eagle_harness/core/chain_width.py
index a7ff31d..993c1ed 100644
--- a/eagle_harness/core/chain_width.py
+++ b/eagle_harness/core/chain_width.py
@@ -46,6 +46,7 @@ from typing import Dict, List, Mapping, Optional, Sequence, Tuple
from eagle_harness.core.cost_table import BatchCostTable, CostCurve, load_cost_tables
from eagle_harness.core.limits import nearest_cost_batch_size
+from eagle_harness.core.live_costs import CostKey, check_key
PROFILE_FILE = os.path.join("accept", "chain.json")
ROUNDS_FILE = os.path.join("rounds", "chain.json")
@@ -216,13 +217,21 @@ def save_round_times(data_path: str, rows: Sequence[Mapping[str, float]]) -> str
def chain_widths(
- data_path: str, max_width: int, max_batch_size: int, *, allow_no_spec: bool = False
+ data_path: str,
+ max_width: int,
+ max_batch_size: int,
+ *,
+ allow_no_spec: bool = False,
+ key: Optional[CostKey] = None,
) -> Optional[List[Tuple[int, int, int]]]:
"""The chain widths per batch size range under ``data_path``: from the served round times when measured, else
- from the cost tables; both need the acceptance profile. None when either part is missing."""
+ from the cost tables; both need the acceptance profile. None when either part is missing. With the server's
+ ``key``, a directory measured on another setup is refused (:func:`~eagle_harness.core.live_costs.check_key`)."""
profile = load_accept_profile(data_path)
if profile is None:
return None
+ if key is not None:
+ check_key(data_path, key)
rounds = load_round_times(data_path)
if rounds is not None:
return served_width_schedule(rounds, profile, max_width, max_batch_size, allow_no_spec=allow_no_spec)
@@ -271,13 +280,18 @@ def _check_tree_row(row: Mapping[str, float]) -> None:
)
-def load_tree_rounds(data_path: str) -> Optional[Dict[int, Dict[int, Tuple[float, float]]]]:
- """``{batch size: {width: (round us, accept)}}`` from ``<data_path>/rounds/tree.json``, None if not measured."""
+def load_tree_rounds(
+ data_path: str, key: Optional[CostKey] = None
+) -> Optional[Dict[int, Dict[int, Tuple[float, float]]]]:
+ """``{batch size: {width: (round us, accept)}}`` from ``<data_path>/rounds/tree.json``, None if not measured.
+ With the server's ``key``, rounds measured on another setup are refused (as by :func:`chain_widths`)."""
try:
with open(os.path.join(data_path, TREE_ROUNDS_FILE), encoding="utf-8") as f:
rows = json.load(f)["rounds"]
except FileNotFoundError:
return None
+ if key is not None:
+ check_key(data_path, key)
rounds: Dict[int, Dict[int, Tuple[float, float]]] = {}
for row in rows:
_check_tree_row(row)
diff --git a/eagle_harness/vllm/config.py b/eagle_harness/vllm/config.py
index d7a6f36..5f7d360 100644
--- a/eagle_harness/vllm/config.py
+++ b/eagle_harness/vllm/config.py
@@ -54,6 +54,7 @@ from eagle_harness.core.limits import (
req_to_token_span,
verify_buckets,
)
+from eagle_harness.core.live_costs import CostKey
METHOD_NAME = "eagle_harness"
# Keys of ``method_config`` that are not HarnessConfig fields.
@@ -148,11 +149,17 @@ def verify_width_schedule(
def _verify_widths(
- intensity: ComputeIntensity, max_verify_width: int, data_path: str | None, *, allow_unpriced: bool
+ intensity: ComputeIntensity,
+ max_verify_width: int,
+ data_path: str | None,
+ *,
+ allow_unpriced: bool,
+ key: CostKey | None = None,
) -> List[Tuple[int, int, int]]:
limit = max_batch_size(intensity)
- widths = derived_chain_widths(data_path, max_verify_width, limit) if intensity is ComputeIntensity.CHAIN else None
- if widths is None and intensity is ComputeIntensity.CHAIN and not allow_unpriced:
+ chain = intensity is ComputeIntensity.CHAIN
+ widths = derived_chain_widths(data_path, max_verify_width, limit, key) if chain else None
+ if widths is None and chain and not allow_unpriced:
raise ValueError(
f"eagle_harness: no priced chain schedule with an acceptance profile under {data_path} "
f"({PROFILE_FILE}); calibrate the chain or explicitly set max_verify_width for a fixed unpriced chain."
@@ -174,12 +181,14 @@ def _draft_token_schedule(widths: List[Tuple[int, int, int]], limit: int) -> Lis
return ranges
-def derived_chain_widths(data_path: str | None, max_verify_width: int, limit: int) -> List[Tuple[int, int, int]] | None:
+def derived_chain_widths(
+ data_path: str | None, max_verify_width: int, limit: int, key: CostKey | None = None
+) -> List[Tuple[int, int, int]] | None:
"""The chain widths per batch size range priced by the served round times (else the cost tables) and the acceptance
profile under ``data_path`` (:func:`eagle_harness.core.chain_width.chain_widths`); None when missing."""
if not data_path:
return None
- widths = chain_widths(data_path, max_verify_width, limit, allow_no_spec=True)
+ widths = chain_widths(data_path, max_verify_width, limit, allow_no_spec=True, key=key)
if widths is not None:
logger.info("eagle_harness: chain widths per batch size from %s: %s", data_path, widths)
@@ -277,10 +286,10 @@ def native_decode_ranges(spec_config: Any) -> List[Tuple[int, int]]:
return _native_ranges(widths)
-def measured_tree_schedule(parsed: VllmMethodConfig) -> List[Tuple[int, int, int]] | None:
+def measured_tree_schedule(parsed: VllmMethodConfig, key: CostKey | None = None) -> List[Tuple[int, int, int]] | None:
"""The static tree's draft-count schedule from served rounds under ``data_path`` (None: one width throughout)."""
data_path = parsed.harness.data_path
- rounds = load_tree_rounds(data_path) if data_path else None
+ rounds = load_tree_rounds(data_path, key) if data_path else None
if not rounds:
return None
limit = max_batch_size(parsed.harness.intensity)
@@ -291,6 +300,18 @@ def measured_tree_schedule(parsed: VllmMethodConfig) -> List[Tuple[int, int, int
return _draft_token_schedule(widths, limit)
+def check_data_key(spec_config: Any, key: CostKey) -> None:
+ """Refuse a ``data_path`` measured on another setup: re-read what :func:`configure_speculative_config` sized the
+ schedule from, with this server's ``key``. Configure runs in the API process, which cannot name the GPU, so every
+ worker checks before it serves."""
+ parsed = parse_method_config(spec_config.method_config)
+ harness = parsed.harness
+ if harness.mode.uses_cost_model:
+ _verify_widths(harness.intensity, parsed.max_verify_width, harness.data_path, allow_unpriced=True, key=key)
+ elif parsed.node_top_k > 1:
+ measured_tree_schedule(parsed, key)
+
+
def num_lookahead_tokens(spec_config: Any) -> int:
"""``SpeculativeMethodSpec.num_lookahead_tokens`` (sglang hook H5's role)."""
return parse_method_config(spec_config.method_config).draft_kv_lookahead
diff --git a/eagle_harness/vllm/speculator.py b/eagle_harness/vllm/speculator.py
index f933108..c7ad693 100644
--- a/eagle_harness/vllm/speculator.py
+++ b/eagle_harness/vllm/speculator.py
@@ -42,7 +42,13 @@ from vllm.v1.worker.gpu.spec_decode.eagle.speculator import EagleSpeculator
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import SpeculatorCudaGraphManager
from eagle_harness.vllm.batch_policy import BatchPolicy, host_speculator_hooks
-from eagle_harness.vllm.config import DERIVED, VllmMethodConfig, native_decode_ranges, parse_method_config
+from eagle_harness.vllm.config import (
+ DERIVED,
+ VllmMethodConfig,
+ check_data_key,
+ native_decode_ranges,
+ parse_method_config,
+)
from eagle_harness.vllm.depth_stop import derived_cost_tables
from eagle_harness.vllm.gdn_tree_verify import LinearStateCommit, check_tree_verifiable
from eagle_harness.vllm.round_commit import KvCompactor, chainize
@@ -88,22 +94,23 @@ class HarnessSpeculator(EagleSpeculator):
)
def load_model(self, target_model: Any) -> None:
+ key = None
+ if self.method_config.harness.data_path:
+ from eagle_harness.vllm.calibrate import cost_key
+
+ vc = self.vllm_config
+ key = cost_key(
+ vc.model_config.model,
+ vc.speculative_config.draft_model_config.model,
+ vc.parallel_config.tensor_parallel_size,
+ vc.model_config.dtype,
+ )
+ check_data_key(self.speculative_config, key)
super().load_model(target_model)
if self.tree_mode:
(self.draft_layer_name,) = self.draft_attn_layer_names
layer = self.vllm_config.compilation_config.static_forward_context[self.draft_layer_name]
gain = self.method_config.depth_stop_gain
- key = None
- if gain == DERIVED and self.method_config.harness.data_path:
- from eagle_harness.vllm.calibrate import cost_key
-
- vc = self.vllm_config
- key = cost_key(
- vc.model_config.model,
- vc.speculative_config.draft_model_config.model,
- vc.parallel_config.tensor_parallel_size,
- vc.model_config.dtype,
- )
self.drafter = TreeDrafter(
self.model,
layer,
diff --git a/tests/test_live_costs.py b/tests/test_live_costs.py
index 33575e2..00b4e12 100644
--- a/tests/test_live_costs.py
+++ b/tests/test_live_costs.py
@@ -136,6 +136,44 @@ def test_offline_tables_are_refused_on_another_setup(tmp_path, caplog):
assert json.load(open(tmp_path / "key.json"))["gpu"] == "NVIDIA B300"
+def test_every_calibration_reader_refuses_another_setup(tmp_path):
+ """A directory measured on B300/TP4 is refused by an H200/TP8 server on each path that reads it, as the cost
+ tables are: the served chain widths, the vLLM tree rounds, and the vLLM worker's re-read of both."""
+ from types import SimpleNamespace
+
+ from eagle_harness.core.chain_width import (
+ chain_widths,
+ load_tree_rounds,
+ merge_tree_rounds,
+ save_accept_profile,
+ save_round_times,
+ )
+ from eagle_harness.vllm.config import check_data_key
+
+ data_path = str(tmp_path)
+ b300 = CostKey("NVIDIA B300", 4, "qwen:abc", "eagle4:def", "vllm 0.11.1", "torch.bfloat16")
+ h200 = CostKey("NVIDIA H200", 8, "qwen:abc", "eagle4:def", "vllm 0.11.1", "torch.bfloat16")
+ write_key(data_path, b300)
+ save_accept_profile(data_path, 100, [90, 80, 70])
+ rows = [{"batch_size": 1, "width": w, "round_us": us, "count": 10} for w, us in ((1, 900.0), (4, 1000.0))]
+ save_round_times(data_path, rows)
+ merge_tree_rounds(data_path, [{"batch_size": 64, "width": 16, "round_us": 48000.0, "accept": 4.0, "count": 10}])
+ chain = SimpleNamespace(method_config={"intensity": "chain_intensity", "data_path": data_path})
+ tree = SimpleNamespace(method_config={"mode": "static", "max_verify_width": 16, "data_path": data_path})
+ assert chain_widths(data_path, 4, 8, key=b300) == [(1, 8, 4)]
+ assert load_tree_rounds(data_path, b300) == {64: {16: (48000.0, 4.0)}}
+ check_data_key(chain, b300)
+ check_data_key(tree, b300)
+ for read in (
+ lambda: chain_widths(data_path, 4, 8, key=h200),
+ lambda: load_tree_rounds(data_path, h200),
+ lambda: check_data_key(chain, h200),
+ lambda: check_data_key(tree, h200),
+ ):
+ with pytest.raises(ValueError, match="another setup.*'gpu_count': \\(4, 8\\)"):
+ read()
+
+
def test_offline_tables_are_validated_and_keyed_at_load(tmp_path):
"""A table the cost model could not decide from fails at load with the directory, not in a decode round."""
from eagle_harness.core.config import HarnessConfig, build_cost_model