Official Inference Code of EAGLE-4
feat(vllm): answer vLLM's suspend_drafting and verify_budget hooks
Seyfal Sultanov6465fb4d925f
diff --git a/eagle_harness/vllm/batch_policy.py b/eagle_harness/vllm/batch_policy.pynew file mode 100644index 0000000..2d710c9--- /dev/null+++ b/eagle_harness/vllm/batch_policy.py@@ -0,0 +1,109 @@+# Copyright 2026 Eagle AI Corp.+# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available++"""Per-batch serving decisions the speculator answers through vLLM's optional speculator hooks.++vLLM review PR 3 lets a speculator decide two things per scheduled batch (:func:`host_speculator_hooks` finds them):++- ``suspend_drafting(num_reqs, req_ids)``: native target decode (native-OFF) for the batch sizes where the priced+ chain schedule picked width 0 (:func:`~eagle_harness.vllm.config.native_decode_ranges`).+- ``verify_budget(req_ids, num_draft_tokens)``: how many scheduled drafts each request verifies. The scheduler sizes a+ request's drafts from the batch size of the step that drafted them; this verifies at most the width the schedule+ gives the batch being verified, so the measured high-load width (``served_tree_schedule``, the chain widths) applies+ from the first step at that batch size instead of one step later. A tree's verify rows put each parent before its+ children, so a prefix of them is a tree.++On a vLLM without these hooks (the 0.1.0 release engine) nothing here runs: the native ranges go through+``SpeculativeConfig.native_decode_batch_sizes`` and the schedule alone sets the width, as before.+"""++from __future__ import annotations++import inspect+import logging+import time+from typing import Callable, FrozenSet, Iterable, Optional, Sequence, Tuple++import numpy as np++logger = logging.getLogger(__name__)++# hook -> its parameters after ``self``: a host whose hook has other parameters does not count as having it+SPECULATOR_HOOKS = {+ "suspend_drafting": ("num_reqs", "req_ids"),+ "verify_budget": ("req_ids", "num_draft_tokens"),+}+LOG_INTERVAL_S = 60.0+++def host_speculator_hooks() -> FrozenSet[str]:+ """The optional ``BaseSpeculator`` hooks this vLLM calls (empty without vLLM or on the release engine)."""+ try:+ from vllm.v1.worker.gpu.spec_decode.speculator import BaseSpeculator+ except ImportError:+ return frozenset()+ found = set()+ for name, params in SPECULATOR_HOOKS.items():+ hook = getattr(BaseSpeculator, name, None)+ if callable(hook) and tuple(inspect.signature(hook).parameters)[1:] == params:+ found.add(name)+ return frozenset(found)+++class BatchPolicy:+ """Answers ``suspend_drafting`` and ``verify_budget`` from static per-batch-size tables (host only, no device+ reads) and logs what they did at most once a minute."""++ def __init__(+ self,+ native_ranges: Iterable[Tuple[int, int]],+ schedule: Optional[Sequence[Sequence[int]]],+ max_num_reqs: int,+ *,+ clock: Callable[[], float] = time.monotonic,+ ):+ self.native_ranges = tuple((int(first), int(last)) for first, last in native_ranges)+ # drafts verified at each batch size (None: the scheduler's count stands)+ self.width_drafts: Optional[np.ndarray] = None+ if schedule:+ drafts = np.full(max_num_reqs + 1, np.iinfo(np.int32).max, dtype=np.int64)+ for first, last, count in schedule:+ drafts[max(int(first), 0) : min(int(last), max_num_reqs) + 1] = int(count)+ self.width_drafts = drafts+ self.clock = clock+ self.batches = self.suspended = self.trimmed = 0+ self._logged: Tuple[int, int, int] = (0, 0, 0)+ self._next_log = 0.0++ def suspend(self, num_reqs: int) -> bool:+ suspend = any(first <= num_reqs <= last for first, last in self.native_ranges)+ if suspend:+ self.suspended += 1+ self._report()+ return suspend++ def verified_drafts(self, num_reqs: int, num_draft_tokens: np.ndarray) -> np.ndarray:+ """``num_draft_tokens`` itself when nothing is trimmed (vLLM then keeps the batch as scheduled)."""+ self.batches += 1+ if self.width_drafts is None or not len(num_draft_tokens):+ return num_draft_tokens+ width = self.width_drafts[min(num_reqs, len(self.width_drafts) - 1)]+ if int(num_draft_tokens.max()) <= width:+ return num_draft_tokens+ self.trimmed += 1+ self._report()+ return np.minimum(num_draft_tokens, width)++ def _report(self) -> None:+ now = self.clock()+ counts = (self.batches, self.trimmed, self.suspended)+ if now < self._next_log or counts == self._logged:+ return+ self._next_log, self._logged = now + LOG_INTERVAL_S, counts+ logger.info(+ "eagle_harness: verify_budget trimmed %d of %d draft batches to their batch size's width; "+ "suspend_drafting suspended %d batches",+ self.trimmed,+ self.batches,+ self.suspended,+ )diff --git a/eagle_harness/vllm/config.py b/eagle_harness/vllm/config.pyindex 7d3ad9a..d7a6f36 100644--- a/eagle_harness/vllm/config.py+++ b/eagle_harness/vllm/config.py@@ -63,7 +63,8 @@ DEFAULT_TREE_DEPTH = 12DERIVED = "derived" # depth_stop_gain from the measured costsNATIVE_DECODE_UNSUPPORTED = ("eagle_harness: the chain schedule selects native plain decode (width 0), "- "which requires vLLM's native_decode_batch_sizes capability. K=0 still extends draft KV. "+ "which requires vLLM's suspend_drafting speculator hook or its native_decode_batch_sizes field. "+ "K=0 still extends draft KV. ""Use a compatible host, run without speculative_config, or calibrate root-only width 1.")@@ -224,11 +225,16 @@ def configure_speculative_config(spec_config: Any) -> None:parsed.harness.data_path,allow_unpriced="max_verify_width" in spec_config.method_config,)- native_ranges = [(first, last) for first, last, width in widths if width == 0]+ native_ranges = _native_ranges(widths)if native_ranges:- if not hasattr(spec_config, "native_decode_batch_sizes"):+ from eagle_harness.vllm.batch_policy import host_speculator_hooks++ if "suspend_drafting" in host_speculator_hooks():+ pass # HarnessSpeculator.suspend_drafting answers from the same widths (native_decode_ranges)+ elif hasattr(spec_config, "native_decode_batch_sizes"): # the 0.1.0 release engine+ spec_config.native_decode_batch_sizes = native_ranges+ else:raise ValueError(NATIVE_DECODE_UNSUPPORTED)- spec_config.native_decode_batch_sizes = native_rangesspec_config.num_speculative_tokens_per_batch_size = _draft_token_schedule(widths, max_batch_size(parsed.harness.intensity))@@ -248,6 +254,29 @@ def configure_speculative_config(spec_config: Any) -> None:)+def _native_ranges(widths: List[Tuple[int, int, int]]) -> List[Tuple[int, int]]:+ return [(first, last) for first, last, width in widths if width == 0]+++def native_decode_ranges(spec_config: Any) -> List[Tuple[int, int]]:+ """The batch-size ranges where the schedule :func:`configure_speculative_config` installed decodes natively+ (width 0); empty when it installed none or the user set the schedule. Deterministic, so the speculator in each+ worker recomputes what configure saw."""+ parsed = parse_method_config(spec_config.method_config)+ if not parsed.harness.mode.uses_cost_model:+ return []+ widths = _verify_widths(+ parsed.harness.intensity,+ parsed.max_verify_width,+ parsed.harness.data_path,+ allow_unpriced="max_verify_width" in spec_config.method_config,+ )+ installed = [tuple(entry) for entry in spec_config.num_speculative_tokens_per_batch_size or ()]+ if installed != _draft_token_schedule(widths, max_batch_size(parsed.harness.intensity)):+ return []+ return _native_ranges(widths)++def measured_tree_schedule(parsed: VllmMethodConfig) -> List[Tuple[int, int, int]] | None:"""The static tree's draft-count schedule from served rounds under ``data_path`` (None: one width throughout)."""data_path = parsed.harness.data_pathdiff --git a/eagle_harness/vllm/speculator.py b/eagle_harness/vllm/speculator.pyindex eb7c33d..f933108 100644--- a/eagle_harness/vllm/speculator.py+++ b/eagle_harness/vllm/speculator.py@@ -20,7 +20,9 @@ Tree mode (``node_top_k > 1``) runs one round as follows (vLLM's hooks in brackewrites the next verify rows (``draft_tokens`` in node order, their parents in the :class:`TreeStore`).vLLM's scheduler fixes the verify width of a step from the batch size (``num_speculative_tokens``), so the-width is the batch-size schedule's, not a per-round choice. The draft depth is per round: a captured growth skips+width is the batch-size schedule's, not a per-round choice. On a vLLM with the optional ``verify_budget`` and+``suspend_drafting`` hooks, :class:`~eagle_harness.vllm.batch_policy.BatchPolicy` applies that width to the batch being+verified and runs the chain schedule's native-OFF batch sizes. The draft depth is per round: a captured growth skipsthe steps whose nodes could not enter the verify set (:mod:`~eagle_harness.vllm.depth_stop`)."""@@ -39,7 +41,8 @@ from vllm.utils.gpu_sync_debug import gpu_sync_allowedfrom vllm.v1.worker.gpu.spec_decode.eagle.speculator import EagleSpeculatorfrom vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import SpeculatorCudaGraphManager-from eagle_harness.vllm.config import DERIVED, VllmMethodConfig, parse_method_config+from eagle_harness.vllm.batch_policy import BatchPolicy, host_speculator_hooks+from eagle_harness.vllm.config import DERIVED, VllmMethodConfig, native_decode_ranges, parse_method_configfrom eagle_harness.vllm.depth_stop import derived_cost_tablesfrom eagle_harness.vllm.gdn_tree_verify import LinearStateCommit, check_tree_verifiablefrom eagle_harness.vllm.round_commit import KvCompactor, chainize@@ -57,6 +60,13 @@ class HarnessSpeculator(EagleSpeculator):super().__init__(vllm_config, device)self.method_config: VllmMethodConfig = parse_method_config(self.speculative_config.method_config)self.tree_mode = self.method_config.node_top_k > 1+ hooks = host_speculator_hooks() # empty on the 0.1.0 release engine: neither hook below is called there+ self.batch_policy = BatchPolicy(+ native_decode_ranges(self.speculative_config) if "suspend_drafting" in hooks else (),+ self.speculative_config.num_speculative_tokens_per_batch_size if "verify_budget" in hooks else None,+ self.max_num_reqs,+ )+ self.may_suspend_drafting = bool(self.batch_policy.native_ranges)if self.tree_mode:width = self.num_speculative_steps + 1self.store = TreeStore(self.max_num_reqs, width, device)@@ -168,6 +178,13 @@ class HarnessSpeculator(EagleSpeculator):)return self._bound+ # ------------------------------------------------------------------ per-batch policy hooks+ def suspend_drafting(self, num_reqs: int, req_ids: Any) -> bool:+ return self.batch_policy.suspend(num_reqs)++ def verify_budget(self, req_ids: Any, num_draft_tokens: Any) -> Any:+ return self.batch_policy.verified_drafts(len(req_ids), num_draft_tokens)+# ------------------------------------------------------------------ verify hooksdef prepare_verify_inputs(self, input_batch: Any, model_inputs: dict[str, Any], *, dummy_run: bool = False) -> None:if not self.tree_mode:diff --git a/tests/test_chain_no_spec.py b/tests/test_chain_no_spec.pyindex 746bebf..85f57ba 100644--- a/tests/test_chain_no_spec.py+++ b/tests/test_chain_no_spec.py@@ -13,6 +13,15 @@ from eagle_harness.core.limits import ComputeIntensityfrom tests.test_chain_width import PROFILE[email protected](autouse=True)+def release_engine_shape(monkeypatch):+ """These tests pin the 0.1.0 release engine's contract (``native_decode_batch_sizes``, no speculator hooks)+ whatever vLLM is installed; tests/test_vllm_batch_policy.py covers the hook engine."""+ from eagle_harness.vllm import batch_policy++ monkeypatch.setattr(batch_policy, "host_speculator_hooks", lambda: frozenset())++def test_native_decode_is_a_distinct_opt_in_action(tmp_path):save_accept_profile(str(tmp_path), 100, [90, 80, 70])save_round_times(diff --git a/tests/test_vllm_batch_policy.py b/tests/test_vllm_batch_policy.pynew file mode 100644index 0000000..2b3e196--- /dev/null+++ b/tests/test_vllm_batch_policy.py@@ -0,0 +1,154 @@+# Copyright 2026 Eagle AI Corp.+# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available++"""The per-batch policy behind vLLM's optional ``suspend_drafting`` / ``verify_budget`` speculator hooks, on both+engine shapes: the 0.1.0 release engine (no hooks: native ranges go through ``native_decode_batch_sizes``) and review+PR 3 (hooks: the speculator answers them)."""++import logging+import sys+import types+from types import SimpleNamespace+from unittest import mock++import numpy as np+import pytest++from eagle_harness.vllm import batch_policy+from eagle_harness.vllm.batch_policy import BatchPolicy, host_speculator_hooks+from eagle_harness.vllm.config import configure_speculative_config, native_decode_ranges+from tests.test_chain_no_spec import _native_schedule_config++HOOKS = frozenset(batch_policy.SPECULATOR_HOOKS)+SCHEDULE = [(1, 19, 15), (20, 512, 3), (513, 1 << 16, 0)]+++def test_suspend_follows_the_native_ranges_only():+ policy = BatchPolicy([(8, 16), (40, 64)], None, 64)+ assert [policy.suspend(n) for n in (1, 7, 8, 16, 17, 40, 64)] == [False, False, True, True, False, True, True]+ assert policy.suspended == 4+ assert not BatchPolicy((), None, 64).suspend(64)+++def test_verify_budget_keeps_the_schedule_unless_the_batch_size_verifies_fewer():+ policy = BatchPolicy((), SCHEDULE, 64)+ drafted_small = np.array([15, 15, 15], dtype=np.int32)+ # the batch drafted at its own batch size: vLLM keeps it as scheduled (same object)+ assert policy.verified_drafts(3, drafted_small) is drafted_small+ within = np.array([3, 2], dtype=np.int32)+ assert policy.verified_drafts(20, within) is within+ # drafted at batch size < 20 (width 16), verified in a batch of 20 (width 4)+ trimmed = policy.verified_drafts(20, np.array([15, 3, 0], dtype=np.int32))+ assert trimmed.tolist() == [3, 3, 0]+ # batch sizes past max_num_reqs take the last entry+ assert policy.verified_drafts(99, np.array([15], dtype=np.int32)).tolist() == [3]+ assert (policy.batches, policy.trimmed) == (4, 2)+++def test_without_a_schedule_every_scheduled_draft_is_verified():+ drafts = np.array([7, 7], dtype=np.int32)+ assert BatchPolicy([(1, 4)], None, 64).verified_drafts(2, drafts) is drafts+ empty = np.array([], dtype=np.int32)+ assert BatchPolicy((), SCHEDULE, 64).verified_drafts(0, empty) is empty+++def test_policy_logs_the_first_event_then_once_a_minute(caplog):+ now = [100.0]+ policy = BatchPolicy([(64, 64)], SCHEDULE, 64, clock=lambda: now[0])+ with caplog.at_level(logging.INFO, logger=batch_policy.__name__):+ policy.verified_drafts(20, np.array([15], dtype=np.int32))+ policy.suspend(64)+ now[0] += batch_policy.LOG_INTERVAL_S+ policy.suspend(64)+ lines = [record.getMessage() for record in caplog.records]+ assert lines == [+ "eagle_harness: verify_budget trimmed 1 of 1 draft batches to their batch size's width; "+ "suspend_drafting suspended 0 batches",+ "eagle_harness: verify_budget trimmed 1 of 1 draft batches to their batch size's width; "+ "suspend_drafting suspended 2 batches",+ ]+++def _fake_vllm(monkeypatch, base):+ module = types.ModuleType("vllm.v1.worker.gpu.spec_decode.speculator")+ module.BaseSpeculator = base+ monkeypatch.setitem(sys.modules, module.__name__, module)+++def test_host_hooks_are_found_by_name_and_parameters(monkeypatch):+ class Release:+ pass++ class ReviewPr3:+ def suspend_drafting(self, num_reqs, req_ids):+ return False++ def verify_budget(self, req_ids, num_draft_tokens):+ return num_draft_tokens++ class Renamed:+ def suspend_drafting(self, batch_size, req_ids):+ return False++ _fake_vllm(monkeypatch, Release)+ assert host_speculator_hooks() == frozenset()+ _fake_vllm(monkeypatch, ReviewPr3)+ assert host_speculator_hooks() == HOOKS+ _fake_vllm(monkeypatch, Renamed)+ assert host_speculator_hooks() == frozenset()+++def test_installed_host_hooks_are_recognized_when_present():+ speculator = pytest.importorskip("vllm.v1.worker.gpu.spec_decode.speculator")+ present = {name for name in batch_policy.SPECULATOR_HOOKS if hasattr(speculator.BaseSpeculator, name)}+ # a hook vLLM has under another signature would be silently ignored: pin the contract+ assert host_speculator_hooks() == present++[email protected]("hooks", [frozenset(), HOOKS], ids=["release-engine", "hook-engine"])+def test_native_ranges_reach_the_host_its_own_way(tmp_path, hooks):+ with mock.patch.object(batch_policy, "host_speculator_hooks", return_value=hooks):+ config = _native_schedule_config(tmp_path, capable=not hooks)+ ranges = native_decode_ranges(config)+ assert ranges and ranges[0][0] >= 1+ if hooks:+ assert not hasattr(config, "native_decode_batch_sizes") # the speculator answers suspend_drafting+ else:+ assert config.native_decode_batch_sizes == ranges+++def test_neither_hook_nor_field_refuses_native_ranges(tmp_path):+ with mock.patch.object(batch_policy, "host_speculator_hooks", return_value=frozenset()):+ with pytest.raises(ValueError, match="suspend_drafting"):+ _native_schedule_config(tmp_path, capable=False)++[email protected]("mode,schedule", [("static", None), ("serve", [(1, 65536, 0)])])+def test_native_ranges_are_empty_unless_configure_installed_the_schedule(tmp_path, mode, schedule):+ with mock.patch.object(batch_policy, "host_speculator_hooks", return_value=HOOKS):+ config = _native_schedule_config(tmp_path, capable=False, schedule=schedule, mode=mode)+ assert native_decode_ranges(config) == []+++def test_native_ranges_survive_a_serialized_schedule(tmp_path):+ with mock.patch.object(batch_policy, "host_speculator_hooks", return_value=HOOKS):+ config = _native_schedule_config(tmp_path, capable=False)+ expected = native_decode_ranges(config)+ as_lists = SimpleNamespace(+ method_config=dict(config.method_config),+ num_speculative_tokens_per_batch_size=[list(entry) for entry in config.num_speculative_tokens_per_batch_size],+ )+ assert native_decode_ranges(as_lists) == expected+ configure_speculative_config(config) # reconstructing keeps the schedule and the ranges+ assert native_decode_ranges(config) == expected+++def test_speculator_hooks_delegate_to_its_batch_policy():+ pytest.importorskip("vllm")+ from eagle_harness.vllm.speculator import HarnessSpeculator++ speculator = HarnessSpeculator.__new__(HarnessSpeculator)+ speculator.batch_policy = BatchPolicy([(4, 8)], SCHEDULE, 64)+ assert speculator.suspend_drafting(4, ["r"] * 4) and not speculator.suspend_drafting(3, ["r"] * 3)+ trimmed = speculator.verify_budget([f"r{i}" for i in range(20)], np.full(20, 15, dtype=np.int32))+ assert trimmed.tolist() == [3] * 20