Skip to content

Official Inference Code of EAGLE-4

feat(vllm): answer vLLM's suspend_drafting and verify_budget hooks

Seyfal Sultanov6465fb4d925f
diff --git a/eagle_harness/vllm/batch_policy.py b/eagle_harness/vllm/batch_policy.py
new file mode 100644
index 0000000..2d710c9
--- /dev/null
+++ b/eagle_harness/vllm/batch_policy.py
@@ -0,0 +1,109 @@
+# Copyright 2026 Eagle AI Corp.
+# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available
+
+"""Per-batch serving decisions the speculator answers through vLLM's optional speculator hooks.
+
+vLLM review PR 3 lets a speculator decide two things per scheduled batch (:func:`host_speculator_hooks` finds them):
+
+- ``suspend_drafting(num_reqs, req_ids)``: native target decode (native-OFF) for the batch sizes where the priced
+ chain schedule picked width 0 (:func:`~eagle_harness.vllm.config.native_decode_ranges`).
+- ``verify_budget(req_ids, num_draft_tokens)``: how many scheduled drafts each request verifies. The scheduler sizes a
+ request's drafts from the batch size of the step that drafted them; this verifies at most the width the schedule
+ gives the batch being verified, so the measured high-load width (``served_tree_schedule``, the chain widths) applies
+ from the first step at that batch size instead of one step later. A tree's verify rows put each parent before its
+ children, so a prefix of them is a tree.
+
+On a vLLM without these hooks (the 0.1.0 release engine) nothing here runs: the native ranges go through
+``SpeculativeConfig.native_decode_batch_sizes`` and the schedule alone sets the width, as before.
+"""
+
+from __future__ import annotations
+
+import inspect
+import logging
+import time
+from typing import Callable, FrozenSet, Iterable, Optional, Sequence, Tuple
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+# hook -> its parameters after ``self``: a host whose hook has other parameters does not count as having it
+SPECULATOR_HOOKS = {
+ "suspend_drafting": ("num_reqs", "req_ids"),
+ "verify_budget": ("req_ids", "num_draft_tokens"),
+}
+LOG_INTERVAL_S = 60.0
+
+
+def host_speculator_hooks() -> FrozenSet[str]:
+ """The optional ``BaseSpeculator`` hooks this vLLM calls (empty without vLLM or on the release engine)."""
+ try:
+ from vllm.v1.worker.gpu.spec_decode.speculator import BaseSpeculator
+ except ImportError:
+ return frozenset()
+ found = set()
+ for name, params in SPECULATOR_HOOKS.items():
+ hook = getattr(BaseSpeculator, name, None)
+ if callable(hook) and tuple(inspect.signature(hook).parameters)[1:] == params:
+ found.add(name)
+ return frozenset(found)
+
+
+class BatchPolicy:
+ """Answers ``suspend_drafting`` and ``verify_budget`` from static per-batch-size tables (host only, no device
+ reads) and logs what they did at most once a minute."""
+
+ def __init__(
+ self,
+ native_ranges: Iterable[Tuple[int, int]],
+ schedule: Optional[Sequence[Sequence[int]]],
+ max_num_reqs: int,
+ *,
+ clock: Callable[[], float] = time.monotonic,
+ ):
+ self.native_ranges = tuple((int(first), int(last)) for first, last in native_ranges)
+ # drafts verified at each batch size (None: the scheduler's count stands)
+ self.width_drafts: Optional[np.ndarray] = None
+ if schedule:
+ drafts = np.full(max_num_reqs + 1, np.iinfo(np.int32).max, dtype=np.int64)
+ for first, last, count in schedule:
+ drafts[max(int(first), 0) : min(int(last), max_num_reqs) + 1] = int(count)
+ self.width_drafts = drafts
+ self.clock = clock
+ self.batches = self.suspended = self.trimmed = 0
+ self._logged: Tuple[int, int, int] = (0, 0, 0)
+ self._next_log = 0.0
+
+ def suspend(self, num_reqs: int) -> bool:
+ suspend = any(first <= num_reqs <= last for first, last in self.native_ranges)
+ if suspend:
+ self.suspended += 1
+ self._report()
+ return suspend
+
+ def verified_drafts(self, num_reqs: int, num_draft_tokens: np.ndarray) -> np.ndarray:
+ """``num_draft_tokens`` itself when nothing is trimmed (vLLM then keeps the batch as scheduled)."""
+ self.batches += 1
+ if self.width_drafts is None or not len(num_draft_tokens):
+ return num_draft_tokens
+ width = self.width_drafts[min(num_reqs, len(self.width_drafts) - 1)]
+ if int(num_draft_tokens.max()) <= width:
+ return num_draft_tokens
+ self.trimmed += 1
+ self._report()
+ return np.minimum(num_draft_tokens, width)
+
+ def _report(self) -> None:
+ now = self.clock()
+ counts = (self.batches, self.trimmed, self.suspended)
+ if now < self._next_log or counts == self._logged:
+ return
+ self._next_log, self._logged = now + LOG_INTERVAL_S, counts
+ logger.info(
+ "eagle_harness: verify_budget trimmed %d of %d draft batches to their batch size's width; "
+ "suspend_drafting suspended %d batches",
+ self.trimmed,
+ self.batches,
+ self.suspended,
+ )
diff --git a/eagle_harness/vllm/config.py b/eagle_harness/vllm/config.py
index 7d3ad9a..d7a6f36 100644
--- a/eagle_harness/vllm/config.py
+++ b/eagle_harness/vllm/config.py
@@ -63,7 +63,8 @@ DEFAULT_TREE_DEPTH = 12
DERIVED = "derived" # depth_stop_gain from the measured costs
NATIVE_DECODE_UNSUPPORTED = (
"eagle_harness: the chain schedule selects native plain decode (width 0), "
- "which requires vLLM's native_decode_batch_sizes capability. K=0 still extends draft KV. "
+ "which requires vLLM's suspend_drafting speculator hook or its native_decode_batch_sizes field. "
+ "K=0 still extends draft KV. "
"Use a compatible host, run without speculative_config, or calibrate root-only width 1."
)
@@ -224,11 +225,16 @@ def configure_speculative_config(spec_config: Any) -> None:
parsed.harness.data_path,
allow_unpriced="max_verify_width" in spec_config.method_config,
)
- native_ranges = [(first, last) for first, last, width in widths if width == 0]
+ native_ranges = _native_ranges(widths)
if native_ranges:
- if not hasattr(spec_config, "native_decode_batch_sizes"):
+ from eagle_harness.vllm.batch_policy import host_speculator_hooks
+
+ if "suspend_drafting" in host_speculator_hooks():
+ pass # HarnessSpeculator.suspend_drafting answers from the same widths (native_decode_ranges)
+ elif hasattr(spec_config, "native_decode_batch_sizes"): # the 0.1.0 release engine
+ spec_config.native_decode_batch_sizes = native_ranges
+ else:
raise ValueError(NATIVE_DECODE_UNSUPPORTED)
- spec_config.native_decode_batch_sizes = native_ranges
spec_config.num_speculative_tokens_per_batch_size = _draft_token_schedule(
widths, max_batch_size(parsed.harness.intensity)
)
@@ -248,6 +254,29 @@ def configure_speculative_config(spec_config: Any) -> None:
)
+def _native_ranges(widths: List[Tuple[int, int, int]]) -> List[Tuple[int, int]]:
+ return [(first, last) for first, last, width in widths if width == 0]
+
+
+def native_decode_ranges(spec_config: Any) -> List[Tuple[int, int]]:
+ """The batch-size ranges where the schedule :func:`configure_speculative_config` installed decodes natively
+ (width 0); empty when it installed none or the user set the schedule. Deterministic, so the speculator in each
+ worker recomputes what configure saw."""
+ parsed = parse_method_config(spec_config.method_config)
+ if not parsed.harness.mode.uses_cost_model:
+ return []
+ widths = _verify_widths(
+ parsed.harness.intensity,
+ parsed.max_verify_width,
+ parsed.harness.data_path,
+ allow_unpriced="max_verify_width" in spec_config.method_config,
+ )
+ installed = [tuple(entry) for entry in spec_config.num_speculative_tokens_per_batch_size or ()]
+ if installed != _draft_token_schedule(widths, max_batch_size(parsed.harness.intensity)):
+ return []
+ return _native_ranges(widths)
+
+
def measured_tree_schedule(parsed: VllmMethodConfig) -> List[Tuple[int, int, int]] | None:
"""The static tree's draft-count schedule from served rounds under ``data_path`` (None: one width throughout)."""
data_path = parsed.harness.data_path
diff --git a/eagle_harness/vllm/speculator.py b/eagle_harness/vllm/speculator.py
index eb7c33d..f933108 100644
--- a/eagle_harness/vllm/speculator.py
+++ b/eagle_harness/vllm/speculator.py
@@ -20,7 +20,9 @@ Tree mode (``node_top_k > 1``) runs one round as follows (vLLM's hooks in bracke
writes the next verify rows (``draft_tokens`` in node order, their parents in the :class:`TreeStore`).
vLLM's scheduler fixes the verify width of a step from the batch size (``num_speculative_tokens``), so the
-width is the batch-size schedule's, not a per-round choice. The draft depth is per round: a captured growth skips
+width is the batch-size schedule's, not a per-round choice. On a vLLM with the optional ``verify_budget`` and
+``suspend_drafting`` hooks, :class:`~eagle_harness.vllm.batch_policy.BatchPolicy` applies that width to the batch being
+verified and runs the chain schedule's native-OFF batch sizes. The draft depth is per round: a captured growth skips
the steps whose nodes could not enter the verify set (:mod:`~eagle_harness.vllm.depth_stop`).
"""
@@ -39,7 +41,8 @@ from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.v1.worker.gpu.spec_decode.eagle.speculator import EagleSpeculator
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import SpeculatorCudaGraphManager
-from eagle_harness.vllm.config import DERIVED, VllmMethodConfig, parse_method_config
+from eagle_harness.vllm.batch_policy import BatchPolicy, host_speculator_hooks
+from eagle_harness.vllm.config import DERIVED, VllmMethodConfig, native_decode_ranges, parse_method_config
from eagle_harness.vllm.depth_stop import derived_cost_tables
from eagle_harness.vllm.gdn_tree_verify import LinearStateCommit, check_tree_verifiable
from eagle_harness.vllm.round_commit import KvCompactor, chainize
@@ -57,6 +60,13 @@ class HarnessSpeculator(EagleSpeculator):
super().__init__(vllm_config, device)
self.method_config: VllmMethodConfig = parse_method_config(self.speculative_config.method_config)
self.tree_mode = self.method_config.node_top_k > 1
+ hooks = host_speculator_hooks() # empty on the 0.1.0 release engine: neither hook below is called there
+ self.batch_policy = BatchPolicy(
+ native_decode_ranges(self.speculative_config) if "suspend_drafting" in hooks else (),
+ self.speculative_config.num_speculative_tokens_per_batch_size if "verify_budget" in hooks else None,
+ self.max_num_reqs,
+ )
+ self.may_suspend_drafting = bool(self.batch_policy.native_ranges)
if self.tree_mode:
width = self.num_speculative_steps + 1
self.store = TreeStore(self.max_num_reqs, width, device)
@@ -168,6 +178,13 @@ class HarnessSpeculator(EagleSpeculator):
)
return self._bound
+ # ------------------------------------------------------------------ per-batch policy hooks
+ def suspend_drafting(self, num_reqs: int, req_ids: Any) -> bool:
+ return self.batch_policy.suspend(num_reqs)
+
+ def verify_budget(self, req_ids: Any, num_draft_tokens: Any) -> Any:
+ return self.batch_policy.verified_drafts(len(req_ids), num_draft_tokens)
+
# ------------------------------------------------------------------ verify hooks
def prepare_verify_inputs(self, input_batch: Any, model_inputs: dict[str, Any], *, dummy_run: bool = False) -> None:
if not self.tree_mode:
diff --git a/tests/test_chain_no_spec.py b/tests/test_chain_no_spec.py
index 746bebf..85f57ba 100644
--- a/tests/test_chain_no_spec.py
+++ b/tests/test_chain_no_spec.py
@@ -13,6 +13,15 @@ from eagle_harness.core.limits import ComputeIntensity
from tests.test_chain_width import PROFILE
[email protected](autouse=True)
+def release_engine_shape(monkeypatch):
+ """These tests pin the 0.1.0 release engine's contract (``native_decode_batch_sizes``, no speculator hooks)
+ whatever vLLM is installed; tests/test_vllm_batch_policy.py covers the hook engine."""
+ from eagle_harness.vllm import batch_policy
+
+ monkeypatch.setattr(batch_policy, "host_speculator_hooks", lambda: frozenset())
+
+
def test_native_decode_is_a_distinct_opt_in_action(tmp_path):
save_accept_profile(str(tmp_path), 100, [90, 80, 70])
save_round_times(
diff --git a/tests/test_vllm_batch_policy.py b/tests/test_vllm_batch_policy.py
new file mode 100644
index 0000000..2b3e196
--- /dev/null
+++ b/tests/test_vllm_batch_policy.py
@@ -0,0 +1,154 @@
+# Copyright 2026 Eagle AI Corp.
+# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available
+
+"""The per-batch policy behind vLLM's optional ``suspend_drafting`` / ``verify_budget`` speculator hooks, on both
+engine shapes: the 0.1.0 release engine (no hooks: native ranges go through ``native_decode_batch_sizes``) and review
+PR 3 (hooks: the speculator answers them)."""
+
+import logging
+import sys
+import types
+from types import SimpleNamespace
+from unittest import mock
+
+import numpy as np
+import pytest
+
+from eagle_harness.vllm import batch_policy
+from eagle_harness.vllm.batch_policy import BatchPolicy, host_speculator_hooks
+from eagle_harness.vllm.config import configure_speculative_config, native_decode_ranges
+from tests.test_chain_no_spec import _native_schedule_config
+
+HOOKS = frozenset(batch_policy.SPECULATOR_HOOKS)
+SCHEDULE = [(1, 19, 15), (20, 512, 3), (513, 1 << 16, 0)]
+
+
+def test_suspend_follows_the_native_ranges_only():
+ policy = BatchPolicy([(8, 16), (40, 64)], None, 64)
+ assert [policy.suspend(n) for n in (1, 7, 8, 16, 17, 40, 64)] == [False, False, True, True, False, True, True]
+ assert policy.suspended == 4
+ assert not BatchPolicy((), None, 64).suspend(64)
+
+
+def test_verify_budget_keeps_the_schedule_unless_the_batch_size_verifies_fewer():
+ policy = BatchPolicy((), SCHEDULE, 64)
+ drafted_small = np.array([15, 15, 15], dtype=np.int32)
+ # the batch drafted at its own batch size: vLLM keeps it as scheduled (same object)
+ assert policy.verified_drafts(3, drafted_small) is drafted_small
+ within = np.array([3, 2], dtype=np.int32)
+ assert policy.verified_drafts(20, within) is within
+ # drafted at batch size < 20 (width 16), verified in a batch of 20 (width 4)
+ trimmed = policy.verified_drafts(20, np.array([15, 3, 0], dtype=np.int32))
+ assert trimmed.tolist() == [3, 3, 0]
+ # batch sizes past max_num_reqs take the last entry
+ assert policy.verified_drafts(99, np.array([15], dtype=np.int32)).tolist() == [3]
+ assert (policy.batches, policy.trimmed) == (4, 2)
+
+
+def test_without_a_schedule_every_scheduled_draft_is_verified():
+ drafts = np.array([7, 7], dtype=np.int32)
+ assert BatchPolicy([(1, 4)], None, 64).verified_drafts(2, drafts) is drafts
+ empty = np.array([], dtype=np.int32)
+ assert BatchPolicy((), SCHEDULE, 64).verified_drafts(0, empty) is empty
+
+
+def test_policy_logs_the_first_event_then_once_a_minute(caplog):
+ now = [100.0]
+ policy = BatchPolicy([(64, 64)], SCHEDULE, 64, clock=lambda: now[0])
+ with caplog.at_level(logging.INFO, logger=batch_policy.__name__):
+ policy.verified_drafts(20, np.array([15], dtype=np.int32))
+ policy.suspend(64)
+ now[0] += batch_policy.LOG_INTERVAL_S
+ policy.suspend(64)
+ lines = [record.getMessage() for record in caplog.records]
+ assert lines == [
+ "eagle_harness: verify_budget trimmed 1 of 1 draft batches to their batch size's width; "
+ "suspend_drafting suspended 0 batches",
+ "eagle_harness: verify_budget trimmed 1 of 1 draft batches to their batch size's width; "
+ "suspend_drafting suspended 2 batches",
+ ]
+
+
+def _fake_vllm(monkeypatch, base):
+ module = types.ModuleType("vllm.v1.worker.gpu.spec_decode.speculator")
+ module.BaseSpeculator = base
+ monkeypatch.setitem(sys.modules, module.__name__, module)
+
+
+def test_host_hooks_are_found_by_name_and_parameters(monkeypatch):
+ class Release:
+ pass
+
+ class ReviewPr3:
+ def suspend_drafting(self, num_reqs, req_ids):
+ return False
+
+ def verify_budget(self, req_ids, num_draft_tokens):
+ return num_draft_tokens
+
+ class Renamed:
+ def suspend_drafting(self, batch_size, req_ids):
+ return False
+
+ _fake_vllm(monkeypatch, Release)
+ assert host_speculator_hooks() == frozenset()
+ _fake_vllm(monkeypatch, ReviewPr3)
+ assert host_speculator_hooks() == HOOKS
+ _fake_vllm(monkeypatch, Renamed)
+ assert host_speculator_hooks() == frozenset()
+
+
+def test_installed_host_hooks_are_recognized_when_present():
+ speculator = pytest.importorskip("vllm.v1.worker.gpu.spec_decode.speculator")
+ present = {name for name in batch_policy.SPECULATOR_HOOKS if hasattr(speculator.BaseSpeculator, name)}
+ # a hook vLLM has under another signature would be silently ignored: pin the contract
+ assert host_speculator_hooks() == present
+
+
[email protected]("hooks", [frozenset(), HOOKS], ids=["release-engine", "hook-engine"])
+def test_native_ranges_reach_the_host_its_own_way(tmp_path, hooks):
+ with mock.patch.object(batch_policy, "host_speculator_hooks", return_value=hooks):
+ config = _native_schedule_config(tmp_path, capable=not hooks)
+ ranges = native_decode_ranges(config)
+ assert ranges and ranges[0][0] >= 1
+ if hooks:
+ assert not hasattr(config, "native_decode_batch_sizes") # the speculator answers suspend_drafting
+ else:
+ assert config.native_decode_batch_sizes == ranges
+
+
+def test_neither_hook_nor_field_refuses_native_ranges(tmp_path):
+ with mock.patch.object(batch_policy, "host_speculator_hooks", return_value=frozenset()):
+ with pytest.raises(ValueError, match="suspend_drafting"):
+ _native_schedule_config(tmp_path, capable=False)
+
+
[email protected]("mode,schedule", [("static", None), ("serve", [(1, 65536, 0)])])
+def test_native_ranges_are_empty_unless_configure_installed_the_schedule(tmp_path, mode, schedule):
+ with mock.patch.object(batch_policy, "host_speculator_hooks", return_value=HOOKS):
+ config = _native_schedule_config(tmp_path, capable=False, schedule=schedule, mode=mode)
+ assert native_decode_ranges(config) == []
+
+
+def test_native_ranges_survive_a_serialized_schedule(tmp_path):
+ with mock.patch.object(batch_policy, "host_speculator_hooks", return_value=HOOKS):
+ config = _native_schedule_config(tmp_path, capable=False)
+ expected = native_decode_ranges(config)
+ as_lists = SimpleNamespace(
+ method_config=dict(config.method_config),
+ num_speculative_tokens_per_batch_size=[list(entry) for entry in config.num_speculative_tokens_per_batch_size],
+ )
+ assert native_decode_ranges(as_lists) == expected
+ configure_speculative_config(config) # reconstructing keeps the schedule and the ranges
+ assert native_decode_ranges(config) == expected
+
+
+def test_speculator_hooks_delegate_to_its_batch_policy():
+ pytest.importorskip("vllm")
+ from eagle_harness.vllm.speculator import HarnessSpeculator
+
+ speculator = HarnessSpeculator.__new__(HarnessSpeculator)
+ speculator.batch_policy = BatchPolicy([(4, 8)], SCHEDULE, 64)
+ assert speculator.suspend_drafting(4, ["r"] * 4) and not speculator.suspend_drafting(3, ["r"] * 3)
+ trimmed = speculator.verify_budget([f"r{i}" for i in range(20)], np.full(20, 15, dtype=np.int32))
+ assert trimmed.tolist() == [3] * 20