Official Inference Code of EAGLE-4
# Copyright 2026 Eagle AI Corp.
# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available
"""The TokenSpeed adapter on CPU, without TokenSpeed: config, hook check, lane masks, and the verify window.
The window TokenSpeed verifies must be the tree the sglang and vLLM adapters verify for the same draft outputs: the
same nodes, tokens and parents (``test_window_verifies_what_sglang_and_vllm_verify``). The lane masks must give each
draft row exactly its own and its ancestors' window slots.
"""
import dataclasses
import sys
import types
import pytest
import torch
from eagle_harness.core.config import Mode
from eagle_harness.core.draft_tree import DraftTree, TreeShape
from eagle_harness.tokenspeed import hooks
from eagle_harness.tokenspeed.config import parse_algorithm_config
from eagle_harness.tokenspeed.tree_lanes import (
advance_lane_masks,
lane_bits,
parent_lanes,
seed_lane_masks,
verify_window,
)
K, STEPS, H, BS, WIDTH = 4, 5, 8, 3, 16
# ------------------------------------------------------------------ config
@pytest.mark.parametrize("algorithm_config", [None, {}])
def test_serve_is_the_default_and_needs_costs(algorithm_config):
# The same default as a config that omits "mode": serve, which refuses to run without its cost inputs.
with pytest.raises(ValueError, match=r"serve mode \(the default\) needs data_path.*\"mode\": \"static\""):
parse_algorithm_config(algorithm_config)
assert parse_algorithm_config({"data_path": "/calibration"}).mode is Mode.SERVE
def test_static_is_chosen_explicitly():
assert parse_algorithm_config({"mode": "static"}).mode is Mode.STATIC
def test_static_config_parses_harness_keys():
config = parse_algorithm_config({"mode": "static", "draft_model": {"cuda_candidate_kernels": False}})
assert config.mode is Mode.STATIC and config.draft_model.cuda_candidate_kernels is False
def test_serve_uses_device_policy_with_calibration():
config = parse_algorithm_config({"mode": "serve", "data_path": "/calibration"})
assert config.mode is Mode.SERVE and config.cost_dir == "/calibration/cost"
@pytest.mark.parametrize("sampled", [False, True])
def test_sampled_proposals_are_explicit_harness_configuration(sampled):
config = parse_algorithm_config({"mode": "serve", "data_path": "/calibration", "sampled_proposals": sampled})
assert config.sampled_proposals is sampled
assert not parse_algorithm_config({"mode": "static"}).sampled_proposals
@pytest.mark.parametrize("value", [None, 1, "true"])
def test_sampled_proposals_rejects_non_boolean_values(value):
with pytest.raises(ValueError, match="sampled_proposals must be a boolean"):
parse_algorithm_config({"mode": "serve", "data_path": "/calibration", "sampled_proposals": value})
@pytest.mark.parametrize("mode", ["static", "calibrate"])
def test_sampled_proposals_requires_serve_mode(mode):
with pytest.raises(ValueError, match="sampled_proposals requires serve mode"):
parse_algorithm_config({"mode": mode, "data_path": "/calibration", "sampled_proposals": True})
def test_calibrate_mode_needs_an_output_directory():
assert parse_algorithm_config({"mode": "calibrate", "data_path": "/nonexistent"}).mode is Mode.CALIBRATE
with pytest.raises(ValueError, match="data_path"):
parse_algorithm_config({"mode": "calibrate"})
@pytest.mark.parametrize(
"overrides",
[
{"intensity": "chain_intensity"},
{"draft_family": "eagle3"},
{"cost_model": {"threshold": 3.0}},
],
)
def test_device_policy_preserves_chain_family_and_custom_cost_config(overrides):
config = parse_algorithm_config({"mode": "serve", "data_path": "/calibration", **overrides})
if "cost_model" in overrides:
assert config.cost_model.threshold == 3.0
else:
key, value = next(iter(overrides.items()))
assert getattr(config, key).value == value
def test_chain_width_schedule_is_immutable_and_preserved():
table = [[1, 16], [64, 4]]
config = parse_algorithm_config({
"mode": "serve", "data_path": "/calibration", "intensity": "chain_intensity",
"chain_width_table": table,
})
table[0][1] = 2
assert config.chain_width_table == ((1, 16), (64, 4))
@pytest.mark.parametrize("table", [None, "1:4", [[1, 1]], [[True, 4]], [[8, 4], [1, 8]], [[1, 4, 8]]])
def test_invalid_chain_width_schedule_is_rejected(table):
with pytest.raises(ValueError, match="chain_width_table"):
parse_algorithm_config({
"mode": "serve", "data_path": "/calibration", "intensity": "chain_intensity",
"chain_width_table": table,
})
def test_tree_mode_rejects_chain_width_schedule():
with pytest.raises(ValueError, match="chain_intensity"):
parse_algorithm_config({
"mode": "serve", "data_path": "/calibration", "chain_width_table": [[1, 4]],
})
@pytest.mark.parametrize("mode", [{}, {"mode": "serve"}])
def test_complete_chain_schedule_needs_no_mode_and_no_calibration(mode):
# Serve is the default, so a complete schedule without "mode" is the same launch as one with it.
config = parse_algorithm_config({"intensity": "chain_intensity", "chain_width_table": [[1, 4], [32, 2]], **mode})
assert config.mode is Mode.SERVE and not config.data_path and config.fully_scheduled
def test_only_a_schedule_from_batch_size_one_is_complete():
config = parse_algorithm_config({
"data_path": "/calibration", "intensity": "chain_intensity", "chain_width_table": [[2, 4], [32, 2]],
})
assert not config.fully_scheduled
assert not parse_algorithm_config({"data_path": "/calibration"}).fully_scheduled
def test_complete_chain_schedule_needs_no_calibration():
config = parse_algorithm_config({
"mode": "serve", "intensity": "chain_intensity", "chain_width_table": [[1, 8], [64, 4]],
})
assert config.mode is Mode.SERVE
assert not config.data_path
assert config.chain_width_table == ((1, 8), (64, 4))
with pytest.raises(ValueError, match="data_path"):
parse_algorithm_config({
"mode": "serve", "intensity": "chain_intensity", "chain_width_table": [[64, 4]],
})
@pytest.mark.parametrize("overrides", [{}, {"intensity": "chain_intensity"}])
def test_serve_without_live_cost_capability_requires_calibration(overrides):
with pytest.raises(ValueError, match="data_path"):
parse_algorithm_config({"mode": "serve", **overrides})
def test_unknown_keys_are_refused():
with pytest.raises(ValueError):
parse_algorithm_config({"mode": "static", "node_top_k": 4})
# ------------------------------------------------------------------ hook check
@pytest.fixture
def fake_host(monkeypatch):
"""A host module ``fakehost_api`` with a drafter registry and a dataclass field, checked through HOOKS."""
module = types.ModuleType("fakehost_api")
def register_drafter(algorithm, drafter_cls, *, model_cls=None, captures_aux_hidden_states=False):
pass
@dataclasses.dataclass
class Context:
gather_ids: object # a field without a class-level default
module.register_drafter, module.Context = register_drafter, Context
monkeypatch.setitem(sys.modules, "fakehost_api", module)
return module
def test_hook_check_passes_a_host_with_every_hook(fake_host, monkeypatch):
monkeypatch.setattr(
hooks,
"HOOKS",
(
hooks.Hook("fakehost_api:register_drafter", ("algorithm", "captures_aux_hidden_states")),
hooks.Hook("fakehost_api:Context.gather_ids"),
),
)
assert [hooks._problem(h) for h in hooks.HOOKS] == [None, None]
def test_hook_check_names_what_is_missing(fake_host):
assert hooks._problem(hooks.Hook("fakehost_api:register_model")) == "fakehost_api:register_model"
assert "no parameter override" in hooks._problem(hooks.Hook("fakehost_api:register_drafter", ("override",)))
assert "not importable" in hooks._problem(hooks.Hook("nosuchhost.api:x"))
@pytest.fixture
def fake_plugins(monkeypatch):
"""A ``tokenspeed.runtime.plugins`` module whose every pinned hook resolves; returns it for the test to shape."""
packages = {name: types.ModuleType(name) for name in ("tokenspeed", "tokenspeed.runtime")}
plugins = types.ModuleType("tokenspeed.runtime.plugins")
for name, module in {**packages, plugins.__name__: plugins}.items():
monkeypatch.setitem(sys.modules, name, module)
monkeypatch.setattr(hooks, "_problem", lambda hook: None)
return plugins
def test_capability_contract_registers_with_every_capability(fake_plugins):
fake_plugins.PLUGIN_API_VERSION = 3
fake_plugins.PLUGIN_CAPABILITIES = hooks.REQUIRED_CAPABILITIES | {"drafter.suspend_drafting"}
assert hooks.contract() == "capabilities"
assert hooks.hook_problems() == []
assert hooks.tree_policy() == hooks.PUBLIC_POLICY
def test_capability_contract_names_a_missing_capability(fake_plugins):
fake_plugins.PLUGIN_API_VERSION = 3
fake_plugins.PLUGIN_CAPABILITIES = hooks.REQUIRED_CAPABILITIES - {"drafter.round_protocol"}
assert hooks.hook_problems() == ["PLUGIN_CAPABILITIES lacks drafter.round_protocol"]
def test_capability_contract_refuses_a_breaking_api_version(fake_plugins):
# A capability engine's version counts breaking changes only: a higher one is not "newer and compatible".
fake_plugins.PLUGIN_API_VERSION = 4
fake_plugins.PLUGIN_CAPABILITIES = hooks.REQUIRED_CAPABILITIES
(problem,) = hooks.hook_problems()
assert "PLUGIN_API_VERSION 4" in problem and "breaking" in problem
def test_release_contract_takes_the_release_names(fake_plugins):
fake_plugins.PLUGIN_API_VERSION = 4
assert hooks.contract() == "release"
assert hooks.hook_problems() == []
assert hooks.tree_policy() == hooks.RELEASE_POLICY
def test_release_contract_refuses_an_older_api_version(fake_plugins):
# Without capabilities, version 3 is the plugin API of TokenSpeed main before the eagle-harness hooks.
fake_plugins.PLUGIN_API_VERSION = 3
assert hooks.hook_problems() == ["PLUGIN_API_VERSION 3 < 4"]
def test_each_contract_pins_its_own_tree_policy_names():
capability = {h.path.rpartition(".")[2] for h in hooks.CAPABILITY_HOOKS}
release = {h.path.rpartition(".")[2] for h in hooks.RELEASE_HOOKS}
assert set(hooks.PUBLIC_POLICY[1:]) <= capability and not set(hooks.RELEASE_POLICY) & capability
assert set(hooks.RELEASE_POLICY[1:]) <= release and not set(hooks.PUBLIC_POLICY) & release
def test_the_drafter_aliases_every_release_name_to_its_public_policy():
# drafter.py imports TokenSpeed, so read its class body instead of importing it.
import ast
from pathlib import Path
tree = ast.parse(Path(hooks.__file__).with_name("drafter.py").read_text())
(cls,) = [node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "HarnessDrafter"]
methods = {node.name for node in cls.body if isinstance(node, ast.FunctionDef)}
aliases = {
node.targets[0].id: node.value.id
for node in cls.body
if isinstance(node, ast.Assign) and isinstance(node.value, ast.Name)
}
assert set(hooks.PUBLIC_POLICY) <= methods
assert aliases == dict(zip(hooks.RELEASE_POLICY, hooks.PUBLIC_POLICY))
# ------------------------------------------------------------------ the tree as TokenSpeed lanes
def _drafted(seed=0):
"""A tree grown as HarnessDrafter grows it, with TokenSpeed's lane masks, plus each node's parent node and the
window slot each node's draft row wrote (depth-1 lanes at step 1, ...; -1 for nodes never forwarded)."""
tree = DraftTree(TreeShape(top_k=K, max_depth=STEPS), BS, H, torch.float32, torch.device("cpu"), max_verify=1)
bits = lane_bits(K, STEPS, torch.device("cpu"))
lane_mask = torch.zeros(BS * K, dtype=torch.int64)
gen = torch.Generator().manual_seed(seed)
tree.stage_root(BS, torch.rand(BS, K, generator=gen).softmax(-1), torch.randint(0, 999, (BS, K), generator=gen),
torch.randn(BS, H, generator=gen)) # fmt: skip
tree.begin(BS)
seed_lane_masks(lane_mask, BS, bits)
parent_of = torch.full((BS, tree.shape.capacity), -1, dtype=torch.int64)
frontier = torch.arange(K).repeat(BS, 1) # node id of each lane
slot_of = {} # (b, node) -> window slot its row wrote
masks = [] # per step: (frontier nodes, lane masks) the step's rows ran with
for step in range(1, STEPS):
for b in range(BS):
for lane in range(K):
slot_of[b, int(frontier[b, lane])] = (step - 1) * K + lane
masks.append((frontier.clone(), lane_mask.view(BS, K).clone()))
probs = torch.rand(BS * K, K, generator=gen).softmax(-1)
tokens = torch.randint(0, 999, (BS * K, K), generator=gen)
tree.step(BS, step, probs, tokens, torch.randn(BS * K, H, generator=gen))
start = tree.shape.layer_start(step + 1)
parent_of[:, start : start + K * K] = frontier.repeat_interleave(K, dim=1)
first = tree.shape.num_parents(step + 1)
new_frontier = tree.parents[:BS, first : first + K].clone()
assert torch.equal(torch.gather(frontier, 1, parent_lanes(tree, BS, step)), parent_of.gather(1, new_frontier))
if step < STEPS - 1:
advance_lane_masks(tree, lane_mask, BS, step, bits)
frontier = new_frontier
return tree, parent_of, slot_of, masks
def _ancestors(parent_of, b, node):
while node >= 0:
yield node
node = int(parent_of[b, node])
@pytest.mark.parametrize("seed", [0, 1, 2])
def test_lane_masks_show_a_row_its_ancestors_and_itself(seed):
_, parent_of, slot_of, masks = _drafted(seed)
for frontier, lane_mask in masks:
for b in range(BS):
for lane in range(K):
want = sum(1 << slot_of[b, n] for n in _ancestors(parent_of, b, int(frontier[b, lane])))
assert int(lane_mask[b, lane]) & ((1 << 64) - 1) == want
def test_lane_bits_use_bit_63_as_the_sign():
bits = lane_bits(8, 9, torch.device("cpu")) # (9 - 1) * 8 = 64 window slots
assert int(bits[7, 7]) == -(1 << 63) and int(bits[0, 0]) == 1
with pytest.raises(ValueError, match="64-bit"):
lane_bits(8, 10, torch.device("cpu"))
@pytest.mark.parametrize("seed", [0, 1, 2])
def test_window_verifies_what_sglang_and_vllm_verify(seed):
tree, parent_of, _, _ = _drafted(seed)
roots = torch.tensor([11, 22, 33], dtype=torch.int32)
window = torch.zeros(BS, WIDTH, dtype=torch.int32)
parents = torch.zeros(BS, WIDTH, dtype=torch.int32)
verify_window(tree, BS, STEPS, roots, window, parents)
# sglang's verify input for the same tree: the ranked nodes, their tokens in node order (DraftTree.select)
ranked, _ = tree.rank(BS, STEPS, WIDTH - 1)
sglang = tree.select(BS, STEPS, ranked, WIDTH)
assert torch.equal(window[:, 0], roots)
assert torch.equal(window[:, 1:].long(), sglang.draft_tokens)
for b in range(BS):
nodes = sglang.top_scores_index[b].tolist()
assert parents[b, 0] == -1
for row, node in enumerate(nodes, start=1):
parent = int(parent_of[b, node])
assert int(parents[b, row]) == (0 if parent < 0 else 1 + nodes.index(parent)) < row
# vLLM's verify rows for the same tree (TreeDrafter.select = DraftTree.verify_rows)
tokens, parent_rows = tree.verify_rows_reference(BS, STEPS, WIDTH - 1)
assert torch.equal(window[:, 1:].long(), tokens) and torch.equal(parents, parent_rows)