Official Inference Code of EAGLE-4
# Copyright 2026 Eagle AI Corp.
# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available
"""The TokenSpeed plugin against the installed TokenSpeed: hook check, registration, the draft model's profile.
No GPU work, but TokenSpeed imports only on a host with a GPU (its kernel package refuses otherwise), so this runs in
the GPU test jobs and is skipped elsewhere. On a TokenSpeed without the adapter's hooks (:mod:`eagle_harness.tokenspeed.hooks`: the release engine or a
capability engine) the registration tests check that the plugin switches itself off instead.
"""
from __future__ import annotations
import sys
from importlib.metadata import entry_points
import pytest
try:
import tokenspeed.runtime.plugins.registry # noqa: F401
except (ImportError, RuntimeError) as exc: # RuntimeError: tokenspeed-kernel on a host without a GPU
pytest.skip(f"TokenSpeed is not importable here: {exc}", allow_module_level=True)
from tokenspeed.runtime.configs.model_profile import ModelProfile
from tokenspeed.runtime.execution import drafter as drafter_registry
from tokenspeed.runtime.plugins import registry
from eagle_harness import compat
from eagle_harness.models.eagle4_config import EAGLE4_ARCH
from eagle_harness.tokenspeed import plugin
from eagle_harness.tokenspeed.config import ALGORITHM_NAME
from eagle_harness.tokenspeed.hooks import hook_problems
HAS_HOOKS = not hook_problems()
needs_hooks = pytest.mark.skipif(
not HAS_HOOKS, reason=f"TokenSpeed without the adapter's hooks: {hook_problems()}"
)
@pytest.fixture
def registered():
drafters, models = dict(drafter_registry._PLUGIN_DRAFTERS), dict(registry._MODELS)
assert plugin.register() is True
yield
drafter_registry._PLUGIN_DRAFTERS.clear()
drafter_registry._PLUGIN_DRAFTERS.update(drafters)
registry._MODELS.clear()
registry._MODELS.update(models)
def test_entry_point_targets_register():
(ep,) = [ep for ep in entry_points(group="tokenspeed.plugins") if ep.name == "eagle_harness"] or [None]
if ep is None:
pytest.skip("eagle-harness is not installed with its entry points here")
assert ep.value == "eagle_harness.tokenspeed.plugin:register"
@needs_hooks
def test_register_adds_the_drafter_and_the_draft_model(registered):
from eagle_harness.tokenspeed.drafter import HarnessDrafter
from eagle_harness.tokenspeed.models import LlamaForCausalLMEagle4
assert drafter_registry.get_drafter_impl(ALGORITHM_NAME, None) is HarnessDrafter
assert drafter_registry.drafter_captures_aux_hidden_states(ALGORITHM_NAME)
assert registry.registered_model(EAGLE4_ARCH).cls is LlamaForCausalLMEagle4
@needs_hooks
def test_draft_profile_is_a_valid_mha_profile():
from eagle_harness.tokenspeed.models import LlamaForCausalLMEagle4
profile = LlamaForCausalLMEagle4.model_profile(None)
assert isinstance(profile, ModelProfile) and profile.cache_family == "mha" and profile.linear_attention is None
@needs_hooks
def test_the_engine_calls_the_harness_tree_policy():
# HarnessDrafter defines the public names and aliases the release ones; the installed engine must call one set.
from tokenspeed.runtime.execution.drafter.eagle import Eagle
from eagle_harness.tokenspeed.drafter import HarnessDrafter
from eagle_harness.tokenspeed.hooks import PUBLIC_POLICY, RELEASE_POLICY, tree_policy
from eagle_harness.tokenspeed.tree_round.dynamic import DynamicDrafter
names = tree_policy()
for public, release in zip(PUBLIC_POLICY, RELEASE_POLICY):
assert getattr(HarnessDrafter, release) is getattr(HarnessDrafter, public)
assert hasattr(DynamicDrafter, "round_candidates")
for name in names[1:]:
assert getattr(HarnessDrafter, name) is not getattr(Eagle, name)
@needs_hooks
def test_a_wrapped_moe_target_is_moe(monkeypatch):
# Experts under text_config (a multimodal wrapper) count, as in core.models.target_is_moe.
from types import SimpleNamespace
from eagle_harness.tokenspeed.drafter import HarnessDrafter
from eagle_harness.tokenspeed.tree_round.dynamic import DynamicDrafter
monkeypatch.setattr(DynamicDrafter, "wire_target", lambda self, target: None)
drafter = HarnessDrafter.__new__(HarnessDrafter)
drafter.wire_target(SimpleNamespace(config=SimpleNamespace(text_config=SimpleNamespace(num_experts=64))))
assert drafter._target_is_moe is True
@needs_hooks
@pytest.mark.parametrize("tp_size", [1, 4])
def test_calibration_uses_the_engine_device_process_group(tp_size, tmp_path, monkeypatch):
from types import SimpleNamespace
import torch
from tokenspeed.runtime.distributed import process_group_manager as groups
from eagle_harness.tokenspeed.config import parse_algorithm_config
from eagle_harness.tokenspeed.drafter import HarnessDrafter
from eagle_harness.tokenspeed.measurement import MeasurePolicy
manager = groups.ProcessGroupManager()
ranks, process_group = tuple(range(tp_size)), object()
manager.register_process_group("nccl", ranks, process_group)
monkeypatch.setattr(groups, "process_group_manager", manager)
monkeypatch.setattr(torch.cuda, "Event", lambda **kwargs: object())
drafter = HarnessDrafter.__new__(HarnessDrafter)
drafter.device = torch.device("cpu")
drafter.config = parse_algorithm_config({"mode": "calibrate", "data_path": str(tmp_path)})
drafter.draft_model_runner = SimpleNamespace(mapping=SimpleNamespace(
rank=0, attn=SimpleNamespace(tp_size=tp_size, tp_group=ranks)))
policy = drafter._make_policy(16, 4, 5)
assert isinstance(policy, MeasurePolicy)
assert drafter.measurement.tp_group is (process_group if tp_size > 1 else None)
@needs_hooks
def test_compat_reports_the_hooks_ok():
assert compat.check("tokenspeed").ok
def test_register_is_off_without_the_hooks(monkeypatch):
# The registry module fails to import, as on a TokenSpeed without plugins: hook_problems names it.
monkeypatch.setitem(sys.modules, "tokenspeed.runtime.plugins.registry", None)
assert any("registry" in p for p in hook_problems())
assert plugin.register() is False
def test_hook_problems_name_a_missing_parameter(monkeypatch):
def register_drafter(algorithm, drafter_cls, *, model_cls=None): # main's signature without H2
raise AssertionError("never called")
monkeypatch.setattr(registry, "register_drafter", register_drafter)
assert any("captures_aux_hidden_states" in p for p in hook_problems())
@needs_hooks
def test_candidate_gather_rides_the_last_dim_as_16_bit_words(monkeypatch):
"""The TP candidate exchange asks the engine for a last-dim gather of 16-bit rows (its barrier-free Lamport
path, which keeps the draft's device-launched round graphs free of NCCL) and unpacks rank-major columns."""
import torch
from eagle_harness.tokenspeed import models
tp, rows, n = 4, 6, 8
g = torch.Generator().manual_seed(7)
shards = [torch.randint(-(2**31), 2**31 - 1, (rows, n), dtype=torch.int32, generator=g) for _ in range(tp)]
calls = []
def all_gather(tensor, group, dim):
calls.append((tensor.dtype, tensor.shape, group, dim))
return torch.cat([s.view(tensor.dtype) for s in shards], dim=-1)
monkeypatch.setattr(models, "all_gather", all_gather)
head = models.LlamaForCausalLMEagle4.__new__(models.LlamaForCausalLMEagle4)
head.tp_size, head.tp_group = tp, "tp"
out = head._gather_candidates(shards[0])
assert calls == [(torch.int16, (rows, 2 * n), "tp", -1)]
assert out.shape == (tp, rows, n) and out.dtype == torch.int32
assert all(torch.equal(out[r], shards[r]) for r in range(tp))
def test_candidate_head_projects_through_the_engine_linear(monkeypatch):
"""The rank-r projection runs through the engine's linear (its GEMM dispatch picks the low-latency kernel for the
few draft rows), and the shard scores are that projection times the score head, ``-inf`` past the real rows."""
import torch
from eagle_harness.tokenspeed import models
rows, hidden, rank, shard_vocab, real_rows = 3, 16, 4, 10, 8
g = torch.Generator().manual_seed(11)
normed = torch.randn(rows, hidden, generator=g)
down = torch.randn(rank, hidden, generator=g)
score_head = torch.randn(shard_vocab, rank, generator=g)
projected = []
def project(x):
projected.append(x)
return x @ down.t(), None
head = models.LlamaForCausalLMEagle4.__new__(models.LlamaForCausalLMEagle4)
torch.nn.Module.__init__(head)
head.efficient_lm_head_down_proj = project
head.efficient_lm_head = torch.nn.Module()
head.efficient_lm_head.weight = torch.nn.Parameter(score_head)
head.lm_head = torch.nn.Module()
head.lm_head.shard_indices = type("Shard", (), {"org_vocab_start_index": 0, "org_vocab_end_index": real_rows})()
head.lm_head.weight = torch.nn.Parameter(torch.zeros(shard_vocab, hidden))
head.num_candidates, head.tp_size, head.cuda_candidate_kernels = 4, 1, False
seen = {}
def candidate_head(scores, *args, **kwargs):
seen["scores"] = scores
return None, None, None
monkeypatch.setattr(models, "fused_candidate_head", candidate_head)
head.candidate_topk(normed, 2)
assert len(projected) == 1 and projected[0] is normed
expected = (normed @ down.t()) @ score_head.t()
assert torch.allclose(seen["scores"][:, :real_rows], expected[:, :real_rows])
assert torch.isinf(seen["scores"][:, real_rows:]).all()