Skip to content

Official Inference Code of EAGLE-4

9.3 KB
Raw Blame
# Copyright 2026 Eagle AI Corp.
# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available

"""The TokenSpeed plugin against the installed TokenSpeed: hook check, registration, the draft model's profile.

No GPU work, but TokenSpeed imports only on a host with a GPU (its kernel package refuses otherwise), so this runs in
the GPU test jobs and is skipped elsewhere. On a TokenSpeed without the adapter's hooks (:mod:`eagle_harness.tokenspeed.hooks`: the release engine or a
capability engine) the registration tests check that the plugin switches itself off instead.
"""

from __future__ import annotations

import sys
from importlib.metadata import entry_points

import pytest

try:
    import tokenspeed.runtime.plugins.registry  # noqa: F401
except (ImportError, RuntimeError) as exc:  # RuntimeError: tokenspeed-kernel on a host without a GPU
    pytest.skip(f"TokenSpeed is not importable here: {exc}", allow_module_level=True)

from tokenspeed.runtime.configs.model_profile import ModelProfile
from tokenspeed.runtime.execution import drafter as drafter_registry
from tokenspeed.runtime.plugins import registry

from eagle_harness import compat
from eagle_harness.models.eagle4_config import EAGLE4_ARCH
from eagle_harness.tokenspeed import plugin
from eagle_harness.tokenspeed.config import ALGORITHM_NAME
from eagle_harness.tokenspeed.hooks import hook_problems

HAS_HOOKS = not hook_problems()
needs_hooks = pytest.mark.skipif(
    not HAS_HOOKS, reason=f"TokenSpeed without the adapter's hooks: {hook_problems()}"
)


@pytest.fixture
def registered():
    drafters, models = dict(drafter_registry._PLUGIN_DRAFTERS), dict(registry._MODELS)
    assert plugin.register() is True
    yield
    drafter_registry._PLUGIN_DRAFTERS.clear()
    drafter_registry._PLUGIN_DRAFTERS.update(drafters)
    registry._MODELS.clear()
    registry._MODELS.update(models)


def test_entry_point_targets_register():
    (ep,) = [ep for ep in entry_points(group="tokenspeed.plugins") if ep.name == "eagle_harness"] or [None]
    if ep is None:
        pytest.skip("eagle-harness is not installed with its entry points here")
    assert ep.value == "eagle_harness.tokenspeed.plugin:register"


@needs_hooks
def test_register_adds_the_drafter_and_the_draft_model(registered):
    from eagle_harness.tokenspeed.drafter import HarnessDrafter
    from eagle_harness.tokenspeed.models import LlamaForCausalLMEagle4

    assert drafter_registry.get_drafter_impl(ALGORITHM_NAME, None) is HarnessDrafter
    assert drafter_registry.drafter_captures_aux_hidden_states(ALGORITHM_NAME)
    assert registry.registered_model(EAGLE4_ARCH).cls is LlamaForCausalLMEagle4


@needs_hooks
def test_draft_profile_is_a_valid_mha_profile():
    from eagle_harness.tokenspeed.models import LlamaForCausalLMEagle4

    profile = LlamaForCausalLMEagle4.model_profile(None)
    assert isinstance(profile, ModelProfile) and profile.cache_family == "mha" and profile.linear_attention is None


@needs_hooks
def test_the_engine_calls_the_harness_tree_policy():
    # HarnessDrafter defines the public names and aliases the release ones; the installed engine must call one set.
    from tokenspeed.runtime.execution.drafter.eagle import Eagle

    from eagle_harness.tokenspeed.drafter import HarnessDrafter
    from eagle_harness.tokenspeed.hooks import PUBLIC_POLICY, RELEASE_POLICY, tree_policy
    from eagle_harness.tokenspeed.tree_round.dynamic import DynamicDrafter

    names = tree_policy()
    for public, release in zip(PUBLIC_POLICY, RELEASE_POLICY):
        assert getattr(HarnessDrafter, release) is getattr(HarnessDrafter, public)
    assert hasattr(DynamicDrafter, "round_candidates")
    for name in names[1:]:
        assert getattr(HarnessDrafter, name) is not getattr(Eagle, name)


@needs_hooks
def test_a_wrapped_moe_target_is_moe(monkeypatch):
    # Experts under text_config (a multimodal wrapper) count, as in core.models.target_is_moe.
    from types import SimpleNamespace

    from eagle_harness.tokenspeed.drafter import HarnessDrafter
    from eagle_harness.tokenspeed.tree_round.dynamic import DynamicDrafter

    monkeypatch.setattr(DynamicDrafter, "wire_target", lambda self, target: None)
    drafter = HarnessDrafter.__new__(HarnessDrafter)
    drafter.wire_target(SimpleNamespace(config=SimpleNamespace(text_config=SimpleNamespace(num_experts=64))))
    assert drafter._target_is_moe is True


@needs_hooks
@pytest.mark.parametrize("tp_size", [1, 4])
def test_calibration_uses_the_engine_device_process_group(tp_size, tmp_path, monkeypatch):
    from types import SimpleNamespace

    import torch
    from tokenspeed.runtime.distributed import process_group_manager as groups

    from eagle_harness.tokenspeed.config import parse_algorithm_config
    from eagle_harness.tokenspeed.drafter import HarnessDrafter
    from eagle_harness.tokenspeed.measurement import MeasurePolicy

    manager = groups.ProcessGroupManager()
    ranks, process_group = tuple(range(tp_size)), object()
    manager.register_process_group("nccl", ranks, process_group)
    monkeypatch.setattr(groups, "process_group_manager", manager)
    monkeypatch.setattr(torch.cuda, "Event", lambda **kwargs: object())
    drafter = HarnessDrafter.__new__(HarnessDrafter)
    drafter.device = torch.device("cpu")
    drafter.config = parse_algorithm_config({"mode": "calibrate", "data_path": str(tmp_path)})
    drafter.draft_model_runner = SimpleNamespace(mapping=SimpleNamespace(
        rank=0, attn=SimpleNamespace(tp_size=tp_size, tp_group=ranks)))

    policy = drafter._make_policy(16, 4, 5)

    assert isinstance(policy, MeasurePolicy)
    assert drafter.measurement.tp_group is (process_group if tp_size > 1 else None)


@needs_hooks
def test_compat_reports_the_hooks_ok():
    assert compat.check("tokenspeed").ok


def test_register_is_off_without_the_hooks(monkeypatch):
    # The registry module fails to import, as on a TokenSpeed without plugins: hook_problems names it.
    monkeypatch.setitem(sys.modules, "tokenspeed.runtime.plugins.registry", None)
    assert any("registry" in p for p in hook_problems())
    assert plugin.register() is False


def test_hook_problems_name_a_missing_parameter(monkeypatch):
    def register_drafter(algorithm, drafter_cls, *, model_cls=None):  # main's signature without H2
        raise AssertionError("never called")

    monkeypatch.setattr(registry, "register_drafter", register_drafter)
    assert any("captures_aux_hidden_states" in p for p in hook_problems())


@needs_hooks
def test_candidate_gather_rides_the_last_dim_as_16_bit_words(monkeypatch):
    """The TP candidate exchange asks the engine for a last-dim gather of 16-bit rows (its barrier-free Lamport
    path, which keeps the draft's device-launched round graphs free of NCCL) and unpacks rank-major columns."""
    import torch

    from eagle_harness.tokenspeed import models

    tp, rows, n = 4, 6, 8
    g = torch.Generator().manual_seed(7)
    shards = [torch.randint(-(2**31), 2**31 - 1, (rows, n), dtype=torch.int32, generator=g) for _ in range(tp)]
    calls = []

    def all_gather(tensor, group, dim):
        calls.append((tensor.dtype, tensor.shape, group, dim))
        return torch.cat([s.view(tensor.dtype) for s in shards], dim=-1)

    monkeypatch.setattr(models, "all_gather", all_gather)
    head = models.LlamaForCausalLMEagle4.__new__(models.LlamaForCausalLMEagle4)
    head.tp_size, head.tp_group = tp, "tp"
    out = head._gather_candidates(shards[0])
    assert calls == [(torch.int16, (rows, 2 * n), "tp", -1)]
    assert out.shape == (tp, rows, n) and out.dtype == torch.int32
    assert all(torch.equal(out[r], shards[r]) for r in range(tp))


def test_candidate_head_projects_through_the_engine_linear(monkeypatch):
    """The rank-r projection runs through the engine's linear (its GEMM dispatch picks the low-latency kernel for the
    few draft rows), and the shard scores are that projection times the score head, ``-inf`` past the real rows."""
    import torch

    from eagle_harness.tokenspeed import models

    rows, hidden, rank, shard_vocab, real_rows = 3, 16, 4, 10, 8
    g = torch.Generator().manual_seed(11)
    normed = torch.randn(rows, hidden, generator=g)
    down = torch.randn(rank, hidden, generator=g)
    score_head = torch.randn(shard_vocab, rank, generator=g)
    projected = []

    def project(x):
        projected.append(x)
        return x @ down.t(), None

    head = models.LlamaForCausalLMEagle4.__new__(models.LlamaForCausalLMEagle4)
    torch.nn.Module.__init__(head)
    head.efficient_lm_head_down_proj = project
    head.efficient_lm_head = torch.nn.Module()
    head.efficient_lm_head.weight = torch.nn.Parameter(score_head)
    head.lm_head = torch.nn.Module()
    head.lm_head.shard_indices = type("Shard", (), {"org_vocab_start_index": 0, "org_vocab_end_index": real_rows})()
    head.lm_head.weight = torch.nn.Parameter(torch.zeros(shard_vocab, hidden))
    head.num_candidates, head.tp_size, head.cuda_candidate_kernels = 4, 1, False
    seen = {}

    def candidate_head(scores, *args, **kwargs):
        seen["scores"] = scores
        return None, None, None

    monkeypatch.setattr(models, "fused_candidate_head", candidate_head)
    head.candidate_topk(normed, 2)
    assert len(projected) == 1 and projected[0] is normed
    expected = (normed @ down.t()) @ score_head.t()
    assert torch.allclose(seen["scores"][:, :real_rows], expected[:, :real_rows])
    assert torch.isinf(seen["scores"][:, real_rows:]).all()