Skip to content

Official Inference Code of EAGLE-4

14.3 KB
Raw Blame
# Copyright 2026 Eagle AI Corp.
# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available

"""Real dynamic-tree kernels: eager depths and GPU-tail-launched rounds agree."""

from __future__ import annotations

import contextlib
import sys
from dataclasses import replace
from types import SimpleNamespace

import pytest
import torch

pytest.importorskip("tvm_ffi", reason="TokenSpeed round kernels require apache-tvm-ffi")

# Keep the CPU suites independent of TokenSpeed's GPU-only package imports.
if not torch.cuda.is_available():
    pytest.skip("needs CUDA", allow_module_level=True)

pytest.importorskip("tokenspeed", reason="requires the optional TokenSpeed engine")

from tokenspeed.runtime.layers.attention.backends.paged.tree_verify import (
    TreeDraftInputs,
)

pytest.importorskip("eagle_harness")

from eagle_harness.core.config import HarnessConfig, Mode
from eagle_harness.core.cost_table import BatchCostTable, CostCurve, IntervalCost
from eagle_harness.tokenspeed.tree_round import dynamic as drafter_module
from eagle_harness.tokenspeed.tree_round.device_policy import DevicePolicy
from eagle_harness.tokenspeed.tree_round.dynamic import DynamicDrafter
from tests.tokenspeed_round.core_costs import policy_costs


class EagleHarness(DynamicDrafter):
    @property
    def dynamic(self):
        return True


BS, NODES, WINDOW, VERIFY_COUNT = 2, 9, 16, 4
TREE_K, TREE_DEPTH, HIDDEN = 2, 6, 8


class _OneDepthPolicy:
    """Drafts depth 1 only and verifies two of its ranked nodes."""

    tree_depth = 2
    chain = False

    def __init__(self) -> None:
        int32 = {"dtype": torch.int32, "device": "cuda"}
        self.cumu_count = torch.tensor([NODES], **int32)
        self.verify_count = torch.tensor([VERIFY_COUNT], **int32)
        self.verify_num = torch.tensor([2], **int32)

    def observe_batch_size(self, batch_size: int) -> None:
        pass

    def deltas_depth(self, depth: int) -> bool:
        return False

    def decide(self, depth: int, work_deltas: torch.Tensor) -> None:
        pass

    def go_ready(self, depth: int, round_seq: int) -> bool:
        return depth < self.tree_depth

    def select_verify(self, top_scores: torch.Tensor) -> None:
        pass


def _drafter() -> EagleHarness:
    drafter = EagleHarness.__new__(EagleHarness)
    drafter.policy = _OneDepthPolicy()
    drafter.config = HarnessConfig(mode=Mode.SERVE, data_path="/unused")
    drafter.spec_num_steps = TREE_DEPTH
    drafter.spec_num_tokens = WINDOW
    drafter.input_buffers = SimpleNamespace(all_extends_mid_chunk=False)
    # Every node a child of the root: any ranking is a valid tree.
    prev = torch.zeros((BS, NODES), dtype=torch.int64, device="cuda")
    prev[:, 0] = -1
    gen = torch.Generator(device="cuda").manual_seed(71)
    scores = torch.rand((BS, NODES), device="cuda", generator=gen)
    scores[:, 0] = 2.0
    drafter._pool = {
        "scores": scores,
        "tokens": torch.arange(BS * NODES, device="cuda").view(BS, NODES),
        "prev": prev,
    }
    drafter._round = {
        "root_tokens": torch.zeros(BS, dtype=torch.int32, device="cuda"),
        "rows": torch.arange(BS, device="cuda"),
        "frontier": torch.zeros(BS, dtype=torch.int64, device="cuda"),
        "live_rows": torch.zeros(1, dtype=torch.int32, device="cuda"),
        "sampled": torch.zeros(1, dtype=torch.bool, device="cuda"),
        "work_deltas": torch.zeros((2, VERIFY_COUNT + 1), device="cuda"),
        "top_scores": torch.zeros((BS, WINDOW), device="cuda"),
        "top_index": torch.zeros((BS, WINDOW), dtype=torch.int64, device="cuda"),
    }
    drafter.runtime_states = SimpleNamespace(
        future_input_map=torch.zeros((BS, WINDOW), dtype=torch.int32, device="cuda"),
        future_parent_map=torch.zeros((BS, WINDOW), dtype=torch.int32, device="cuda"),
    )
    drafter._live_rows = 0
    drafter._live_sampled = False
    drafter._round_sampled = False
    drafter._sampled_q = None
    drafter._round_seq = 0
    drafter._depth_done = [torch.cuda.Event() for _ in range(2)]
    drafter._decide_stream = torch.cuda.Stream()
    drafter._round_stream = torch.cuda.Stream()
    drafter._chain = None
    drafter._target_done = torch.cuda.Event()
    drafter._round_done = torch.cuda.Event()
    drafter._launched_rows = None
    drafter._pending = None
    # The depth's forward and pool update: nothing the stream ordering reads.
    drafter._lane_context = lambda *args: None
    drafter._draft_depth = lambda *args, **kwargs: None
    drafter.attn_backend = SimpleNamespace(
        override_num_extends=lambda n: contextlib.nullcontext()
    )
    return drafter


@pytest.mark.parametrize("mode", [Mode.STATIC, Mode.SERVE])
def test_missing_tree_is_rejected_before_first_forward(mode):
    from eagle_harness.tokenspeed.drafter import HarnessDrafter

    drafter = HarnessDrafter.__new__(HarnessDrafter)
    drafter.config = HarnessConfig(mode=mode, data_path="/unused")
    drafter.tree = None
    drafter.tree_lanes = None
    with pytest.raises(NotImplementedError, match="binds explicit draft ancestry"):
        drafter.run(None, None, None, None)


def test_dynamic_binding_has_no_native_sorter_depth_limit():
    drafter = _drafter()
    drafter.input_buffers.max_bs = BS
    drafter.draft_reads_token_history = False
    drafter.device = torch.device("cuda")
    drafter.draft_model_runner = SimpleNamespace(
        model=SimpleNamespace(
            logits_processor=SimpleNamespace(
                do_argmax=True, config=SimpleNamespace(vocab_size=128)
            )
        )
    )
    drafter._allocate = lambda *args: None
    bound = []
    drafter.attn_backend.bind_tree_draft = bound.append
    spec = SimpleNamespace(config=SimpleNamespace(topk=4, num_steps=12, num_nodes=16))
    drafter.bind_dynamic_tree(spec, _OneDepthPolicy())
    assert drafter.tree_spec is spec
    assert bound == [drafter.tree_lanes]
    assert drafter.tree_lanes.num_slots == 44
    assert drafter.lanes_per_request == 4
    assert not drafter.draft_model_runner.model.logits_processor.do_argmax


def _kernel_drafter(
    cost: BatchCostTable, scale: torch.Tensor, sleep: bool
) -> EagleHarness:
    """A drafter of K=2 trees to depth 6 on the device policy and the real
    pool, ranking and verify tree kernels, its depths captured; its draft
    forward is scripted from its inputs and the round's ``scale``, and with
    ``sleep`` runs long."""
    drafter, k = _drafter(), TREE_K
    del drafter._draft_depth
    drafter.config = HarnessConfig(mode=Mode.SERVE, data_path="/unused")
    drafter.spec_num_steps = TREE_DEPTH
    drafter.device = torch.device("cuda")
    drafter.policy = DevicePolicy(
        policy_costs(
            {bs: replace(cost, batch_size=bs) for bs in (1, 2)},
            k=k,
            depth=TREE_DEPTH,
            verify=WINDOW,
            moe=False,
        ),
        verify_num=WINDOW,
        node_top_k=k,
        tree_depth=TREE_DEPTH,
        compute_intensity="high_intensity",
        is_moe_target=False,
        exec_mode=0,
        benchmark_config="",
        chain_width_table=(),
        device=drafter.device,
    )
    drafter.lanes_per_request = k
    drafter.tree_lanes = TreeDraftInputs(k, TREE_DEPTH, BS, drafter.device)
    drafter.token_to_kv_pool = None
    drafter.attn_backend = SimpleNamespace(
        publish_draft_step_locations=lambda **kw: None,
        override_num_extends=lambda n: contextlib.nullcontext(),
        begin_tree_lanes=lambda rows, live_rows: None,
    )

    def forward(input_ids, positions, captured_hidden_states, spec_step_idx, **kw):
        if sleep:
            torch.cuda._sleep(50_000_000)
        base = (input_ids * 31 + positions * 7 + spec_step_idx) % 997
        steps = torch.arange(k, device="cuda")
        probs = torch.sigmoid(scale * torch.sin(base[:, None] + steps).float())
        return SimpleNamespace(
            draft_topk=(
                probs.sort(dim=1, descending=True).values,
                (base[:, None] * 13 + steps * 101) % 5000,
            ),
            hidden_states=captured_hidden_states * 0.5 + 1,
        )

    drafter.draft_model_runner = SimpleNamespace(
        model=SimpleNamespace(config=SimpleNamespace(hidden_size=HIDDEN)),
        model_config=SimpleNamespace(dtype=torch.float32),
        forward=forward,
    )
    drafter.round_candidates = lambda rows: rows.draft_topk
    drafter._allocate(BS, k, TREE_DEPTH, 1024, 64)
    drafter._depth_graphs, drafter._finalize_graphs = {}, {}
    drafter.capture_round_graphs(BS, lo_bs=0, stream=torch.cuda.Stream(), pool=None)
    return drafter


def _seed_round(drafter: EagleHarness, inputs: dict[str, torch.Tensor]) -> None:
    """What ``draft`` leaves for depth 1 inside the decode graph: the root,
    its children in the pool and depth 1's frontier and lanes."""
    pool, state = drafter._pool, drafter._round
    pool["tokens"][:BS, 0] = inputs["root"]
    drafter_module.eagle_harness_update_node_data(
        pool["scores"][:BS],
        pool["tokens"][:BS],
        pool["work"][:BS],
        pool["sort"][:BS],
        inputs["top_p"],
        inputs["top_tokens"],
    )
    state["root_tokens"][:BS].copy_(inputs["root"])
    state["tokens"][:BS].copy_(inputs["top_tokens"])
    state["scores"][:BS].copy_(inputs["top_p"])
    state["hidden"][:BS].copy_(inputs["hidden"])
    state["frontier"][:BS].copy_(inputs["frontier"])
    state["rows"][:BS].copy_(inputs["rows"])
    drafter._start_lanes(BS, BS)


@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
def test_a_device_chained_round_drafts_the_host_launched_round() -> None:
    """Rounds whose depths and finalize the GPU chains, launched behind a
    decode graph or by ``finish_round``, grow the same pool, publish the same
    go flags and decisions and write the same verify tree as rounds whose
    depths the host launches one by one, also once their graphs were
    released and captured again; behind a decode graph, the forward's stream
    waits for that whole graph and not for the round."""
    target = CostCurve(
        (1.0,) * 5, (1.0, 1.02, 1.05, 1.1, 1.3), (0, 1, 3, 7, 15), (), ()
    )
    draft = CostCurve((1.0, 1.0), (1.0, 1.2), (0, 1), (), ())
    cost = BatchCostTable(
        1, 1000, 1000, 16, 16384, 2, (IntervalCost(target, draft),), (20.0,)
    )
    scale = torch.ones((), device="cuda")
    host, launched, behind = (
        _kernel_drafter(cost, scale, sleep) for sleep in (False, False, True)
    )
    inputs = {
        "root": torch.zeros(BS, dtype=torch.int64, device="cuda"),
        "top_p": torch.zeros((BS, TREE_K), device="cuda"),
        "top_tokens": torch.zeros((BS, TREE_K), dtype=torch.int64, device="cuda"),
        "hidden": torch.zeros((BS, TREE_K, HIDDEN), device="cuda"),
        "frontier": torch.zeros(BS, dtype=torch.int64, device="cuda"),
        "rows": torch.arange(BS, device="cuda"),
    }
    # The decode graph a chained round follows: the draft's seeding, then a
    # slow tail whose result the forward's stream reads.
    tail = torch.zeros(1, dtype=torch.int64, device="cuda")
    decode = torch.cuda.CUDAGraph()
    with torch.cuda.graph(decode):
        _seed_round(behind, inputs)
        torch.cuda._sleep(20_000_000)
        tail.copy_(inputs["frontier"][:1])
    ctx = SimpleNamespace(bs=BS, num_extends=0, global_num_tokens=None, global_bs=None)
    ctx.all_decode_or_idle = True
    depths = set()
    for i in range(12):
        if i == 10:
            for drafter in (host, launched, behind):
                drafter.release_round_graphs()
                drafter.capture_round_graphs(
                    BS, lo_bs=0, stream=torch.cuda.Stream(), pool=None
                )
        gen = torch.Generator().manual_seed(i)
        scale.fill_(0.5 + 0.4 * i)
        inputs["root"].copy_(torch.randint(0, 5000, (BS,), generator=gen))
        inputs["top_p"].copy_(
            torch.rand((BS, TREE_K), generator=gen).sort(dim=1, descending=True).values
        )
        inputs["top_tokens"].copy_(torch.randint(0, 5000, (BS, TREE_K), generator=gen))
        inputs["hidden"].copy_(torch.rand((BS, TREE_K, HIDDEN), generator=gen))
        inputs["frontier"].copy_(torch.tensor([100 + 37 * i, 900 + 11 * i]))
        for drafter, graph_bs in ((host, None), (launched, BS)):
            _seed_round(drafter, inputs)
            drafter.finish_round(ctx, graph_bs)
            drafter.complete_round()
        behind.replay_decode_graph(decode, bs=BS, graph_bs=BS)
        tail_seen = tail.clone()
        seeded = torch.cuda.Event()
        seeded.record()
        behind.finish_round(ctx, BS)
        seeded.synchronize()
        # The forward's stream ran past the decode graph while the round drafts.
        assert not behind._round_stream.query()
        behind.complete_round()
        assert not torch.cuda.current_stream().query()
        torch.cuda.synchronize()
        assert tail_seen.item() == 100 + 37 * i
        depths.add(host.policy.last_depth.item())
        expected = _round_outputs(host)
        for drafter in (launched, behind):
            outputs = _round_outputs(drafter)
            assert outputs.keys() == expected.keys()
            for name, value in expected.items():
                assert torch.equal(outputs[name], value), (i, name)
    # Rounds stopped at several depths, the deepest at the last.
    assert len(depths) > 2 and TREE_DEPTH - 1 in depths


def _round_outputs(drafter: EagleHarness) -> dict[str, torch.Tensor]:
    """What a round leaves: its pool and round buffers, the policy's state,
    slots (go flags included) and their tagged host mirror, and the verify
    tree."""
    policy = drafter.policy
    return {
        **{f"pool.{name}": value for name, value in drafter._pool.items()},
        **{f"round.{name}": value for name, value in drafter._round.items()},
        "state": policy._state,
        "slots": policy._slots,
        "go_host": policy.go_host,
        "tree": drafter.runtime_states.future_input_map,
        "parents": drafter.runtime_states.future_parent_map,
    }


def test_release_round_graphs_drops_what_named_the_probe_pool() -> None:
    """Graphs recorded on the probe pool must not survive the recapture."""
    harness = EagleHarness.__new__(EagleHarness)
    harness._chain = None
    harness._depth_graphs = {(1, 1): object()}
    harness._finalize_graphs = {(1, False): object()}
    harness.release_round_graphs()
    assert harness._depth_graphs == {} and harness._finalize_graphs == {}


if __name__ == "__main__":
    raise SystemExit(pytest.main([__file__, *sys.argv[1:]]))