Skip to content

Official Inference Code of EAGLE-4

10.2 KB
Raw Blame
# Copyright 2026 Eagle AI Corp.
# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available

"""The T=1 sampled tree is wired only on engines whose verify takes ``sampling_step``.

The draft worker is built the way sglang builds it: sglang's own ``EagleDraftWorker.__init__`` and
``init_attention_backends`` run, with only the distributed contexts, the draft model build and the GPU-side
harness objects replaced, so a name the harness reads from the wrong scope fails here instead of in a GPU job.
"""

from __future__ import annotations

import contextlib
from types import SimpleNamespace

import pytest
import torch

pytest.importorskip("sglang.srt.speculative.eagle_worker_v2")

from sglang.srt.runtime_context import get_context  # noqa: E402
from sglang.srt.speculative import eagle_worker_v2  # noqa: E402

from eagle_harness.sglang import worker as worker_mod  # noqa: E402


def _verify_without_hook(batch, *, target_worker, req_to_token_pool, token_to_kv_pool_allocator, plan_stream,
                         plan_stream_ctx, topk, num_draft_tokens, device, metadata_ready_pre_pad, finalize_tree_path,
                         grammar_barrier, accept_step=None):
    return SimpleNamespace(called_with={"accept_step": accept_step})


def _verify_with_hook(batch, *, target_worker, req_to_token_pool, token_to_kv_pool_allocator, plan_stream,
                      plan_stream_ctx, topk, num_draft_tokens, device, metadata_ready_pre_pad, finalize_tree_path,
                      grammar_barrier, accept_step=None, sampling_step=None):
    return SimpleNamespace(called_with={"accept_step": accept_step, "sampling_step": sampling_step})


@pytest.mark.parametrize("engine_verify, seed", [(_verify_without_hook, None), (_verify_with_hook, 721)])
def test_draft_worker_built_like_sglang_seeds_the_sampled_tree_only_with_the_hook(monkeypatch, engine_verify, seed):
    for name in dir(eagle_worker_v2):
        if name.endswith(("_context", "_scope")) and callable(getattr(eagle_worker_v2, name)):
            monkeypatch.setattr(eagle_worker_v2, name, lambda *args, **kwargs: contextlib.nullcontext())
    model = SimpleNamespace(
        config=SimpleNamespace(hidden_size=8),
        model=SimpleNamespace(fc=SimpleNamespace(in_features=16)),
        get_attention_sliding_window_size=lambda: None,
    )
    runner = SimpleNamespace(model=model, dtype=torch.float32, model_config=SimpleNamespace(hf_config=SimpleNamespace()))
    monkeypatch.setattr(
        eagle_worker_v2, "TpModelWorker", lambda **kwargs: SimpleNamespace(model_runner=runner, init_attention_backends=lambda: None)
    )
    monkeypatch.setattr(eagle_worker_v2, "get_plan_stream", lambda device: (None, None))
    monkeypatch.setattr(eagle_worker_v2, "get_pp_stage_load_group", lambda: None)
    built = {}
    monkeypatch.setattr(worker_mod, "run_eagle_verify", engine_verify)
    monkeypatch.setattr(worker_mod, "DraftBackendFactory", lambda *args: SimpleNamespace(create_draft_extend_backend=lambda: None))
    for name in ("DraftTree", "DraftRowAttention", "DraftExtend"):
        monkeypatch.setattr(worker_mod, name, lambda *args, **kwargs: SimpleNamespace())
    steps = object()
    monkeypatch.setattr(worker_mod, "DraftSteps", lambda *args, **kwargs: built.update(kwargs) or steps)
    target = SimpleNamespace(random_seed=721, model_runner=SimpleNamespace(model_config=SimpleNamespace(context_len=64)))
    policy = SimpleNamespace(intensity=None, max_history_verify_num=16, launch_depths=())
    with get_context().override_server_args(
        speculative_eagle_topk=4, speculative_num_steps=6, speculative_num_draft_tokens=16, device="cpu",
        max_running_requests=8,
    ):
        draft = worker_mod.HarnessDraftWorker(None, 0, 0, target, policy=policy)
        draft.init_attention_backends()
    assert built == {"sampled_seed": seed}
    assert draft.steps is steps  # every sglang-built draft worker has its DraftSteps before the first round


@pytest.mark.parametrize("vocab_size", [32, 128256, 248320])
def test_startup_softmax_uses_the_served_vocabulary(monkeypatch, vocab_size):
    """Shape-based backend dispatch must see the same vocabulary during warm-up and serving."""
    worker = SimpleNamespace(
        device="cuda", topk=1,
        draft_runner=SimpleNamespace(model=SimpleNamespace(logits_processor=SimpleNamespace(vocab_size=vocab_size))),
    )
    zeros = torch.zeros
    allocations, calls = [], []

    def allocate(*shape, dtype, device):
        allocations.append((shape, dtype, device))
        return zeros(*shape, dtype=dtype)

    def softmax(logits):
        calls.append((tuple(logits.shape), logits.dtype))
        return logits.softmax(dim=-1)

    monkeypatch.setattr(worker_mod.torch, "zeros", allocate)
    monkeypatch.setattr(worker_mod.spec_utils, "sampling_softmax", softmax)
    worker_mod.HarnessDraftWorker._warm_up_kernels(worker)
    assert allocations == [((1, vocab_size), torch.float32, "cuda")]
    assert calls == [((1, vocab_size), torch.float32)]


def test_tree_warm_up_moves_the_kv_of_the_target_pool(monkeypatch):
    """A tree's verify moves the accepted tokens' KV in the target's pool: its move kernels compile before serving."""
    pool = object()
    moved, accept_tail = [], []
    monkeypatch.setattr(worker_mod, "warm_up_accept_tail", lambda *args: accept_tail.append(args))
    monkeypatch.setattr(worker_mod, "warm_up_kv_moves", lambda kv_cache, device: moved.append((kv_cache, device)))
    monkeypatch.setattr(worker_mod.spec_utils, "sampling_softmax", None)
    allocator = SimpleNamespace(get_kvcache=lambda: pool)
    worker = SimpleNamespace(
        device="cuda", topk=4, extend=SimpleNamespace(hidden_rows=lambda rows: rows),
        target_worker=SimpleNamespace(model_runner=SimpleNamespace(dtype=torch.bfloat16,
                                                                   token_to_kv_pool_allocator=allocator)),
    )
    worker_mod.HarnessDraftWorker._warm_up_kernels(worker)
    assert moved == [(pool, "cuda")] and len(accept_tail) == 1
    worker.topk = 1  # a chain's accepted tokens are already in place: nothing moves
    moved.clear()
    worker_mod.HarnessDraftWorker._warm_up_kernels(worker)
    assert moved == []


def test_kv_move_warm_up_covers_every_program_a_round_can_compile():
    """sglang's tiled KV move compiles one program per next power of two of the moved slots (up to its chunk) and
    per slot count specialization (1, a multiple of 16, any other): the warm-up runs each, on the padding slot."""
    from eagle_harness.sglang.verify_step import KV_MOVE_MAX_SLOTS, warm_up_kv_moves

    moves = []
    warm_up_kv_moves(SimpleNamespace(move_kv_cache=lambda tgt, src: moves.append((tgt.clone(), src.clone()))), "cpu")
    assert all(tgt.dtype == torch.int64 and not tgt.any() and not src.any() for tgt, src in moves)

    def program(count):
        return 1 << (count - 1).bit_length(), "one" if count == 1 else "x16" if count % 16 == 0 else "other"

    warmed = {program(len(tgt)) for tgt, _ in moves}
    assert warmed == {program(count) for count in range(1, KV_MOVE_MAX_SLOTS + 1)}
    assert KV_MOVE_MAX_SLOTS >= 256  # sglang's largest chunk (smaller KV strides)


@pytest.mark.parametrize("engine_verify, active", [(_verify_without_hook, False), (_verify_with_hook, False), (_verify_with_hook, True)])
def test_verify_passes_the_tree_sampler_only_for_an_active_sampled_tree(monkeypatch, engine_verify, active):
    monkeypatch.setattr(worker_mod, "run_eagle_verify", engine_verify)
    monkeypatch.setattr(worker_mod, "fused_accept_supported", lambda *args: False)
    sampled = None if engine_verify is _verify_without_hook else SimpleNamespace(active=active, sample=lambda *args: args)
    worker = SimpleNamespace(
        _draft_worker=SimpleNamespace(steps=SimpleNamespace(sampled=sampled)),
        topk=4,
        token_to_kv_pool_allocator=SimpleNamespace(get_kvcache=lambda: None),
        **dict.fromkeys(("target_worker", "req_to_token_pool", "plan_stream", "plan_stream_ctx", "device")),
    )
    batch = SimpleNamespace(spec_info="verify-input", enable_overlap=False)
    result, staged = worker_mod.EagleHarnessWorker._verify(worker, batch, 8, None)
    assert not staged
    step = result.called_with.get("sampling_step")
    if active:
        assert step("target", "coins", "final") == ("verify-input", "target", "coins", "final")
    else:
        assert step is None


def _sampled_tree(rows=6):
    from eagle_harness.sglang.sampled_tree import SampledTree

    shape = SimpleNamespace(max_depth=3, top_k=2, layer_start=lambda depth: 1 + (depth - 1) * 2)
    tree = SimpleNamespace(shape=shape, tokens=torch.zeros((rows, 9), dtype=torch.int64), max_bs=rows)
    return SampledTree(tree, 8, 721, rows=rows)


def test_prefill_records_each_request_sampling_settings_at_its_slot(monkeypatch):
    """A prefill stages its requests' sampler settings, so their later proposals follow the target's sampler."""
    sampled = _sampled_tree()
    calls = []
    parent = worker_mod.EagleHarnessWorker.__mro__[1]
    monkeypatch.setattr(parent, "forward_batch_generation", lambda self, *args, **kwargs: calls.append(args) or "prefilled")
    worker = worker_mod.EagleHarnessWorker.__new__(worker_mod.EagleHarnessWorker)
    worker._chain_rounds = SimpleNamespace(interrupt=lambda: None)
    worker._drafter = SimpleNamespace(cancel_prelaunch=lambda: None)
    worker._draft_worker = SimpleNamespace(steps=SimpleNamespace(sampled=sampled))
    sampled.root_valid.fill_(True)
    info = SimpleNamespace(temperatures=torch.tensor([[0.5], [1.0]]), top_ks=torch.tensor([20, 1 << 30], dtype=torch.int32),
                           top_ps=torch.tensor([0.95, 1.0]), min_ps=torch.tensor([0.0, 0.1]))
    batch = SimpleNamespace(forward_mode=SimpleNamespace(is_extend=lambda: True), is_extend_in_batch=True,
                            req_pool_indices=torch.tensor([3, 0]), sampling_info=info)
    assert worker.forward_batch_generation(batch) == "prefilled" and calls
    identity = torch.tensor([1.0, float(1 << 30), 1.0, 0.0])
    torch.testing.assert_close(sampled.filters[3], torch.tensor([2.0, 20.0, 0.95, 0.0]))
    torch.testing.assert_close(sampled.filters[0], torch.tensor([1.0, float(1 << 30), 1.0, 0.1]))
    for slot in (1, 2, 4, 5):
        torch.testing.assert_close(sampled.filters[slot], identity)
    assert sampled.root_valid.tolist() == [False, True, True, False, True, True]