Skip to content

Official Inference Code of EAGLE-4

3.8 KB
Raw Blame
# Copyright 2026 Eagle AI Corp.
# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available

"""The draft steps' layer input (:meth:`LlamaForCausalLMEagle4._layer_input`, :func:`stack_layer_input`): the two-launch
path gives the same bits as the embedding + two norms + cat it replaces."""

from types import SimpleNamespace

import pytest
import torch

from tests.triton_probe import needs_triton

DEV = "cuda" if torch.cuda.is_available() else "cpu"


@needs_triton
@pytest.mark.parametrize("rows, hidden", [(4, 5120), (3, 1000)])
def test_stack_layer_input_puts_the_token_row_beside_the_hidden_row(rows, hidden):
    from eagle_harness.vllm.draft_rows import stack_layer_input

    gen = torch.Generator().manual_seed(rows)
    table = torch.randn(50, hidden, generator=gen).to(torch.bfloat16).to(DEV)
    ids = torch.randint(0, 50, (rows,), generator=gen).to(DEV)
    states = torch.randn(rows, 2 * hidden, generator=gen).to(torch.bfloat16).to(DEV)[:, :hidden]  # a strided view
    got = stack_layer_input(table, ids, states)
    assert got.shape == (rows, 2, hidden) and got.is_contiguous()
    assert torch.equal(got, torch.stack([table[ids], states], dim=1))


def _draft(hidden: int, vocab: int, *, rank_start: int = 0):
    """A ``LlamaForCausalLMEagle4`` shell with the parts ``_layer_input`` touches (TP 1)."""
    pytest.importorskip("vllm")
    from vllm.config import VllmConfig, set_current_vllm_config
    from vllm.model_executor.layers.layernorm import RMSNorm

    from eagle_harness.vllm import models

    torch.manual_seed(0)
    with set_current_vllm_config(VllmConfig()):
        norms = [RMSNorm(hidden, eps=1e-6).to(DEV, torch.bfloat16) for _ in range(2)]
    for norm in norms:
        norm.weight.data.normal_(1.0, 0.1)
    layer = SimpleNamespace(input_layernorm=norms[0], hidden_norm=norms[1])
    weight = torch.randn(vocab, hidden, device=DEV, dtype=torch.bfloat16)
    shard = SimpleNamespace(num_added_elements=0, num_org_vocab_padding=0, org_vocab_start_index=rank_start)
    embed = SimpleNamespace(weight=weight, shard_indices=shard)
    model = SimpleNamespace(layers=[layer], embed_tokens=embed, embed_input_ids=lambda ids: weight[ids])
    draft = object.__new__(models.LlamaForCausalLMEagle4)
    draft.__dict__.update(
        model=model,
        tp_size=1,
        _max_rows=8,
        _full_embed=None,
        _input_norm_weights=None,
        _norm_before_residual=False,
        _modules={},
        _parameters={},
    )
    from eagle_harness.vllm import model_rows

    return model_rows, draft


def test_layer_input_falls_back_off_cuda_or_with_a_padded_vocab(monkeypatch):
    models, draft = _draft(64, 32, rank_start=7)
    monkeypatch.setattr(models, "get_tensor_model_parallel_rank", lambda: 0)
    states = torch.randn(4, 64, device=DEV, dtype=torch.bfloat16)
    assert not draft._fused_layer_input(draft.model.layers[0], states)
    assert draft._full_embed is None  # nothing was built
    out, residual = draft._layer_input(torch.tensor([1, 2, 3, 4], device=DEV), states)
    assert out.shape == (4, 128) and residual is states


@pytest.mark.skipif(not torch.cuda.is_available(), reason="vLLM's RMSNorm kernel needs a GPU")
@pytest.mark.parametrize("rows", [1, 4, 8])
def test_two_launch_layer_input_is_bit_identical(rows, monkeypatch):
    models, draft = _draft(5120, 1000)
    monkeypatch.setattr(models, "get_tensor_model_parallel_rank", lambda: 0)
    ids = torch.randint(0, 1000, (rows,), device=DEV)
    states = torch.randn(rows, 5120, device=DEV, dtype=torch.bfloat16)
    with models._kernel_ops():
        layer = draft.model.layers[0]
        want = torch.cat([layer.input_layernorm(draft.model.embed_input_ids(ids)), layer.hidden_norm(states)], dim=-1)
        got, residual = draft._layer_input(ids, states)
    assert draft._full_embed is not None  # the two-launch path ran
    assert residual is states and torch.equal(got, want)