Skip to content

Official Inference Code of EAGLE-4

6.8 KB
Raw Blame
# Copyright 2026 Eagle AI Corp.
# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available

"""The Triton tree step, work deltas, ranking and selection against DraftTree's torch reference (skipped
without CUDA).

Run in a GPU job: ``pytest -q tests/test_tree_step_gpu.py``.
"""

import pytest
import torch

from eagle_harness.core.draft_tree import DraftTree, TreeShape

pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA device")

BUFFERS = (
    "scores",
    "tokens",
    "parents",
    "front_scores",
    "front_tokens",
    "front_hidden",
    "ancestors",
    "best",
    "best_prev",
    "layer_top",
)


def _tree(k, depth, bs, h, cap):
    return DraftTree(TreeShape(k, depth, work_cap=cap), bs, h, torch.bfloat16, torch.device("cuda"), max_verify=cap)


@pytest.mark.parametrize(
    "k,depth,bs,h,cap",
    [(4, 12, 1, 5120, 64), (4, 12, 3, 5120, 64), (8, 6, 2, 256, 32), (1, 6, 1, 256, 8)]
    + [(4, 6, bs, 256, 32) for bs in (33, 64, 128, 256, 512)],
)
def test_tree_step_kernel_matches_reference(k, depth, bs, h, cap):
    gen = torch.Generator(device="cuda").manual_seed(k * 1000 + bs)
    trees = [_tree(k, depth, bs, h, cap) for _ in range(2)]
    root_p = torch.rand(bs, k, device="cuda", generator=gen).softmax(-1)
    root_ids = torch.randint(0, 1000, (bs, k), device="cuda", generator=gen)
    root_h = torch.randn(bs, h, device="cuda", generator=gen).to(torch.bfloat16)
    for tree in trees:
        tree.stage_root(bs, root_p, root_ids, root_h)
        tree.begin(bs)
    for step in range(1, depth):
        p = torch.sort(torch.rand(bs * k, k, device="cuda", generator=gen).softmax(-1), dim=-1, descending=True).values
        ids = torch.randint(0, 248320, (bs * k, k), device="cuda", generator=gen)
        hidden = torch.randn(bs * k, h, device="cuda", generator=gen).to(torch.bfloat16)
        slots = torch.randint(0, 10**6, (bs, k), device="cuda", generator=gen)
        for tree in trees:
            tree.set_step_slots(bs, step, slots)
        trees[0].step(bs, step, p, ids, hidden)  # CUDA: the Triton kernels
        trees[1].step_reference(bs, step, p, ids, hidden)
        trees[1]._store_deltas_reference(bs, step)
        for name in BUFFERS:
            got, want = getattr(trees[0], name)[:bs], getattr(trees[1], name)[:bs]
            assert torch.equal(got, want), (step, name)
        # the kernel sums in another order than torch.cumsum: equal to fp32 rounding
        torch.testing.assert_close(trees[0].deltas[step], trees[1].deltas[step], rtol=1e-5, atol=1e-6)


@pytest.mark.parametrize(
    "k,bs,h,cap",
    [(4, 1, 5120, 64), (4, 5, 5120, 64), (8, 3, 256, 32)] + [(4, bs, 256, 32) for bs in (33, 64, 128, 256, 512)],
)
def test_tree_begin_kernel_matches_reference(k, bs, h, cap):
    gen = torch.Generator(device="cuda").manual_seed(k * 100 + bs)
    trees = [_tree(k, 12, bs, h, cap) for _ in range(2)]
    root_p = torch.rand(bs, k, device="cuda", generator=gen).softmax(-1)
    root_p[0, 1] = root_p[0, 2]  # a tie
    root_ids = torch.randint(0, 248320, (bs, k), device="cuda", generator=gen)
    root_h = torch.randn(bs, h, device="cuda", generator=gen).to(torch.bfloat16)
    for tree in trees:
        tree.stage_root(bs, root_p, root_ids, root_h)
    trees[0].begin(bs)  # CUDA: the Triton kernels
    trees[1].begin_reference(bs)
    trees[1]._store_deltas_reference(bs, 0)
    for name in BUFFERS:
        got, want = getattr(trees[0], name)[:bs], getattr(trees[1], name)[:bs]
        assert torch.equal(got, want), name
    torch.testing.assert_close(trees[0].deltas[0], trees[1].deltas[0], rtol=1e-5, atol=1e-6)


@pytest.mark.parametrize(
    "k,depth,bs,count,width",
    [(4, 12, 1, 15, 16), (4, 4, 3, 63, 8), (8, 6, 2, 32, 2), (4, 2, 1, 1, 2)]
    + [(4, 6, bs, 31, 16) for bs in (33, 64, 128, 256, 512)],
)
def test_rank_and_select_kernels_match_reference(k, depth, bs, count, width):
    gen = torch.Generator(device="cuda").manual_seed(k * 10 + depth)
    tree = _tree(k, 12, bs, 64, 64)
    tree.scores.copy_(torch.rand(tree.scores.shape, device="cuda", generator=gen))
    tree.scores[:, 1] = tree.scores[:, 2]  # a tie: the lower node first
    tree.tokens.copy_(torch.randint(0, 248320, tree.tokens.shape, device="cuda", generator=gen))
    count = min(count, tree.shape.num_nodes(depth))
    ranked, cum = tree.rank(bs, depth, count)  # CUDA: the Triton kernel
    want_ranked, want_cum = tree.rank_reference(bs, depth, count)
    assert torch.equal(ranked, want_ranked)
    torch.testing.assert_close(cum, want_cum, rtol=1e-6, atol=1e-6)  # another summation order
    got = tree.select(bs, depth, ranked, width)
    index = torch.sort(want_ranked[:, : width - 1], dim=-1).values
    assert torch.equal(got.top_scores_index, index)
    assert torch.equal(got.draft_tokens, torch.gather(tree.tokens[:bs], 1, index))


_STEP_AT_FULL_BATCH = """
import torch
from eagle_harness.core.draft_tree import DraftTree, TreeShape

k, depth, bs, h = {k}, 4, 32, 256
tree = DraftTree(TreeShape(k, depth, work_cap=64), bs, h, torch.bfloat16, torch.device("cuda"), max_verify=32)
tree.stage_root(bs, torch.rand(bs, k, device="cuda").softmax(-1), torch.zeros(bs, k, dtype=torch.int64,
                device="cuda"), torch.zeros(bs, h, device="cuda", dtype=torch.bfloat16))
tree.begin(bs)
tree.set_step_slots(bs, 1, torch.zeros(bs, k, dtype=torch.int64, device="cuda"))
p = torch.rand(bs * k, k, device="cuda").softmax(-1).sort(-1, descending=True).values
tree.step(bs, 1, p, torch.zeros(bs * k, k, dtype=torch.int64, device="cuda"),
          torch.zeros(bs * k, h, device="cuda", dtype=torch.bfloat16))
torch.cuda.synchronize()
"""


@pytest.mark.parametrize("k", [2, 4, 8])
def test_tree_step_work_deltas_read_only_their_rows_layer_scores(tmp_path, k):
    """The step's work-delta merge loaded next_pow2(K + 1) - 1 layer scores per row, past the row's K: the last row
    of a full batch read past ``layer_top``, an illegal address where that tensor ended its allocation (one GPU,
    batch size 32, --disable-radix-cache, at startup). compute-sanitizer reports any such read."""
    import os
    import shutil
    import subprocess
    import sys

    sanitizer = shutil.which("compute-sanitizer") or os.path.join(
        os.environ.get("CUDA_HOME", "/usr/local/cuda"), "bin", "compute-sanitizer"
    )
    if not os.path.exists(sanitizer):
        pytest.skip("needs compute-sanitizer")
    script = tmp_path / "step.py"
    script.write_text(_STEP_AT_FULL_BATCH.format(k=k))
    run = subprocess.run(
        [sanitizer, "--tool", "memcheck", "--error-exitcode", "3", sys.executable, str(script)],
        capture_output=True,
        text=True,
        timeout=900,
        check=False,
        # One cudaMalloc per tensor, so the sanitizer sees each tensor's own bounds.
        env=dict(os.environ, PYTORCH_NO_CUDA_MEMORY_CACHING="1"),
    )
    assert run.returncode == 0, run.stdout[-3000:] + run.stderr[-2000:]