Skip to content

Official Inference Code of EAGLE-4

9.5 KB
Raw Blame
# Copyright 2026 Eagle AI Corp.
# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available

"""The one-launch recorded-q draw equals the torch reference draw and its Philox stream is TP-rank independent."""

import os

import pytest
import torch

from eagle_harness.core.sampled_tree import draw_children

pytest.importorskip("triton")
from tests.triton_probe import needs_triton  # noqa: E402

INTERPRET = os.environ.get("TRITON_INTERPRET") == "1"
DEV = "cpu" if INTERPRET else "cuda"
pytestmark = needs_triton


def _stores(bs, slots, nc, nodes):
    q = torch.zeros((bs, slots, nc), device=DEV)
    ids = torch.zeros((bs, slots, nc), dtype=torch.int64, device=DEV)
    positions = torch.zeros((bs, nodes), dtype=torch.int64, device=DEV)
    return q, ids, positions


@pytest.mark.parametrize("k, rows_per_req, step", [(4, 1, 0), (4, 4, 2), (2, 2, 1), (1, 1, 0)])
def test_fused_draw_matches_reference_on_shared_uniforms(k, rows_per_req, step):
    from eagle_harness.kernels.sampled_draw import sample_children

    bs, nc = 3, 128
    gen = torch.Generator().manual_seed(11 + k + step)
    rows = bs * rows_per_req
    logits = (torch.randn(rows, nc, generator=gen) * 2).to(DEV)
    token_ids = torch.stack([torch.randperm(5000, generator=gen)[:nc] for _ in range(rows)]).to(DEV)
    uniforms = torch.rand(rows, nc, generator=gen).to(DEV)
    reference = draw_children(logits, token_ids, k, uniforms=uniforms)
    slots, nodes = 1 + 3 * k, 1 + 4 * k * k
    slot = 0 if step == 0 else 1 + (step - 1) * k
    start = 1 + step * k * rows_per_req  # any layout offset: the kernel writes where it is told
    q, ids, positions = _stores(bs, slots, nc, nodes)
    scores, tokens = sample_children(
        logits, token_ids, k, seed=1, launch=step, counter=torch.ones(1, dtype=torch.int32, device=DEV),
        q=q, q_ids=ids, positions=positions, slot=slot, start=start, rows_per_req=rows_per_req, uniforms=uniforms,
    )
    if not INTERPRET:
        torch.cuda.synchronize()
    torch.testing.assert_close(tokens, reference.tokens, rtol=0, atol=0)
    torch.testing.assert_close(positions[:, start : start + rows_per_req * k].reshape(rows, k), reference.positions, rtol=0, atol=0)
    torch.testing.assert_close(scores, reference.scores, rtol=1e-6, atol=1e-7)
    torch.testing.assert_close(q[:, slot : slot + rows_per_req].reshape(rows, nc), reference.probs, rtol=1e-6, atol=1e-7)
    torch.testing.assert_close(ids[:, slot : slot + rows_per_req].reshape(rows, nc), reference.token_ids, rtol=0, atol=0)
    # nothing written outside the addressed slots / children
    untouched = torch.ones(slots, dtype=torch.bool)
    untouched[slot : slot + rows_per_req] = False
    assert q[:, untouched].abs().sum() == 0 and ids[:, untouched].abs().sum() == 0
    mask = torch.ones(nodes, dtype=torch.bool)
    mask[start : start + rows_per_req * k] = False
    assert positions[:, mask].abs().sum() == 0


def test_philox_draws_are_deterministic_per_round_and_launch_and_fresh_across_rounds():
    from eagle_harness.kernels.sampled_draw import sample_children

    nc, k = 64, 4
    gen = torch.Generator().manual_seed(5)
    logits = torch.randn(2, nc, generator=gen).to(DEV)
    token_ids = torch.arange(nc).expand(2, -1).contiguous().to(DEV)

    def draw(seed, rnd, launch):
        q, ids, positions = _stores(2, 1, nc, k)
        counter = torch.full((1,), rnd, dtype=torch.int32, device=DEV)
        _, tokens = sample_children(logits, token_ids, k, seed=seed, launch=launch, counter=counter, q=q, q_ids=ids,
                                    positions=positions, slot=0, start=0, rows_per_req=1)
        if not INTERPRET:
            torch.cuda.synchronize()
        return tokens.cpu()

    a, b = draw(721, 3, 1), draw(721, 3, 1)  # two ranks: same seed, same round, same launch -> same tree
    torch.testing.assert_close(a, b, rtol=0, atol=0)
    others = [draw(721, 4, 1), draw(721, 3, 2), draw(722, 3, 1)]
    assert any(not torch.equal(a, o) for o in others)
    assert sum(not torch.equal(a, o) for o in others) >= 2


def test_philox_first_child_follows_q():
    """The first Gumbel-top-K draw is a sample from q: chi-square over many rows of one distribution."""
    from eagle_harness.kernels.sampled_draw import sample_children

    nc, k, rows = 8, 2, (2048 if INTERPRET else 65536)
    q_true = torch.tensor([0.4, 0.25, 0.15, 0.1, 0.05, 0.03, 0.015, 0.005])
    logits = q_true.log().expand(rows, -1).contiguous().to(DEV)
    token_ids = torch.arange(nc).expand(rows, -1).contiguous().to(DEV)
    q, ids, positions = _stores(rows, 1, nc, k)
    _, tokens = sample_children(logits, token_ids, k, seed=9, launch=0, counter=torch.ones(1, dtype=torch.int32, device=DEV),
                                q=q, q_ids=ids, positions=positions, slot=0, start=0, rows_per_req=1)
    if not INTERPRET:
        torch.cuda.synchronize()
    counts = torch.bincount(tokens[:, 0].cpu(), minlength=nc).double()
    expected = q_true.double() * rows
    chi2 = ((counts - expected) ** 2 / expected).sum().item()
    assert chi2 < 30.0, chi2  # 7 dof: 30 is far beyond the 99.99% point (24.3)
    assert (tokens[:, 0] != tokens[:, 1]).all()  # without replacement


def test_stores_follow_the_request_slot_map():
    from eagle_harness.kernels.sampled_draw import sample_children

    nc, k = 32, 2
    gen = torch.Generator().manual_seed(3)
    logits = torch.randn(2, nc, generator=gen).to(DEV)
    token_ids = torch.arange(nc).expand(2, -1).contiguous().to(DEV)
    uniforms = torch.rand(2, nc, generator=gen).to(DEV)
    reference = draw_children(logits, token_ids, k, uniforms=uniforms)
    q, ids, positions = _stores(8, 3, nc, 5)
    req = torch.tensor([5, 2], device=DEV)
    sample_children(logits, token_ids, k, seed=1, launch=1, counter=torch.ones(1, dtype=torch.int32, device=DEV),
                    q=q, q_ids=ids, positions=positions, slot=1, start=3, rows_per_req=1, uniforms=uniforms, req_index=req)
    if not INTERPRET:
        torch.cuda.synchronize()
    torch.testing.assert_close(q[req, 1], reference.probs, rtol=1e-6, atol=1e-7)
    torch.testing.assert_close(positions[req, 3:5], reference.positions, rtol=0, atol=0)
    assert q[[0, 1, 3, 4, 6, 7]].abs().sum() == 0 and q[:, [0, 2]].abs().sum() == 0


# (1/temperature, top-k, top-p, min-p): coding, narrow top-k, min-p, top-1, and narrow top-p.
FILTER_CASES = [(1.0, 20, 0.95, 0.0), (2.0, 3, 1.0, 0.0), (1.0, 1 << 30, 1.0, 0.1),
                (1.0, 1, 1.0, 0.0), (0.5, 1 << 30, 0.3, 0.0)]


@pytest.mark.parametrize("inv_t, top_k, top_p, min_p", FILTER_CASES)
def test_filtered_draw_matches_reference_on_filtered_logits(inv_t, top_k, top_p, min_p):
    from eagle_harness.core.sampled_tree import filter_proposal_logits
    from eagle_harness.kernels.sampled_draw import sample_children

    bs, nc, k = 4, 128, 4
    gen = torch.Generator().manual_seed(17 + top_k)
    logits = torch.randn(bs, nc, generator=gen) * 3
    token_ids = torch.stack([torch.randperm(5000, generator=gen)[:nc] for _ in range(bs)])
    uniforms = torch.rand(bs, nc, generator=gen)
    req = torch.tensor([6, 1, 3, 0])
    filters = torch.tensor([1.0, float(1 << 30), 1.0, 0.0]).repeat(8, 1)
    filters[req] = torch.tensor([inv_t, float(top_k), top_p, min_p])
    rows = torch.ones(bs)
    filtered = filter_proposal_logits(logits, rows / inv_t, rows * top_k, rows * top_p, rows * min_p)
    kept = torch.isfinite(filtered).sum(dim=1)
    reference = draw_children(filtered, token_ids, k, uniforms=uniforms)
    q, ids, positions = _stores(8, 1, nc, k)
    scores, tokens = sample_children(
        logits.to(DEV), token_ids.to(DEV), k, seed=1, launch=0, counter=torch.ones(1, dtype=torch.int32, device=DEV),
        q=q, q_ids=ids, positions=positions, slot=0, start=0, rows_per_req=1, uniforms=uniforms.to(DEV),
        req_index=req.to(DEV), filters=filters.to(DEV),
    )
    if not INTERPRET:
        torch.cuda.synchronize()
    q, positions, scores, tokens = q.cpu(), positions.cpu(), scores.cpu(), tokens.cpu()
    torch.testing.assert_close(q[req, 0], reference.probs, rtol=1e-5, atol=1e-7)
    assert ((q[req, 0] > 0) == torch.isfinite(filtered)).all()
    torch.testing.assert_close(scores, reference.scores, rtol=1e-5, atol=1e-7)
    for b in range(bs):
        live = min(k, int(kept[b]))  # draws past the kept support are inert (q = 0) and need not match
        torch.testing.assert_close(positions[req[b], :live], reference.positions[b, :live], rtol=0, atol=0)
        torch.testing.assert_close(tokens[b, :live], reference.tokens[b, :live], rtol=0, atol=0)
        assert positions[req[b]].unique().numel() == k  # inert draws stay distinct candidates
        assert (q[req[b], 0].gather(0, positions[req[b], live:]) == 0).all()


def test_identity_filters_draw_bitwise_like_no_filters():
    from eagle_harness.kernels.sampled_draw import sample_children

    nc, k = 128, 4
    gen = torch.Generator().manual_seed(5)
    logits = (torch.randn(3, nc, generator=gen) * 2).to(DEV)
    token_ids = torch.arange(nc).expand(3, -1).contiguous().to(DEV)
    identity = torch.tensor([1.0, float(1 << 30), 1.0, 0.0]).repeat(3, 1).to(DEV)
    out = []
    for filters in (None, identity):
        q, ids, positions = _stores(3, 1, nc, k)
        scores, tokens = sample_children(logits, token_ids, k, seed=4, launch=2,
                                         counter=torch.full((1,), 7, dtype=torch.int32, device=DEV), q=q, q_ids=ids,
                                         positions=positions, slot=0, start=0, rows_per_req=1, filters=filters)
        out.append((q.cpu(), positions.cpu(), scores.cpu(), tokens.cpu()))
    for a, b in zip(*out):
        torch.testing.assert_close(a, b, rtol=0, atol=0)