Official Inference Code of EAGLE-4
# Copyright 2026 Eagle AI Corp.
# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available
"""The served-round chain policy of a serve-mode chain without data_path (:class:`ServedChainPolicy`), on CPU."""
from __future__ import annotations
import json
from types import SimpleNamespace
import pytest
import torch
from eagle_harness.core.config import HarnessConfig, Mode
from eagle_harness.core.limits import ComputeIntensity
from eagle_harness.sglang import chain_rounds
from eagle_harness.sglang.chain_rounds import ChainRounds, ServedChainPolicy, ServedTreePolicy, served_chain, served_tree
class Clock:
"""``time.perf_counter_ns`` under test control."""
def __init__(self):
self.ns = 0
def __call__(self):
return self.ns
def advance(self, ms: float) -> None:
self.ns += int(ms * 1e6)
@pytest.fixture
def clock(monkeypatch):
clock = Clock()
monkeypatch.setattr(chain_rounds.time, "perf_counter_ns", clock)
return clock
def _policy(
tmp_path=None, *, widths=(1, 2, 4, 8), native=False, writer=True, sync=lambda step: step, key=None, packaged=None
):
return ServedChainPolicy(
widths,
torch.device("cpu"),
max_batch_size=512,
native_decode=native,
writer=writer,
sync=sync,
cache_path=None if tmp_path is None else str(tmp_path / "chain.json"),
key=key,
packaged=packaged,
)
def _reqs(*ids):
return [SimpleNamespace(rid=i) for i in ids]
def _serve(policy, clock, bs, period_ms, accept, rounds=10_000):
"""Serve rounds of ``bs`` requests until their batch key is decided; ``period_ms[w]`` is the whole period of a
width-``w`` round (0: native decode) and ``accept[w]`` its accept length. Returns the widths served."""
requests = _reqs(*range(bs))
served = []
for _ in range(rounds):
width = policy.width(bs)
policy.round_start(bs, width, requests)
if not policy.exploring(bs):
return served
served.append(width)
if width:
policy.accepted(width, torch.full((bs,), accept[width]))
clock.advance(period_ms[width])
raise AssertionError("no decision")
def test_a_prefill_between_two_rounds_is_part_of_the_first_rounds_period(clock):
policy = _policy()
width = policy.width(64)
policy.round_start(64, width, _reqs(1, 2))
clock.advance(10)
# A prefill admits request 3 (no decode round): its time stays in the previous round's period.
clock.advance(25)
policy.round_start(64, policy.width(64), _reqs(1, 2, 3))
assert policy._cells[(64, width)] == [35_000.0, 64, 1] # us, requests (the batch size), periods
def test_a_complete_turnover_does_not_count_as_a_served_interval(clock):
policy = _policy()
width = policy.width(8)
policy.round_start(8, width, _reqs(1, 2))
clock.advance(5_000) # the batch drained: the server may have idled before the next requests came
policy.round_start(8, policy.width(8), _reqs(3, 4))
assert policy._cells.get((8, width), [0.0, 0, 0])[2] == 0
def test_a_burst_of_arrivals_counts_toward_the_comparison_without_being_priced(clock):
policy = _policy()
width = policy.width(64)
policy.round_start(64, width, _reqs(*range(64)))
clock.advance(200) # a large prefill admits 9 requests (more than 1/8 of the batch)
policy.round_start(64, policy.width(64), _reqs(*range(9, 73)))
assert policy._cells[(64, width)] == [0.0, 0, 1]
clock.advance(10) # one refill: priced
policy.round_start(64, policy.width(64), _reqs(*range(10, 74)))
assert policy._cells[(64, width)] == [10_000.0, 64, 2]
def test_a_turnover_measures_native_decode_and_its_best_width_again_together(clock):
policy = _policy(native=True)
policy._pair[64] = (4, 0)
requests = _reqs(*range(64))
while policy.width(64) == 4:
policy.round_start(64, 4, requests)
clock.advance(10)
policy.round_start(64, 0, requests) # native decode's first round: the best width has its block
clock.advance(5_000)
policy.round_start(64, policy.width(64), _reqs(*range(100, 164))) # a new load after the batch drained
assert policy._cells[(64, 4)] == [0.0, 0, 0] and policy._cells[(64, 0)] == [0.0, 0, 0]
assert policy.width(64) == 4
def test_the_search_widens_while_the_wider_chain_wins(clock):
policy = _policy()
# Accept lengths 1, 2, 3, 3 at widths 1, 2, 4, 8: tokens per ms 0.1, 0.167, 0.2, 0.1 per request.
served = _serve(policy, clock, 64, {1: 10, 2: 12, 4: 15, 8: 30}, {1: 1, 2: 2, 4: 3, 8: 3})
assert policy.decided == {64: 4} and set(served) == {1, 2, 4, 8}
# Another batch key searches on its own: compute-bound rounds keep the root-only verify.
_serve(policy, clock, 256, {1: 20, 2: 45, 4: 80, 8: 160}, {1: 1, 2: 2, 4: 3, 8: 3})
assert policy.decided == {64: 4, 256: 1}
def test_a_larger_batch_never_searches_wider_than_a_smaller_one_chose(clock):
policy = _policy()
_serve(policy, clock, 16, {1: 10, 2: 12, 4: 15, 8: 30}, {1: 1, 2: 2, 4: 3, 8: 3})
# At batch 64 width 8 would win, but width 4 won at batch 16: the search stops there.
served = _serve(policy, clock, 64, {1: 10, 2: 12, 4: 15, 8: 16}, {1: 1, 2: 2, 4: 3, 8: 5})
assert 8 not in served and policy.decided == {16: 4, 64: 4}
def test_native_decode_is_measured_last_and_spreads_to_larger_batches(clock):
policy = _policy(native=True)
# Width 2 beats width 1 (2/15 > 1/10) and width 4 (2/18); native decode (1/7) beats width 2.
served = _serve(policy, clock, 64, {0: 7, 1: 10, 2: 15, 4: 18, 8: 25}, {1: 1, 2: 2, 4: 2, 8: 2})
first_native = served.index(0)
assert 0 not in served[:first_native] and set(served[first_native:]) == {0} # once native, the batch stays
assert policy.decided[64] == 0
assert all(policy.decided[bs] == 0 for bs in (96, 128, 192, 256, 384, 512)) # larger batch keys: unmeasured
assert policy.exploring(48)
def test_a_losing_native_decode_leaves_the_best_width(clock):
policy = _policy(native=True)
_serve(policy, clock, 16, {0: 9, 1: 10, 2: 12, 4: 15, 8: 30}, {1: 1, 2: 2, 4: 3, 8: 3})
assert policy.decided == {16: 4}
def test_no_smaller_batch_measures_native_decode_once_speculation_won(clock):
policy = _policy(native=True)
periods, accept = {0: 9, 1: 10, 2: 12, 4: 15, 8: 30}, {1: 1, 2: 2, 4: 3, 8: 3}
assert 0 in _serve(policy, clock, 64, periods, accept) and policy.decided == {64: 4}
assert 0 not in _serve(policy, clock, 16, periods, accept) and policy.decided[16] == 4
def test_native_decode_is_not_chosen_without_the_engine_route(clock):
policy = _policy(native=False)
served = _serve(policy, clock, 64, {0: 1, 1: 10, 2: 15, 4: 18, 8: 25}, {1: 1, 2: 2, 4: 2, 8: 2})
assert 0 not in served and policy.decided == {64: 2}
def test_comparisons_interleave_the_two_widths_in_blocks(clock):
policy = _policy()
plan = []
for _ in range(4 * chain_rounds.BLOCK_ROUNDS):
width = policy.width(64)
plan.append(width)
policy.round_start(64, width, _reqs(1))
block = chain_rounds.BLOCK_ROUNDS
assert plan == ([1] * block + [2] * block) * 2
def test_decisions_survive_a_restart_on_the_same_setup_only(tmp_path, clock):
key = {"gpu": "NVIDIA B300", "gpu_count": 4}
policy = _policy(tmp_path, key=key)
_serve(policy, clock, 64, {1: 10, 2: 12, 4: 15, 8: 30}, {1: 1, 2: 2, 4: 3, 8: 3})
restarted = _policy(tmp_path, key=key, writer=False)
assert restarted.decided == {64: 4} and restarted.width(60) == 4 and not restarted.exploring(64)
assert _policy(tmp_path, key={**key, "gpu_count": 8}).decided == {} # another setup explores again
assert _policy(tmp_path, key=key, native=True).decided == {} # another action set too
(tmp_path / "chain.json").write_text(
json.dumps({**json.loads((tmp_path / "chain.json").read_text()), "chain": {"64": 3}})
)
assert _policy(tmp_path, key=key).decided == {} # a width without a graph is refused
def test_a_first_start_serves_the_packaged_choices_of_its_setup(clock):
policy = _policy(native=True, packaged={16: 4, 64: 4, 256: 0})
assert {k: w for k, w in policy.decided.items() if w} == {16: 4, 64: 4}
assert all(policy.decided[k] == 0 for k in (256, 384, 512)) # native decode serves every larger batch key
assert policy.width(60) == 4 and policy.width(300) == 0 and not policy.exploring(64)
assert policy.exploring(128) and policy.exploring(8) # unmeasured keys are searched on the served rounds
# A search between packaged keys stays within the width a smaller key chose.
served = _serve(policy, clock, 128, {0: 9, 1: 10, 2: 12, 4: 15, 8: 30}, {1: 1, 2: 2, 4: 3, 8: 3})
assert 8 not in served and policy.decided[128] == 4
def test_the_servers_own_choices_take_precedence_over_the_packaged_ones(tmp_path, clock):
key = {"gpu": "NVIDIA B300 SXM6 AC", "gpu_count": 4}
periods, accept = {0: 9, 1: 10, 2: 12, 4: 30, 8: 60}, {1: 1, 2: 2, 4: 3, 8: 3}
_serve(_policy(tmp_path, key=key, native=True), clock, 64, periods, accept)
assert _policy(tmp_path, key=key, native=True, packaged={16: 4, 64: 4, 256: 0}).decided == {64: 2}
def test_packaged_choices_that_are_not_actions_of_the_server_are_not_served():
assert _policy(native=False, packaged={16: 4, 64: 4, 256: 0}).decided == {} # no native decode route
assert _policy(native=True, widths=(1, 2, 8), packaged={16: 4}).decided == {} # no graph for width 4
def test_other_ranks_apply_rank_zeros_step_without_reading_the_gpu(clock):
class NoDeviceCounts:
def __getitem__(self, width):
raise AssertionError("a non-writer rank counted accept lengths")
def tolist(self):
raise AssertionError("a non-writer rank read the accept counts")
received = []
policy = _policy(writer=False, sync=lambda step: received.append(step) or ("decide", 2, None))
policy._accept = NoDeviceCounts()
rounds = ChainRounds(None, None, lambda batch: 0, policy)
requests = _reqs(1)
while policy.exploring(8):
width = policy.width(8)
policy.round_start(8, width, requests)
rounds.accepted(width, torch.ones(8, dtype=torch.int64))
clock.advance(1)
assert received == [None] and policy.decided == {8: 2}
def test_batch_sizes_share_the_nearest_calibration_batch_size():
policy = _policy()
policy.decided = {64: 4, 256: 0}
rounds = ChainRounds(None, None, lambda batch: 0, policy)
assert rounds.draft_width(57) == rounds.draft_width(70) == 4
assert rounds.plain_width(256) == 0 and rounds.draft_width(256) == 1 # a draft never asks for width 0
assert policy.exploring(8) and not policy.exploring(250)
def test_only_a_serve_mode_chain_without_data_path_is_served_by_the_policy():
chain = HarnessConfig(intensity=ComputeIntensity.CHAIN)
assert served_chain(chain, 1)
assert not served_chain(chain, 4)
assert not served_chain(HarnessConfig(intensity=ComputeIntensity.CHAIN, data_path="/cost"), 1)
assert not served_chain(HarnessConfig(intensity=ComputeIntensity.CHAIN, mode=Mode.STATIC), 1)
assert not served_chain(HarnessConfig(), 1)
def _decide_width_4_or_native(clock, period_ms):
"""Width 4 (1.05 tokens per request and round) against native decode at batch 64; ``period_ms(i)`` is the i-th
period of either."""
policy = _policy(native=True)
policy._pair[64] = (4, 0)
policy.profile = lambda: chain_rounds.AcceptProfile((0.05, 0.0, 0.0))
requests = _reqs(*range(64))
i = 0
while policy.exploring(64):
policy.round_start(64, policy.width(64), requests)
clock.advance(period_ms(i))
i += 1
return policy.decided[64]
def test_speculation_beats_native_decode_only_beyond_the_noise_of_the_two_rates(clock):
# Width 4 commits 5% more per period, but every other period of both takes twice as long: 32 periods each cannot
# tell them apart, and native decode serves.
assert _decide_width_4_or_native(clock, lambda i: 20 if i % 2 else 10) == 0
# Without the noise the same 5% decides for speculation.
assert _decide_width_4_or_native(clock, lambda i: 10) == 4
def test_a_tied_comparison_with_native_decode_is_measured_again(clock):
"""Width 4 commits 1.5 tokens per request and round. Its first 32 periods are slow and uneven (4 and 24 ms, as on a
cold start: no single period is a stall), which ties it with native decode's 10 ms. The comparison runs again,
and the clean periods that follow (10 ms) decide for speculation; a tie that persists still falls to native decode
(test above). Before, the first tie decided native decode for good."""
policy = _policy(native=True)
policy._pair[64] = (4, 0)
policy.profile = lambda: chain_rounds.AcceptProfile((0.5, 0.0, 0.0))
requests = _reqs(*range(64))
i = 0
while policy.exploring(64):
policy.round_start(64, policy.width(64), requests)
clock.advance((4 if i % 2 else 24) if i < 33 else 10)
i += 1
assert policy.decided[64] == 4 and policy._remeasured[64] == 1
def _tree(tmp_path=None, key=None, writer=True, sync=lambda step: step, native_us=None):
return ServedTreePolicy(
16,
torch.device("cpu"),
max_batch_size=512,
writer=writer,
sync=sync,
cache_path=None if tmp_path is None else str(tmp_path / "tree.json"),
key=key,
native_us=native_us,
)
def _serve_tree(policy, clock, bs, spec_ms, native_ms, accept, rounds=10_000, seed_rounds=0, widths=(16, 7, 3)):
"""Serve ``bs`` requests until their batch key is decided: speculative rounds (the cost model's widths, cycling
through ``widths``) commit ``accept`` tokens per request in ``spec_ms``, native decode one in ``native_ms``. The
first ``seed_rounds`` speculative rounds seed the live costs. Returns the actions served (16: speculation)."""
requests = _reqs(*range(bs))
served = []
for i in range(rounds):
action = policy.width(bs)
width = widths[i % len(widths)] if action else 0
settled = not (action and i < seed_rounds)
policy.round_start(bs, width, requests, settled=settled)
if not policy.exploring(bs):
return served
served.append(action)
if width:
policy.accepted(width, torch.full((bs,), accept, dtype=torch.int32))
clock.advance(spec_ms if action and settled else (100 * spec_ms if action else native_ms))
raise AssertionError("no decision")
def test_a_tree_decodes_natively_where_its_rounds_commit_fewer_tokens_per_time(clock):
policy = _tree()
# 2.5 tokens per request in 30 ms (0.083 per ms) against native decode's 1 in 10 ms (0.1 per ms).
served = _serve_tree(policy, clock, 256, spec_ms=30, native_ms=10, accept=2.5)
first_native = served.index(0)
assert set(served[:first_native]) == {16} and set(served[first_native:]) == {0} # native decode last, sticky
assert all(policy.decided[k] == 0 for k in (256, 384, 512)) and policy.exploring(192)
@pytest.mark.parametrize("stalled, spec_ms, winner", [(0, 30, 0), (16, 15, 16)])
def test_a_one_time_stall_does_not_decide_the_comparison(clock, stalled, spec_ms, winner):
"""A kernel compile or graph capture in one period (3 s against 10-30 ms rounds) is counted, not priced: the
action that commits more tokens per time (2.5 in ``spec_ms`` against 1 in 10 ms) wins although it stalled."""
policy, requests, rounds = _tree(), _reqs(*range(256)), {0: 0, 16: 0}
while policy.exploring(256):
action = policy.width(256)
policy.round_start(256, action, requests)
if action:
policy.accepted(action, torch.full((256,), 2.5, dtype=torch.int32))
rounds[action] += 1
clock.advance(3000 if action == stalled and rounds[action] == 2 else spec_ms if action else 10)
assert rounds[stalled] > 2 and policy.decided[256] == winner
assert chain_rounds.STALL_FACTOR * 30 < 3000
def test_the_first_round_after_a_switch_of_action_is_counted_not_priced(clock):
"""Native decode's first round after speculation also finishes speculation's work (a prelaunched draft): its
period (95 ms here, under the stall factor) is counted toward the comparison but not priced."""
policy, requests, compared = _tree(), _reqs(*range(256)), []
policy._compare = lambda key, pair: compared.append(policy._rates(key, pair)) or policy._decide(key, 0)
native = 0
while policy.exploring(256):
action = policy.width(256)
policy.round_start(256, action, requests)
if action:
policy.accepted(action, torch.full((256,), 2.5, dtype=torch.int32))
native += not action
clock.advance(30 if action else 95 if native == 1 else 10)
(rates,) = compared
assert rates[0][0] == pytest.approx(256 / 10_000) and native == chain_rounds.EXPLORE_ROUNDS + 1
def test_a_tree_keeps_speculating_where_it_wins_and_so_does_every_smaller_batch(clock):
policy = _tree()
_serve_tree(policy, clock, 64, spec_ms=15, native_ms=10, accept=2.5)
assert policy.decided[64] == 16
assert all(policy.decided[k] == 16 for k in (1, 2, 8, 16, 48)) and policy.exploring(96)
rounds = ChainRounds(None, None, lambda batch: 0, policy)
assert rounds.plain_width(1) == 16 and rounds.plain_width(64) == 16 # never native below a speculating key
def test_rounds_that_seed_the_live_costs_are_not_counted(clock):
policy = _tree()
# The seed rounds force widths the cost model would not choose (100x the settled period here): were they counted,
# native decode would win.
served = _serve_tree(policy, clock, 64, spec_ms=15, native_ms=10, accept=2.5, seed_rounds=40)
assert policy.decided[64] == 16
assert served[:40] == [16] * 40 and served.count(0) == chain_rounds.EXPLORE_ROUNDS
def test_speculation_far_ahead_of_native_decodes_estimate_never_decodes_natively(clock):
# 4 tokens per request in 10 ms (0.4 per ms) against an estimated 1 in 10 ms: native decode would hold the batch.
priced = []
policy = _tree(native_us=lambda bs: priced.append(bs) or 10_000.0)
served = _serve_tree(policy, clock, 8, spec_ms=10, native_ms=10, accept=4)
assert 0 not in served and len(served) == chain_rounds.EXPLORE_ROUNDS
assert set(priced) == {8} and all(policy.decided[k] == 16 for k in (1, 2, 8))
@pytest.mark.parametrize("native_us", [lambda bs: 10_000.0, lambda bs: None])
def test_speculation_near_native_decodes_estimate_or_without_one_measures_native_decode(clock, native_us):
# 2 tokens per request in 15 ms against an estimated 1 in 10 ms (1.33x, inside the skip ratio), or no estimate.
policy = _tree(native_us=native_us)
served = _serve_tree(policy, clock, 8, spec_ms=15, native_ms=10, accept=2)
assert served.count(0) == chain_rounds.EXPLORE_ROUNDS and policy.decided[8] == 16
def test_noisy_speculation_measures_native_decode_even_ahead_of_its_estimate(clock):
# Periods of 1 and 49 ms: 1.6x the estimated 0.8 tokens per ms on average, but not beyond the noise.
policy = _tree(native_us=lambda bs: 10_000.0)
requests, served = _reqs(*range(8)), []
while policy.exploring(8):
action = policy.width(8)
policy.round_start(8, action, requests)
served.append(action)
if action:
policy.accepted(action, torch.full((8,), 4, dtype=torch.int32))
clock.advance((1, 49)[len(served) % 2] if action else 10)
assert 0 in served
def test_a_chain_without_a_native_estimate_never_checks_one(clock):
policy = _policy(native=True)
policy._check_native = lambda key, best: pytest.fail("checked native decode without an estimate")
_serve(policy, clock, 64, {0: 9, 1: 10, 2: 12, 4: 15, 8: 30}, {1: 1, 2: 2, 4: 3, 8: 3})
assert 64 in policy.decided
def test_other_ranks_take_rank_zeros_native_check_without_pricing(clock):
received = []
def sync(step):
received.append(step)
return ("decide", 16, None)
policy = _tree(writer=False, sync=sync, native_us=lambda bs: pytest.fail("a non-writer rank priced native decode"))
served = _serve_tree(policy, clock, 8, spec_ms=10, native_ms=10, accept=4)
assert received == [None] and 0 not in served and policy.decided[8] == 16
def test_a_first_tree_start_serves_the_packaged_choices_of_its_setup():
policy = ServedTreePolicy(
16, torch.device("cpu"), max_batch_size=512, writer=True, sync=lambda step: step, packaged={8: 1, 64: 0}
)
assert all(policy.decided[k] == 16 for k in (1, 2, 8)) and all(policy.decided[k] == 0 for k in (64, 96, 512))
assert policy.exploring(16) and policy.exploring(48) # unmeasured keys are searched on the served rounds
def test_tree_decisions_survive_a_restart_on_the_same_setup(tmp_path, clock):
key = {"gpu": "NVIDIA B300 SXM6 AC", "gpu_count": 4}
_serve_tree(_tree(tmp_path, key=key), clock, 128, spec_ms=30, native_ms=10, accept=2.5)
restarted = _tree(tmp_path, key=key, writer=False)
assert restarted.width(130) == 0 and restarted.width(512) == 0 and restarted.exploring(64)
assert _tree(tmp_path, key={**key, "gpu_count": 8}).decided == {}
def test_other_ranks_never_read_the_trees_tokens(clock):
policy = _tree(writer=False, sync=lambda step: ("decide", 0, None))
requests = _reqs(1)
while policy.exploring(256):
width = policy.width(256)
policy.round_start(256, width, requests)
policy.accepted(width, torch.ones(256, dtype=torch.int32))
clock.advance(1)
assert policy._tokens is None and policy.decided[256] == 0
def test_every_serve_mode_tree_chooses_native_decode_on_its_served_rounds():
assert served_tree(HarnessConfig(), 4)
assert served_tree(HarnessConfig(data_path="/cost"), 4)
assert not served_tree(HarnessConfig(), 1)
assert not served_tree(HarnessConfig(mode=Mode.STATIC), 4)