Official Inference Code of EAGLE-4
# Copyright 2026 Eagle AI Corp.
# SPDX-License-Identifier: LicenseRef-Eagle-4-Source-Available
"""No-spec pricing and request-lifetime native decode contracts."""
from types import SimpleNamespace
import pytest
from eagle_harness.core.chain_width import AcceptProfile, save_round_times, served_chain_width
def test_real_no_spec_is_a_separate_price_from_width_one():
profile = AcceptProfile.from_counts(10, [8, 5, 2])
prices = {0: 100, 1: 150, 2: 300, 4: 400}
assert served_chain_width(prices, profile, 4, allow_no_spec=True) == 0
# Adapters without a native decode route retain their old action set.
assert served_chain_width(prices, profile, 4) == 1
assert served_chain_width({**prices, 4: 200}, profile, 4, allow_no_spec=True) == 4
def test_no_spec_round_measurement_can_be_saved(tmp_path):
save_round_times(str(tmp_path), [{"batch_size": 64, "width": 0, "round_us": 100, "count": 20}])
def test_calibration_times_off_last_and_never_resumes_stale_draft_kv(tmp_path):
from collections import Counter
import torch
from eagle_harness.sglang.chain_rounds import MIN_ROUNDS, ChainCalibration
calibration = ChainCalibration(str(tmp_path), [1, 2, 4], torch.device("cpu"), writer=False, native_decode=True)
choices = [calibration.width(8) for _ in range(100)]
assert set(choices) == {0, 1, 2, 4}
assert all(Counter(choices)[width] >= MIN_ROUNDS + 2 for width in (1, 2, 4))
assert set(choices[choices.index(0) :]) == {0}
calibration.interrupt()
assert calibration.width(8) == 1
def test_native_decode_has_a_separate_graph_from_width_one():
pytest.importorskip("sglang")
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.runner.decode_graph_variants import DecodeGraphVariant
from eagle_harness.core.limits import ComputeIntensity
from eagle_harness.sglang.verify_graphs import VerifyWidthGraphs
if not hasattr(DecodeGraphVariant, "forward_mode"):
pytest.skip("host has no per-variant forward mode")
graphs = VerifyWidthGraphs(ComputeIntensity.CHAIN, 8, None, plain_decode=True)
variants = list(graphs.capture_variants(8))
native = [v for v in variants if v.forward_mode is ForwardMode.DECODE]
assert len(native) == 1 and native[0].num_tokens_per_req == 1
assert any(v.num_tokens_per_req == 1 and v.forward_mode is None for v in variants)
batch = SimpleNamespace(forward_mode=ForwardMode.DECODE, spec_info=None)
assert graphs.replay_variant(batch) == native[0]
def test_disabled_requests_never_resume_with_stale_draft_kv():
pytest.importorskip("sglang")
from eagle_harness.sglang.plain_decode import NoSpecRequests
class Request:
pass
a, b, fresh = Request(), Request(), Request()
policy = NoSpecRequests()
assert not policy.select([a], 8)
assert policy.select([a, b], 0)
assert policy.select([b], 8) # Shrinking the batch cannot revive stale KV.
assert policy.select([b, fresh], 8) # Mixed batches follow the same route.
assert policy.select([fresh], 8)
assert not policy.select([Request()], 8)
def test_worker_off_dispatch_skips_draft_verify_and_prelaunch(monkeypatch):
pytest.importorskip("sglang")
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from eagle_harness.sglang import plain_decode as plain_module
from eagle_harness.sglang import worker as module
from eagle_harness.sglang.chain_rounds import ChainRounds
class Request:
pass
calls = []
worker = module.EagleHarnessWorker.__new__(module.EagleHarnessWorker)
worker._chain_rounds = ChainRounds([(1, 256, 0)], None, lambda batch: 0)
worker._drafter = SimpleNamespace(cancel_prelaunch=lambda: calls.append("cancel"))
worker._verified = SimpleNamespace(record=lambda: calls.append("record"), query=lambda: True)
worker._target_worker = object()
worker.req_to_token_pool = SimpleNamespace(req_to_token=object())
worker._native_decode = plain_module.NativeDecode(
worker.target_worker, worker.req_to_token_pool.req_to_token, worker._chain_rounds, enabled=True
)
batch = SimpleNamespace(forward_mode=ForwardMode.DECODE, is_extend_in_batch=False, reqs=[Request()])
result = SimpleNamespace(new_seq_lens=[5], new_seq_lens_cpu=[5])
monkeypatch.setattr(plain_module, "plain_decode", lambda *args, **kwargs: calls.append("decode") or result)
worker._native_decode._requests.publishes_host_lengths = True
on_publish = lambda lengths, **host: calls.append(("publish", lengths, host)) # noqa: E731
assert worker.forward_batch_generation(batch, on_publish=on_publish) is result
assert calls == ["cancel", "decode", "record", ("publish", [5], {"new_seq_lens_cpu": [5]})]
def test_off_publish_sends_host_lengths_only_to_a_host_that_reads_them():
pytest.importorskip("sglang")
from eagle_harness.sglang.plain_decode import NoSpecRequests
calls = []
policy = NoSpecRequests()
publish = lambda lengths, **host: calls.append((lengths, host)) # noqa: E731
policy.publishes_host_lengths = True
policy.publish(publish, SimpleNamespace(new_seq_lens=[5], new_seq_lens_cpu=[5]))
policy.publish(publish, SimpleNamespace(new_seq_lens=[6], new_seq_lens_cpu=None))
policy.publishes_host_lengths = False
policy.publish(publish, SimpleNamespace(new_seq_lens=[7], new_seq_lens_cpu=[7]))
assert calls == [([5], {"new_seq_lens_cpu": [5]}), ([6], {}), ([7], {})]
def test_plain_decode_consumes_reserved_slot_once_and_preserves_roots():
pytest.importorskip("sglang")
import torch
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from eagle_harness.sglang.plain_decode import plain_decode
draft = SimpleNamespace(bonus_tokens=torch.tensor([17, 19], dtype=torch.int32), hidden_states=object())
algorithm = object()
batch = SimpleNamespace(
spec_info=draft,
spec_algorithm=algorithm,
req_pool_indices=torch.tensor([3, 1]),
seq_lens=torch.tensor([5, 7]),
seq_lens_cpu=torch.tensor([5, 7]),
return_logprob=False,
)
slots = torch.arange(50).reshape(5, 10)
calls = []
def forward(actual, **kwargs):
calls.append(actual)
assert actual.forward_mode is ForwardMode.DECODE
assert actual.spec_algorithm is SpeculativeAlgorithm.NONE
assert actual.spec_info is None
assert actual.input_ids.tolist() == [17, 19]
assert actual.out_cache_loc.tolist() == [35, 17]
assert actual.seq_lens.tolist() == actual.seq_lens_cpu.tolist() == [6, 8]
return SimpleNamespace(next_token_ids=torch.tensor([23, 29]))
result = plain_decode(batch, SimpleNamespace(forward_batch_generation=forward), slots)
assert len(calls) == 1
assert result.new_seq_lens.tolist() == [6, 8]
assert result.new_seq_lens_cpu.tolist() == [6, 8] # Host-known: the relay needs no device pull.
assert result.accept_lens.tolist() == [1, 1]
assert result.accept_index.tolist() == [[0], [1]]
assert result.next_draft_input.bonus_tokens.tolist() == [23, 29]
assert result.next_draft_input.bonus_tokens is result.next_token_ids
assert result.next_draft_input.hidden_states is draft.hidden_states
assert draft.bonus_tokens.tolist() == [17, 19]
assert batch.spec_info is draft and batch.spec_algorithm is algorithm
def test_plain_decode_logprobs_are_consumed_as_one_accepted_token():
pytest.importorskip("sglang")
import torch
from sglang.srt.managers.scheduler_components.batch_result_processor import SchedulerBatchResultProcessor
from eagle_harness.sglang.plain_decode import plain_decode
batch = SimpleNamespace(
spec_info=SimpleNamespace(bonus_tokens=torch.tensor([17, 19])),
spec_algorithm=SimpleNamespace(is_none=lambda: False),
req_pool_indices=torch.tensor([0, 1]),
seq_lens=torch.tensor([2, 3]),
seq_lens_cpu=None,
return_logprob=True,
)
logits = SimpleNamespace(
next_token_logprobs=torch.tensor([-0.25, -0.5]),
next_token_top_logprobs_val=[[-0.25, -1.0], [-0.5, -2.0]],
next_token_top_logprobs_idx=[[23, 7], [29, 11]],
next_token_token_ids_logprobs_val=[[-3.0], [-4.0]],
next_token_token_ids_logprobs_idx=[[13], [13]],
)
native = SimpleNamespace(next_token_ids=torch.tensor([23, 29]), logits_output=logits)
result = plain_decode(
batch,
SimpleNamespace(forward_batch_generation=lambda *args, **kwargs: native),
torch.arange(10).reshape(2, 5),
)
assert result.new_seq_lens_cpu is None # No host lengths without a host mirror to advance.
for i, token in enumerate(result.next_token_ids.tolist()):
logprob = SimpleNamespace(
top_logprobs_num=2,
token_ids_logprob=[13],
output_token_logprobs_val=[],
output_token_logprobs_idx=[],
output_top_logprobs_val=[],
output_top_logprobs_idx=[],
output_token_ids_logprobs_val=[],
output_token_ids_logprobs_idx=[],
)
SchedulerBatchResultProcessor._apply_decode_logprobs(
None,
req=SimpleNamespace(logprob=logprob),
i=i,
batch=batch,
next_token_id=[token],
next_token_logprobs=result.logits_output.next_token_logprobs.tolist(),
logits_output=result.logits_output,
)
assert logprob.output_token_logprobs_val == [[-0.25], [-0.5]][i]
assert logprob.output_token_logprobs_idx == [token]
assert logprob.output_top_logprobs_val == [logits.next_token_top_logprobs_val[i]]
assert logprob.output_top_logprobs_idx == [logits.next_token_top_logprobs_idx[i]]
assert logprob.output_token_ids_logprobs_val == [logits.next_token_token_ids_logprobs_val[i]]
assert logprob.output_token_ids_logprobs_idx == [[13]]
def test_off_lifetime_survives_retraction_and_ignores_reused_pool_slot():
pytest.importorskip("sglang")
from eagle_harness.sglang.plain_decode import NoSpecRequests
class Request:
def __init__(self, slot):
self.req_pool_idx = slot
retracted, other = Request(3), Request(7)
policy = NoSpecRequests()
assert policy.select([retracted, other], 0)
assert policy.select([other], 8)
fresh = Request(3)
assert not policy.select([fresh], 8)
retracted.req_pool_idx = 9
assert policy.select([retracted], 8)
assert policy.select([fresh, retracted], 8)
assert policy.select([fresh], 8)
def test_finished_off_requests_are_not_kept_alive():
import gc
import weakref
pytest.importorskip("sglang")
from eagle_harness.sglang.plain_decode import NoSpecRequests
class Request:
pass
request = Request()
reference = weakref.ref(request)
policy = NoSpecRequests()
assert policy.select([request], 0)
del request
gc.collect()
assert reference() is None
assert not policy.select([Request()], 8)
def test_plain_decode_restores_spec_state_when_target_raises():
pytest.importorskip("sglang")
import torch
from eagle_harness.sglang.plain_decode import plain_decode
draft = SimpleNamespace(bonus_tokens=torch.tensor([17], dtype=torch.int32))
algorithm = object()
batch = SimpleNamespace(
spec_info=draft,
spec_algorithm=algorithm,
req_pool_indices=torch.tensor([0]),
seq_lens=torch.tensor([2]),
seq_lens_cpu=None,
)
def forward(actual, **kwargs):
assert actual.spec_info is None
assert actual.seq_lens_cpu is None
raise ValueError("target forward failed")
with pytest.raises(ValueError, match="target forward failed"):
plain_decode(batch, SimpleNamespace(forward_batch_generation=forward), torch.arange(4).reshape(1, 4))
assert batch.spec_info is draft and batch.spec_algorithm is algorithm
assert draft.bonus_tokens.tolist() == [17]
@pytest.mark.parametrize("tracking", [False, True])
def test_plain_decode_saves_mamba_states_on_the_track_grid_like_a_no_spec_decode(monkeypatch, tracking):
pytest.importorskip("sglang")
import torch
from sglang.srt import runtime_context
from sglang.srt.speculative import spec_utils
from eagle_harness.sglang.plain_decode import plain_decode
monkeypatch.setattr(runtime_context, "mamba_track_grid", lambda page_size: 8, raising=False)
planned = []
def prepare(batch): # the engine's verify-time track plan: slots only, no mask
planned.append(batch.seq_lens.tolist())
batch.mamba_track_indices = torch.tensor([40, 41])
batch.mamba_track_mask = None
monkeypatch.setattr(spec_utils, "prepare_mamba_track_for_verify", prepare, raising=False)
draft = SimpleNamespace(bonus_tokens=torch.tensor([17, 19], dtype=torch.int32))
batch = SimpleNamespace(
spec_info=draft,
spec_algorithm=object(),
req_pool_indices=torch.tensor([0, 1]),
seq_lens=torch.tensor([5, 7]),
seq_lens_cpu=torch.tensor([5, 7]),
return_logprob=False,
tree_cache=SimpleNamespace(page_size=1),
mamba_track_indices=None,
mamba_track_mask=None,
)
seen = {}
def forward(actual, **kwargs):
seen["indices"], seen["mask"] = actual.mamba_track_indices, actual.mamba_track_mask
return SimpleNamespace(next_token_ids=torch.tensor([23, 29]))
target = SimpleNamespace(forward_batch_generation=forward)
plain_decode(batch, target, torch.arange(20).reshape(2, 10), track_mamba=tracking)
if tracking:
# The new lengths are 6 and 8: only the second row lands on the grid of 8, after its token is in the state.
assert planned == [[6, 8]]
assert seen["indices"].tolist() == [40, 41]
assert seen["mask"].tolist() == [False, True]
else:
assert planned == [] and seen == {"indices": None, "mask": None}
@pytest.mark.parametrize("engine_tracks", [False, True])
def test_native_decode_needs_tracking_graphs_only_with_the_radix_cache(monkeypatch, engine_tracks):
pytest.importorskip("sglang")
from sglang.srt import runtime_context
from eagle_harness.sglang import plain_decode as plain_module
from eagle_harness.sglang import verify_graphs
monkeypatch.setattr(verify_graphs, "native_decode_graphs", lambda: True)
monkeypatch.setattr(plain_module, "native_decode_tracks_mamba", lambda: engine_tracks)
monkeypatch.setattr(runtime_context, "get_parallel", lambda: SimpleNamespace(dp_size=1))
for tracking in (False, True):
monkeypatch.setattr(
runtime_context,
"get_exec",
lambda tracking=tracking: SimpleNamespace(mamba=SimpleNamespace(enable_mamba_extra_buffer=tracking)),
)
available = engine_tracks or not tracking
assert plain_module.native_decode_available() is available
assert plain_module.native_decode_blocked() is not available
@pytest.mark.parametrize("native_route", [False, True])
@pytest.mark.parametrize("intensity,default", [("high_intensity", 32), ("chain_intensity", 48)])
def test_without_a_running_cap_admission_follows_memory_where_large_batches_decode_natively(
native_route, intensity, default
):
pytest.importorskip("sglang")
from eagle_harness.core.config import harness_config_from_dict
from eagle_harness.core.limits import max_batch_size
from eagle_harness.sglang.algorithm import default_batch_limits
config = harness_config_from_dict({"intensity": intensity})
running, verify = default_batch_limits(config, native_route)
if native_route:
# The verify states cover the old default only; the engine sizes admission as without speculation.
assert (running, verify) == (max_batch_size(config.intensity), default)
else:
assert (running, verify) == (default, None)
explicit = harness_config_from_dict({"intensity": intensity, "max_verify_batch_size": 8})
assert default_batch_limits(explicit, native_route) == (
(max_batch_size(config.intensity), 8) if native_route else (8, 8)
)
def test_batches_above_the_verify_limit_decode_natively_without_asking_the_policy():
pytest.importorskip("sglang")
from eagle_harness.sglang.plain_decode import NativeDecode
asked = []
rounds = SimpleNamespace(plain_width=lambda bs: asked.append(bs) or 4)
native = NativeDecode(None, None, rounds, enabled=True, verify_limit=2)
class Request:
pass
def batch(n):
return SimpleNamespace(forward_mode=SimpleNamespace(is_idle=lambda: False), reqs=[Request() for _ in range(n)])
assert not native.select(batch(2)) and asked == [2]
assert native.select(batch(3)) and asked == [2]
def test_batches_above_the_verify_limit_capture_only_the_native_decode_graph():
pytest.importorskip("sglang")
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.runner.decode_graph_variants import DecodeGraphVariant
from eagle_harness.core.limits import ComputeIntensity
from eagle_harness.sglang.verify_graphs import VerifyWidthGraphs
if not hasattr(DecodeGraphVariant, "forward_mode"):
pytest.skip("host has no per-variant forward mode")
graphs = VerifyWidthGraphs(ComputeIntensity.CHAIN, 8, None, plain_decode=True, max_verify_bs=8)
assert len(graphs.capture_variants(8)) > 1
(native,) = graphs.capture_variants(16)
assert native.forward_mode is ForwardMode.DECODE
@pytest.mark.parametrize("enabled", [True, False])
def test_the_native_route_builds_its_decode_position_kernel_before_serving(monkeypatch, enabled):
"""The engine computes native decode's positions with a JIT module (nvcc, about 3 s); a server with the native route
builds it at startup, not at its first native round mid-serving."""
pytest.importorskip("sglang")
import torch
from sglang.kernels.ops.attention import clamp_position as engine_positions
from eagle_harness.sglang import plain_decode as plain_module
built = []
monkeypatch.setattr(engine_positions, "clamp_position", lambda seq_lens: built.append(seq_lens.dtype))
monkeypatch.setattr(plain_module.torch, "ones", lambda *shape, dtype, device: torch.zeros(*shape, dtype=dtype))
on_gpu = SimpleNamespace(device=torch.device("cuda"))
plain_module.NativeDecode(None, on_gpu, None, enabled=enabled)
assert built == ([torch.int64] if enabled else [])
plain_module.NativeDecode(None, SimpleNamespace(device=torch.device("cpu")), None, enabled=enabled)
plain_module.NativeDecode(None, None, None, enabled=enabled)
assert built == ([torch.int64] if enabled else [])
def test_prefills_joining_a_batch_above_the_verify_limit_skip_the_draft_for_life():
pytest.importorskip("sglang")
from eagle_harness.sglang.plain_decode import NativeDecode
native = NativeDecode(None, None, SimpleNamespace(plain_width=lambda bs: 4), enabled=True, verify_limit=2)
class Request:
done = False
def finished(self):
return self.done
def batch(reqs):
return SimpleNamespace(forward_mode=SimpleNamespace(is_idle=lambda: False), reqs=reqs)
a, b, c, d, e, f = Request(), Request(), Request(), Request(), Request(), Request()
assert not native.skips_draft(batch([a])) and not native.skips_draft(batch([b])) # running 1, 2
assert native.skips_draft(batch([c])) # running 3 > 2: c is off before its first decode
assert native.select(batch([a, b, c]))
assert native.select(batch([a])) # a decoded natively above the limit: off for life
assert not native.skips_draft(batch([d])) # running 1 + d = 2: drafts again
assert not native.skips_draft(batch([d])) # d's next chunk is not counted twice
assert native.skips_draft(batch([c])) # a chunk of a request that is off skips at any load
assert not native.select(batch([d]))
assert native.select(batch([a, b, c])) # a burst's last decode step: every row finishes in it
a.done = b.done = c.done = True
assert not native.skips_draft(batch([e])) # the batch e joins is empty: e drafts
assert not native.skips_draft(batch([f])) # e + f = 2
def test_stale_draft_kv_counts_reused_slots_the_draft_window_reads():
import torch
from eagle_harness.sglang.draft_kv import StaleDraftKV
req_to_token = torch.arange(1, 17).view(2, 8) # row 0: slots 1-8, row 1: slots 9-16
kv = StaleDraftKV(20, "cpu")
kv.target_only(torch.tensor([3, 4, 12]))
kv.drafted(torch.tensor([4])) # a later drafted write of slot 4
kv.reused(req_to_token, rows=[0, 1], prefix_lens=[4, 0], seq_lens=[6, 3]) # row 0 reads slots 1-4
assert kv._reused.tolist() == [1, 1]
windowed = StaleDraftKV(20, "cpu", window=3)
windowed.target_only(torch.tensor([3]))
windowed.reused(req_to_token, rows=[0], prefix_lens=[4], seq_lens=[6]) # first drafted key: position 3 (slot 4)
assert windowed._reused.tolist() == [0, 0]
def test_stale_draft_kv_never_waits_for_the_gpu():
import torch
if not torch.cuda.is_available():
pytest.skip("needs a GPU")
from eagle_harness.sglang.draft_kv import StaleDraftKV
kv = StaleDraftKV(1 << 16, "cuda")
req_to_token = torch.arange(1, 1 + 4 * 512, device="cuda").view(4, 512)
slots = torch.arange(100, 213, device="cuda")
torch.cuda.synchronize()
torch.cuda.set_sync_debug_mode("error")
try:
kv.target_only(slots)
kv.drafted(slots[:8])
kv.reused(req_to_token, rows=[0, 1], prefix_lens=[300, 0], seq_lens=[400, 10])
finally:
torch.cuda.set_sync_debug_mode(0)
assert kv._reused.tolist() == [1, 105]
def test_a_kv_only_prefill_shifts_the_ids_writes_the_draft_kv_and_relays_a_placeholder(monkeypatch):
pytest.importorskip("sglang.srt.speculative.eagle_info")
import torch
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from eagle_harness.sglang.draft_kv import kv_only_prefill
calls = []
out = SimpleNamespace(next_token_ids=torch.tensor([7, 8]), kv_loc_plan="plan",
logits_output=SimpleNamespace(hidden_states="target hidden"))
def forward(batch, pp_proxy_tensors=None, capture_hidden_mode=None, return_kv_loc_plan=False):
calls.append(("target", capture_hidden_mode.name, return_kv_loc_plan))
return out
def init_new(batch, runner, **kw):
calls.append(("draft batch", batch.spec_info.hidden_states, kw["kv_loc_plan"]))
return SimpleNamespace(input_ids=batch.input_ids.clone(), extend_seq_lens=torch.tensor([3, 2]))
monkeypatch.setattr(ForwardBatch, "init_new", init_new)
runner = SimpleNamespace(token_to_kv_pool="draft pool", model=SimpleNamespace(
write_prefill_kv=lambda fb, pool: calls.append(("kv", fb.input_ids.tolist(), pool))))
batch = SimpleNamespace(reqs=[object(), object()], seq_lens=torch.tensor([5, 9]), input_ids=torch.tensor([1, 2, 3, 4, 5]),
chunked_req_next_prompt_token=None, chunked_req=None)
result = kv_only_prefill(
batch, SimpleNamespace(forward_batch_generation=forward), runner, topk=4, hidden=(16, torch.bfloat16),
on_publish=lambda lengths: calls.append(("publish", lengths.tolist())),
)
assert calls == [("target", "FULL", True), ("publish", [5, 9]), ("draft batch", "target hidden", "plan"),
("kv", [2, 3, 7, 5, 8], "draft pool")] # each position's next token; the last: sampled tokens
assert result is out and out.kv_loc_plan is None
draft = result.next_draft_input
assert draft.bonus_tokens is out.next_token_ids
assert draft.topk_p.shape == (2, 4) and draft.topk_index.shape == (2, 4) and draft.hidden_states.shape == (2, 16)
def test_the_kv_only_pass_writes_the_keys_and_values_the_attention_would():
pytest.importorskip("sglang")
import torch
memory_pool = pytest.importorskip("sglang.srt.mem_cache.memory_pool")
if not hasattr(memory_pool, "KVWriteLoc"):
pytest.skip("engine without KV write plans")
from eagle_harness.sglang.models.llama_eagle4 import LlamaForCausalLMEagle4
torch.manual_seed(0)
heads, dim, hidden = 2, 4, 8 # MHA: q, k and v each 8 wide; the layer reads concat(embeds, hidden) = 16
weight = torch.randn(3 * heads * dim, 2 * hidden)
written = {}
radix = SimpleNamespace(is_cross_attention=False, k_scale=None, v_scale=None)
attn = SimpleNamespace(
qkv_proj=SimpleNamespace(weight=weight), q_size=heads * dim, kv_size=heads * dim, head_dim=dim,
num_kv_heads=heads, attn=radix, rotary_emb=lambda positions, q, k: (q, k + positions[:, None]),
)
layer = SimpleNamespace(input_layernorm=lambda x: x, hidden_norm=lambda x: 2 * x, attn_norm=False, self_attn=attn)
embeds = torch.randn(3, hidden)
model = SimpleNamespace(hidden_size=hidden, layers=[layer], _embed=lambda ids: embeds[ids])
pool = SimpleNamespace(set_kv_buffer=lambda layer, loc, k, v, ks, vs: written.update(layer=layer, loc=loc, k=k, v=v))
target = torch.randn(3, hidden)
fb = SimpleNamespace(spec_info=SimpleNamespace(hidden_states=target), input_ids=torch.tensor([2, 0, 1]),
positions=torch.tensor([10.0, 11.0, 12.0]),
out_cache_loc=torch.tensor([5, 6, 7]), out_cache_loc_is_physical=True)
draft = SimpleNamespace(model=model)
LlamaForCausalLMEagle4.write_prefill_kv(draft, fb, pool)
_, k, v = (torch.cat([embeds[fb.input_ids], 2 * target], -1) @ weight.T).split(heads * dim, -1)
assert written["layer"] is radix and torch.equal(written["loc"].loc, fb.out_cache_loc) and written["loc"].physical
torch.testing.assert_close(written["k"], (k + fb.positions[:, None]).view(3, heads, dim))
torch.testing.assert_close(written["v"], v.view(3, heads, dim))
@pytest.mark.parametrize("skip", [True, False])
def test_the_worker_prefill_routes_and_marks_the_written_slots(monkeypatch, skip):
pytest.importorskip("sglang")
import torch
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.speculative import eagle_utils
from eagle_harness.sglang import worker as module
from eagle_harness.sglang.draft_kv import StaleDraftKV
calls = []
worker = module.EagleHarnessWorker.__new__(module.EagleHarnessWorker)
worker._stale_kv = StaleDraftKV(16, "cpu")
worker._stale_kv.target_only(torch.tensor([2])) # slot 2: decoded natively earlier
worker._native_decode = SimpleNamespace(skips_draft=lambda batch: skip)
worker.req_to_token_pool = SimpleNamespace(req_to_token=torch.arange(16).view(2, 8))
worker.topk, worker._target_worker = 4, "target"
worker._draft_worker = SimpleNamespace(draft_runner="draft")
worker._kv_only_prefill = True
monkeypatch.setattr(module.EagleHarnessWorker, "_draft_context", lambda self: __import__("contextlib").nullcontext())
monkeypatch.setattr(eagle_utils, "get_draft_recurrent_hidden_state_spec", lambda runner: (8, torch.float32))
monkeypatch.setattr(module, "kv_only_prefill", lambda batch, target, runner, **kw: calls.append(("kv only", target, runner, kw["hidden"])) or "t")
monkeypatch.setattr(module.EAGLEWorkerV2, "forward_batch_generation", lambda self, *a: calls.append("drafted") or "d")
batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND, is_extend_in_batch=False, return_hidden_states=False,
out_cache_loc=torch.tensor([4, 5]), reqs=[SimpleNamespace(multimodal_inputs=None)], prefix_lens=[4], extend_lens=[2],
req_pool_indices_cpu=torch.tensor([0]))
assert worker._prefill(batch, None, None, None) == ("t" if skip else "d")
assert calls == ([("kv only", "target", "draft", (8, torch.float32))] if skip else ["drafted"])
assert worker._stale_kv.stale[[4, 5]].tolist() == [False, False] # both routes write the draft KV
assert worker._stale_kv._reused.tolist() == ([0, 0] if skip else [1, 1]) # the drafted prefill read slot 2
def test_the_prefill_reads_batch_fields_the_engine_has():
"""The engine fields the prefill routing and the KV-only pass read (the fakes above cannot catch a renamed
field): ScheduleBatch, and ForwardBatch where the engine has KV write plans."""
pytest.importorskip("sglang")
import dataclasses
from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from eagle_harness.sglang.draft_kv import kv_only_prefill_supported
fields = {f.name for f in dataclasses.fields(ScheduleBatch)}
assert {"forward_mode", "is_extend_in_batch", "return_hidden_states", "out_cache_loc", "reqs", "prefix_lens",
"extend_lens", "seq_lens", "input_ids", "chunked_req"} <= fields
assert "req_pool_indices_cpu" in fields or hasattr(Req, "req_pool_idx")
if kv_only_prefill_supported():
fb = {f.name for f in dataclasses.fields(ForwardBatch)}
assert {"input_ids", "extend_seq_lens", "positions", "spec_info", "out_cache_loc",
"out_cache_loc_is_physical"} <= fb