Skip to content

Official Inference Code of EAGLE-4

fix(sglang): size chain rounds with the chain-depth rule

Seyfal Sultanov511eb934cf53
diff --git a/eagle_harness/core/cost_model.py b/eagle_harness/core/cost_model.py
index 5673fec..1b4fc34 100644
--- a/eagle_harness/core/cost_model.py
+++ b/eagle_harness/core/cost_model.py
@@ -203,10 +203,10 @@ class WidthDecision:
class WidthStep(enum.Enum):
"""What ``tree_width`` needs at a depth."""
- KEEP = "keep" # tiny batch, depth < 2: keep the top-k width
+ KEEP = "keep" # tiny batch, depth < 2 (a chain: depth 0): keep the top-k width
LAST = "last" # last depth: stop
- LAUNCH = "launch" # even depth: launch the score-delta copy, keep width
- DECIDE = "decide" # odd depth: decide from the host deltas
+ LAUNCH = "launch" # even depth (a chain: odd): launch the score-delta copy, keep width
+ DECIDE = "decide" # odd depth (a chain: even): decide from the host deltas
def find_majority(nums: Sequence[int]) -> int:
@@ -543,10 +543,26 @@ class CostModel:
return self.verify_num
return max(self.verify_num, 32, *_MIN_HISTORY_VERIFY.values())
+ @property
+ def chain_rounds(self) -> bool:
+ """Rounds of a top-1 chain at chain intensity: sized by :meth:`chain_stop`, not the tree rule, and every
+ drafted token is verified."""
+ return self.intensity is ComputeIntensity.CHAIN and self.node_top_k == 1
+
+ @property
+ def verifies_whole_draft(self) -> bool:
+ """Whether a round verifies every drafted token, without the paged verify budget."""
+ return self.chain_rounds
+
+ def deltas_budget(self, past_verify_num: int) -> int:
+ """The verify budget whose work deltas a ``DECIDE`` step reads: the round's history budget, or the whole
+ chain for chain rounds."""
+ return self.verify_num if self.chain_rounds else past_verify_num
+
@property
def launch_depths(self) -> tuple[int, ...]:
"""Depths at which :meth:`width_step` can return ``LAUNCH`` (the depths whose work deltas are read)."""
- return tuple(d for d in range(0, self.tree_depth - 1, 2))
+ return tuple(range(1 if self.chain_rounds else 0, self.tree_depth - 1, 2))
def record_verify_num(self, batch_size: int, verify_num: int) -> None:
"""Push this round's verify budget into the batch size's history."""
@@ -567,7 +583,16 @@ class CostModel:
return min(self.node_top_k, 4)
def width_step(self, depth: int) -> WidthStep:
- """How the frontier width changes after depth ``depth``."""
+ """How the frontier width changes after depth ``depth``.
+
+ A chain round follows :meth:`chain_stop`'s pattern: depth 0 keeps, odd depths launch, even depths decide.
+ """
+ if self.chain_rounds:
+ if depth == self.tree_depth - 1:
+ return WidthStep.LAST
+ if depth == 0:
+ return WidthStep.KEEP
+ return WidthStep.LAUNCH if depth % 2 == 1 else WidthStep.DECIDE
if self.tier.tiny and depth < 2:
return WidthStep.KEEP
if depth == self.tree_depth - 1:
@@ -597,9 +622,15 @@ class CostModel:
"""Width of the next draft level, or stop.
``deltas`` are the host copy of the top work-score deltas launched at
- the previous (even) depth; only the ``DECIDE`` step reads them.
+ the previous (even; a chain: odd) depth; only the ``DECIDE`` step reads them.
+ A chain round decides with :meth:`chain_stop` and keeps width 1.
"""
step = self.width_step(depth)
+ if self.chain_rounds and step is not WidthStep.LAST:
+ if step is WidthStep.DECIDE and deltas is None:
+ raise ValueError(f"tree_width at depth {depth} needs the host score deltas.")
+ stop = step is WidthStep.DECIDE and self.chain_stop(depth, batch_size, deltas)
+ return WidthDecision(1, past_verify_num, stop)
if step is WidthStep.KEEP:
return WidthDecision(self.node_top_k, past_verify_num, False)
if step is WidthStep.LAST:
@@ -854,11 +885,20 @@ class FixedShapePolicy:
launch_depths: tuple[int, ...] = ()
+ @property
+ def verifies_whole_draft(self) -> bool:
+ """No: the verify budget comes from :meth:`verify_num_paged`."""
+ return False
+
@property
def max_history_verify_num(self) -> int:
"""The widest verify budget this policy asks the tree for."""
return self.verify_num
+ def deltas_budget(self, past_verify_num: int) -> int:
+ """Never read: this policy has no ``DECIDE`` step."""
+ return past_verify_num
+
def width_step(self, depth: int) -> WidthStep:
"""Keep drafting through the last depth; never ask for host deltas."""
return WidthStep.LAST if depth >= self.tree_depth - 1 else WidthStep.KEEP
diff --git a/eagle_harness/sglang/drafter.py b/eagle_harness/sglang/drafter.py
index 6362112..cef8518 100644
--- a/eagle_harness/sglang/drafter.py
+++ b/eagle_harness/sglang/drafter.py
@@ -561,7 +561,8 @@ class DynamicDrafter:
self._deltas_slot = depth
def _decide(self, depth: int, bs: int, row: Optional[DraftCostRow], past: int, policy) -> bool:
- """Decide whether to stop growing after the layer of ``depth`` (stop = True).
+ """Decide whether to stop growing after the layer of ``depth`` (stop = True); chain rounds decide with
+ :meth:`~eagle_harness.core.cost_model.CostModel.chain_stop`.
The work deltas of a LAUNCH depth are already on their way to the host
(the step's graph copies them); the next depth reads them.
@@ -575,8 +576,9 @@ class DynamicDrafter:
self._launch_deltas(depth)
return False
spin_wait(self._deltas_ready)
- # Row past - 1: deltas for verify budget ``past`` (sized by the cost model's bound, not the verify width).
- deltas = self.steps.host_deltas.numpy()[self._deltas_slot, past - 1].tolist()
+ # Row budget - 1: deltas for the policy's verify budget (sized by the cost model's bound, not the verify width).
+ budget = policy.deltas_budget(past)
+ deltas = self.steps.host_deltas.numpy()[self._deltas_slot, budget - 1].tolist()
return policy.tree_width(depth, bs, past, row, deltas).stop
# ------------------------------------------------------------------ width
@@ -584,7 +586,9 @@ class DynamicDrafter:
self, policy, scores: List[float], verify_count: int, bs: int, context: int, *, lagged: bool = True
) -> int:
"""The verify width from the policy, fed cumulative verify scores: the previous round's
- (``lagged``), or this round's on the first round."""
+ (``lagged``), or this round's on the first round. A chain verifies every drafted token."""
+ if policy.verifies_whole_draft:
+ return verify_count
width, _ = policy.verify_num_paged(scores, verify_count, bs, context, lagged=lagged)
policy.record_verify_num(bs, width)
return min(width, verify_count)
diff --git a/eagle_harness/sglang/worker.py b/eagle_harness/sglang/worker.py
index 3d6e9b2..6f8c60b 100644
--- a/eagle_harness/sglang/worker.py
+++ b/eagle_harness/sglang/worker.py
@@ -252,7 +252,7 @@ class EagleHarnessWorker(EAGLEWorkerV2):
if schedule is None:
logger.warning(
"EAGLE_HARNESS: no served chain round times with an acceptance profile under %s; the cost model "
- "sizes each chain round (tree rule)",
+ "sizes each chain round (the chain-depth rule)",
config.data_path,
)
return None
diff --git a/tests/test_cost_golden.py b/tests/test_cost_golden.py
index 7f99fe4..82c826a 100644
--- a/tests/test_cost_golden.py
+++ b/tests/test_cost_golden.py
@@ -112,6 +112,28 @@ def test_decisions_match_fork(golden, calibrated):
_replay(_model(root, intensity, case["case"]), case["script"])
+def test_chain_rounds_decide_with_the_fork_chain_rule(golden, calibrated):
+ """A top-1 chain's width decision (what the sglang drafter asks) is chain_stop: the golden chain decisions
+ replayed through tree_width (the last depth always ends the draft)."""
+ intensity, root = calibrated
+ if intensity is not ComputeIntensity.CHAIN:
+ pytest.skip("chain rule only")
+ for case in golden["decisions_chain_intensity"]:
+ cm = _model(root, intensity, case["case"])
+ assert cm.chain_rounds and cm.verifies_whole_draft and cm.deltas_budget(4) == cm.verify_num
+ decided = 0
+ for step in case["script"]:
+ if step["op"] == "update_interval":
+ cm.update_interval(*step["args"])
+ elif step["op"] == "chain":
+ depth, bs, deltas = step["args"]
+ d = cm.tree_width(depth, bs, cm.verify_num, None, deltas)
+ assert d.width in (1, -1)
+ assert d.stop == (step["out"] or depth == cm.tree_depth - 1), step
+ decided += step["out"]
+ assert decided > 0
+
+
def _replay(cm: CostModel, script: list) -> None:
row = None
assert len(script) > 100
diff --git a/tests/test_drafter.py b/tests/test_drafter.py
index b0e64af..2aede05 100644
--- a/tests/test_drafter.py
+++ b/tests/test_drafter.py
@@ -154,6 +154,97 @@ def test_fixed_shape_runs_every_step_without_host_decisions():
assert fixed.verify_count(1, 5) == 5 and fixed.history_verify_num(1) == WIDTH
+CHAIN_DEPTH, CHAIN_WIDTH = 15, 16
+
+
+class _ChainStopAt:
+ """A chain cost model whose chain rule stops at depth ``stop``; records the deltas each decision reads."""
+
+ def __init__(self, stop: int):
+ self.cost = CostModel(
+ tables={},
+ params=CostModelParams(),
+ intensity=ComputeIntensity.CHAIN,
+ node_top_k=1,
+ tree_depth=CHAIN_DEPTH,
+ verify_num=CHAIN_WIDTH,
+ )
+ self.stop, self.read = stop, []
+ self.cost.chain_stop = self._chain_stop
+
+ def _chain_stop(self, depth, bs, deltas):
+ self.read.append((depth, deltas))
+ return depth >= self.stop
+
+
+def _chain_sequenced(stop: int):
+ policy = _ChainStopAt(stop)
+ cost = policy.cost
+ slots = cost.launch_depths
+ tree = DraftTree(
+ TreeShape(1, CHAIN_DEPTH), 1, H, torch.float32, torch.device("cpu"), max_verify=CHAIN_WIDTH, delta_slots=slots
+ )
+ # Host deltas: slot d, budget row v - 1 holds (d, v) so a test can tell which row a decision read.
+ host = torch.tensor([[[float(d), float(v + 1)] for v in range(CHAIN_WIDTH)] for d in range(CHAIN_DEPTH)])
+ steps = _Steps(tree=tree, ran=[], max_step=CHAIN_DEPTH - 1, host_deltas=host)
+ drafter = DynamicDrafter.__new__(DynamicDrafter)
+ drafter.steps, drafter.tree, drafter.cost = steps, tree, cost
+ drafter.live, drafter.timer, drafter._seeding = None, None, None
+ drafter._deltas_ready, drafter._deltas_slot = _Event(), 0
+ drafter.policy = lambda bs: cost
+ return drafter, steps, policy
+
+
[email protected]("stop", [2, 6, 12])
+def test_chain_rounds_stop_with_the_chain_rule(stop):
+ """A top-1 chain at chain intensity asks the chain rule (CostModel.chain_stop), not the tree rule:
+ depth 0 keeps, odd depths launch their deltas, even depths decide from the whole-chain budget's deltas."""
+ drafter, steps, policy = _chain_sequenced(stop)
+ cost = policy.cost
+ assert cost.launch_depths == (1, 3, 5, 7, 9, 11, 13)
+ first = drafter._queue_until_decision(1, cost)
+ assert first == 2 and steps.ran == [1, 2] and drafter._deltas_slot == 1
+ for _ in range(16): # a history of narrow verifies: the tree rule's budget would read another deltas row
+ cost.record_verify_num(1, 4)
+ past = cost.history_verify_num(1)
+ assert past < CHAIN_WIDTH
+ assert drafter._grow(1, first, None, past, cost) == stop + 1
+ assert steps.ran == list(range(1, stop + 1))
+ assert policy.read == [(d, [float(d - 1), float(CHAIN_WIDTH)]) for d in range(2, stop + 1, 2)]
+
+
+def test_chain_rounds_draft_the_whole_chain_when_the_rule_never_stops():
+ drafter, steps, policy = _chain_sequenced(stop=99)
+ cost = policy.cost
+ first = drafter._queue_until_decision(1, cost)
+ assert drafter._grow(1, first, None, cost.history_verify_num(1), cost) == CHAIN_DEPTH
+ assert steps.ran == list(range(1, CHAIN_DEPTH))
+ assert [d for d, _ in policy.read] == list(range(2, CHAIN_DEPTH - 1, 2))
+
+
+def test_chain_rounds_verify_every_drafted_token():
+ """A chain verifies every drafted token: the paged verify budget is not asked; the history is not touched."""
+ drafter, _, policy = _chain_sequenced(stop=4)
+ cost = policy.cost
+
+ def refuse(*args, **kwargs):
+ raise AssertionError("a chain round asked for the paged verify budget")
+
+ cost.verify_num_paged = refuse
+ history = [list(h) for h in cost._history]
+ assert cost.verify_count(1, 6) == 6
+ assert drafter._verify_width(cost, [0.0] * 6, 6, 1, 100) == 6
+ assert cost._history == history
+
+
+def test_tree_rounds_keep_the_tree_rule():
+ cost = _cost()
+ cost.tier.tiny = True
+ assert not cost.verifies_whole_draft and cost.deltas_budget(32) == 32
+ assert [cost.width_step(d).value for d in range(4)] == ["keep", "keep", "launch", "decide"]
+ assert cost.launch_depths == tuple(range(0, DEPTH - 1, 2))
+
+
def test_batch_key_is_the_block_table_rows():
from eagle_harness.sglang.drafter import batch_key