Official Inference Code of EAGLE-4
fix(sglang): size chain rounds with the chain-depth rule
Seyfal Sultanov511eb934cf53
diff --git a/eagle_harness/core/cost_model.py b/eagle_harness/core/cost_model.pyindex 5673fec..1b4fc34 100644--- a/eagle_harness/core/cost_model.py+++ b/eagle_harness/core/cost_model.py@@ -203,10 +203,10 @@ class WidthDecision:class WidthStep(enum.Enum):"""What ``tree_width`` needs at a depth."""- KEEP = "keep" # tiny batch, depth < 2: keep the top-k width+ KEEP = "keep" # tiny batch, depth < 2 (a chain: depth 0): keep the top-k widthLAST = "last" # last depth: stop- LAUNCH = "launch" # even depth: launch the score-delta copy, keep width- DECIDE = "decide" # odd depth: decide from the host deltas+ LAUNCH = "launch" # even depth (a chain: odd): launch the score-delta copy, keep width+ DECIDE = "decide" # odd depth (a chain: even): decide from the host deltasdef find_majority(nums: Sequence[int]) -> int:@@ -543,10 +543,26 @@ class CostModel:return self.verify_numreturn max(self.verify_num, 32, *_MIN_HISTORY_VERIFY.values())+ @property+ def chain_rounds(self) -> bool:+ """Rounds of a top-1 chain at chain intensity: sized by :meth:`chain_stop`, not the tree rule, and every+ drafted token is verified."""+ return self.intensity is ComputeIntensity.CHAIN and self.node_top_k == 1++ @property+ def verifies_whole_draft(self) -> bool:+ """Whether a round verifies every drafted token, without the paged verify budget."""+ return self.chain_rounds++ def deltas_budget(self, past_verify_num: int) -> int:+ """The verify budget whose work deltas a ``DECIDE`` step reads: the round's history budget, or the whole+ chain for chain rounds."""+ return self.verify_num if self.chain_rounds else past_verify_num+@propertydef launch_depths(self) -> tuple[int, ...]:"""Depths at which :meth:`width_step` can return ``LAUNCH`` (the depths whose work deltas are read)."""- return tuple(d for d in range(0, self.tree_depth - 1, 2))+ return tuple(range(1 if self.chain_rounds else 0, self.tree_depth - 1, 2))def record_verify_num(self, batch_size: int, verify_num: int) -> None:"""Push this round's verify budget into the batch size's history."""@@ -567,7 +583,16 @@ class CostModel:return min(self.node_top_k, 4)def width_step(self, depth: int) -> WidthStep:- """How the frontier width changes after depth ``depth``."""+ """How the frontier width changes after depth ``depth``.++ A chain round follows :meth:`chain_stop`'s pattern: depth 0 keeps, odd depths launch, even depths decide.+ """+ if self.chain_rounds:+ if depth == self.tree_depth - 1:+ return WidthStep.LAST+ if depth == 0:+ return WidthStep.KEEP+ return WidthStep.LAUNCH if depth % 2 == 1 else WidthStep.DECIDEif self.tier.tiny and depth < 2:return WidthStep.KEEPif depth == self.tree_depth - 1:@@ -597,9 +622,15 @@ class CostModel:"""Width of the next draft level, or stop.``deltas`` are the host copy of the top work-score deltas launched at- the previous (even) depth; only the ``DECIDE`` step reads them.+ the previous (even; a chain: odd) depth; only the ``DECIDE`` step reads them.+ A chain round decides with :meth:`chain_stop` and keeps width 1."""step = self.width_step(depth)+ if self.chain_rounds and step is not WidthStep.LAST:+ if step is WidthStep.DECIDE and deltas is None:+ raise ValueError(f"tree_width at depth {depth} needs the host score deltas.")+ stop = step is WidthStep.DECIDE and self.chain_stop(depth, batch_size, deltas)+ return WidthDecision(1, past_verify_num, stop)if step is WidthStep.KEEP:return WidthDecision(self.node_top_k, past_verify_num, False)if step is WidthStep.LAST:@@ -854,11 +885,20 @@ class FixedShapePolicy:launch_depths: tuple[int, ...] = ()+ @property+ def verifies_whole_draft(self) -> bool:+ """No: the verify budget comes from :meth:`verify_num_paged`."""+ return False+@propertydef max_history_verify_num(self) -> int:"""The widest verify budget this policy asks the tree for."""return self.verify_num+ def deltas_budget(self, past_verify_num: int) -> int:+ """Never read: this policy has no ``DECIDE`` step."""+ return past_verify_num+def width_step(self, depth: int) -> WidthStep:"""Keep drafting through the last depth; never ask for host deltas."""return WidthStep.LAST if depth >= self.tree_depth - 1 else WidthStep.KEEPdiff --git a/eagle_harness/sglang/drafter.py b/eagle_harness/sglang/drafter.pyindex 6362112..cef8518 100644--- a/eagle_harness/sglang/drafter.py+++ b/eagle_harness/sglang/drafter.py@@ -561,7 +561,8 @@ class DynamicDrafter:self._deltas_slot = depthdef _decide(self, depth: int, bs: int, row: Optional[DraftCostRow], past: int, policy) -> bool:- """Decide whether to stop growing after the layer of ``depth`` (stop = True).+ """Decide whether to stop growing after the layer of ``depth`` (stop = True); chain rounds decide with+ :meth:`~eagle_harness.core.cost_model.CostModel.chain_stop`.The work deltas of a LAUNCH depth are already on their way to the host(the step's graph copies them); the next depth reads them.@@ -575,8 +576,9 @@ class DynamicDrafter:self._launch_deltas(depth)return Falsespin_wait(self._deltas_ready)- # Row past - 1: deltas for verify budget ``past`` (sized by the cost model's bound, not the verify width).- deltas = self.steps.host_deltas.numpy()[self._deltas_slot, past - 1].tolist()+ # Row budget - 1: deltas for the policy's verify budget (sized by the cost model's bound, not the verify width).+ budget = policy.deltas_budget(past)+ deltas = self.steps.host_deltas.numpy()[self._deltas_slot, budget - 1].tolist()return policy.tree_width(depth, bs, past, row, deltas).stop# ------------------------------------------------------------------ width@@ -584,7 +586,9 @@ class DynamicDrafter:self, policy, scores: List[float], verify_count: int, bs: int, context: int, *, lagged: bool = True) -> int:"""The verify width from the policy, fed cumulative verify scores: the previous round's- (``lagged``), or this round's on the first round."""+ (``lagged``), or this round's on the first round. A chain verifies every drafted token."""+ if policy.verifies_whole_draft:+ return verify_countwidth, _ = policy.verify_num_paged(scores, verify_count, bs, context, lagged=lagged)policy.record_verify_num(bs, width)return min(width, verify_count)diff --git a/eagle_harness/sglang/worker.py b/eagle_harness/sglang/worker.pyindex 3d6e9b2..6f8c60b 100644--- a/eagle_harness/sglang/worker.py+++ b/eagle_harness/sglang/worker.py@@ -252,7 +252,7 @@ class EagleHarnessWorker(EAGLEWorkerV2):if schedule is None:logger.warning("EAGLE_HARNESS: no served chain round times with an acceptance profile under %s; the cost model "- "sizes each chain round (tree rule)",+ "sizes each chain round (the chain-depth rule)",config.data_path,)return Nonediff --git a/tests/test_cost_golden.py b/tests/test_cost_golden.pyindex 7f99fe4..82c826a 100644--- a/tests/test_cost_golden.py+++ b/tests/test_cost_golden.py@@ -112,6 +112,28 @@ def test_decisions_match_fork(golden, calibrated):_replay(_model(root, intensity, case["case"]), case["script"])+def test_chain_rounds_decide_with_the_fork_chain_rule(golden, calibrated):+ """A top-1 chain's width decision (what the sglang drafter asks) is chain_stop: the golden chain decisions+ replayed through tree_width (the last depth always ends the draft)."""+ intensity, root = calibrated+ if intensity is not ComputeIntensity.CHAIN:+ pytest.skip("chain rule only")+ for case in golden["decisions_chain_intensity"]:+ cm = _model(root, intensity, case["case"])+ assert cm.chain_rounds and cm.verifies_whole_draft and cm.deltas_budget(4) == cm.verify_num+ decided = 0+ for step in case["script"]:+ if step["op"] == "update_interval":+ cm.update_interval(*step["args"])+ elif step["op"] == "chain":+ depth, bs, deltas = step["args"]+ d = cm.tree_width(depth, bs, cm.verify_num, None, deltas)+ assert d.width in (1, -1)+ assert d.stop == (step["out"] or depth == cm.tree_depth - 1), step+ decided += step["out"]+ assert decided > 0++def _replay(cm: CostModel, script: list) -> None:row = Noneassert len(script) > 100diff --git a/tests/test_drafter.py b/tests/test_drafter.pyindex b0e64af..2aede05 100644--- a/tests/test_drafter.py+++ b/tests/test_drafter.py@@ -154,6 +154,97 @@ def test_fixed_shape_runs_every_step_without_host_decisions():assert fixed.verify_count(1, 5) == 5 and fixed.history_verify_num(1) == WIDTH+CHAIN_DEPTH, CHAIN_WIDTH = 15, 16+++class _ChainStopAt:+ """A chain cost model whose chain rule stops at depth ``stop``; records the deltas each decision reads."""++ def __init__(self, stop: int):+ self.cost = CostModel(+ tables={},+ params=CostModelParams(),+ intensity=ComputeIntensity.CHAIN,+ node_top_k=1,+ tree_depth=CHAIN_DEPTH,+ verify_num=CHAIN_WIDTH,+ )+ self.stop, self.read = stop, []+ self.cost.chain_stop = self._chain_stop++ def _chain_stop(self, depth, bs, deltas):+ self.read.append((depth, deltas))+ return depth >= self.stop+++def _chain_sequenced(stop: int):+ policy = _ChainStopAt(stop)+ cost = policy.cost+ slots = cost.launch_depths+ tree = DraftTree(+ TreeShape(1, CHAIN_DEPTH), 1, H, torch.float32, torch.device("cpu"), max_verify=CHAIN_WIDTH, delta_slots=slots+ )+ # Host deltas: slot d, budget row v - 1 holds (d, v) so a test can tell which row a decision read.+ host = torch.tensor([[[float(d), float(v + 1)] for v in range(CHAIN_WIDTH)] for d in range(CHAIN_DEPTH)])+ steps = _Steps(tree=tree, ran=[], max_step=CHAIN_DEPTH - 1, host_deltas=host)+ drafter = DynamicDrafter.__new__(DynamicDrafter)+ drafter.steps, drafter.tree, drafter.cost = steps, tree, cost+ drafter.live, drafter.timer, drafter._seeding = None, None, None+ drafter._deltas_ready, drafter._deltas_slot = _Event(), 0+ drafter.policy = lambda bs: cost+ return drafter, steps, policy++[email protected]("stop", [2, 6, 12])+def test_chain_rounds_stop_with_the_chain_rule(stop):+ """A top-1 chain at chain intensity asks the chain rule (CostModel.chain_stop), not the tree rule:+ depth 0 keeps, odd depths launch their deltas, even depths decide from the whole-chain budget's deltas."""+ drafter, steps, policy = _chain_sequenced(stop)+ cost = policy.cost+ assert cost.launch_depths == (1, 3, 5, 7, 9, 11, 13)+ first = drafter._queue_until_decision(1, cost)+ assert first == 2 and steps.ran == [1, 2] and drafter._deltas_slot == 1+ for _ in range(16): # a history of narrow verifies: the tree rule's budget would read another deltas row+ cost.record_verify_num(1, 4)+ past = cost.history_verify_num(1)+ assert past < CHAIN_WIDTH+ assert drafter._grow(1, first, None, past, cost) == stop + 1+ assert steps.ran == list(range(1, stop + 1))+ assert policy.read == [(d, [float(d - 1), float(CHAIN_WIDTH)]) for d in range(2, stop + 1, 2)]+++def test_chain_rounds_draft_the_whole_chain_when_the_rule_never_stops():+ drafter, steps, policy = _chain_sequenced(stop=99)+ cost = policy.cost+ first = drafter._queue_until_decision(1, cost)+ assert drafter._grow(1, first, None, cost.history_verify_num(1), cost) == CHAIN_DEPTH+ assert steps.ran == list(range(1, CHAIN_DEPTH))+ assert [d for d, _ in policy.read] == list(range(2, CHAIN_DEPTH - 1, 2))+++def test_chain_rounds_verify_every_drafted_token():+ """A chain verifies every drafted token: the paged verify budget is not asked; the history is not touched."""+ drafter, _, policy = _chain_sequenced(stop=4)+ cost = policy.cost++ def refuse(*args, **kwargs):+ raise AssertionError("a chain round asked for the paged verify budget")++ cost.verify_num_paged = refuse+ history = [list(h) for h in cost._history]+ assert cost.verify_count(1, 6) == 6+ assert drafter._verify_width(cost, [0.0] * 6, 6, 1, 100) == 6+ assert cost._history == history+++def test_tree_rounds_keep_the_tree_rule():+ cost = _cost()+ cost.tier.tiny = True+ assert not cost.verifies_whole_draft and cost.deltas_budget(32) == 32+ assert [cost.width_step(d).value for d in range(4)] == ["keep", "keep", "launch", "decide"]+ assert cost.launch_depths == tuple(range(0, DEPTH - 1, 2))++def test_batch_key_is_the_block_table_rows():from eagle_harness.sglang.drafter import batch_key