Skip to content

Official Inference Code of EAGLE-4

feat(sglang): serve with only the draft path by default

Seyfal Sultanov733fa938dbab
diff --git a/docs/configuration.md b/docs/configuration.md
index 3721946..80a55cf 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1,8 +1,8 @@
# Configuration reference
The method takes one JSON object. On sglang it goes in `--speculative-algorithm-config`, inline or as a path to a
-JSON file. On vLLM it goes in the `method_config` of `--speculative-config`. On TokenSpeed it goes in
-`--speculative-algorithm-config` (inline JSON). The adapters share the core configuration parser and add
+JSON file, and leaving the flag out means `{}`. On vLLM it goes in the `method_config` of `--speculative-config`.
+On TokenSpeed it goes in `--speculative-algorithm-config` (inline JSON). The adapters share the core configuration parser and add
engine-specific keys and restrictions described below. Unknown keys are
rejected with the list of valid keys, so a typo cannot fall back to a default silently.
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index 58e4b44..165062d 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -44,12 +44,11 @@ no supported host installed (one of: sglang, vllm, tokenspeed)
| message | cause | fix |
|---|---|---|
| `Unknown speculative algorithm name: EAGLE_HARNESS` (sglang), vLLM rejects the method `eagle_harness`, or TokenSpeed does not know the algorithm `EAGLE_HARNESS` | the plugin did not register | run `eagle-harness-check` and read the error the engine logged at startup |
-| `EAGLE_HARNESS needs --speculative-algorithm-config (JSON or a path to JSON), ...` | sglang got no config | pass `{}` (live costs), `{"data_path": "<calibration dir>"}` or `{"mode": "static"}` |
| `Cost tables under ... were measured on another setup (stored vs this server): {...}` | `data_path` tables keyed for another GPU, TP size, model pair, engine build or dtype | calibrate for this setup, or drop `data_path` to measure the costs on the served rounds |
| `TokenSpeed serve mode (the default) needs data_path or a complete explicit chain schedule ...` | serve mode, set explicitly or by default, with no cost tables and no complete chain schedule (TokenSpeed) | set `data_path` to matching TokenSpeed costs ([Calibration](calibration.md)); static mode is not a correctness-qualified substitute |
| `config: unknown keys ['intensty']; valid keys are [...]` | a misspelt key | use a key from the list ([Configuration](configuration.md)) |
| `mode: 'fast' is not one of static, serve, eval, calibrate.` | a bad value | use one of the listed values |
-| `EAGLE_HARNESS needs --speculative-num-steps, --speculative-eagle-topk and --speculative-num-draft-tokens ...` | sglang got no tree shape | pass all three |
+| `EAGLE_HARNESS needs --speculative-num-steps, --speculative-eagle-topk and --speculative-num-draft-tokens ...` | sglang got part of the tree shape, or a chain config without one | pass all three, or none of them for the default tree (depth 12, top-k 4, verify width 16) |
| `max_running_requests=... exceeds the tree draft batch limit 512.` | the explicit request cap exceeds the supported range | set `--max-running-requests` to at most 512, and check that KV and recurrent-state memory can hold the requested batch |
| `eagle-harness kernels are off: ... is older than NVIDIA sm_80 (A100).` | the GPU is outside the kernel tiers | use a supported GPU ([Kernels](kernels.md)) |
| `page_size > 1 needs --speculative-draft-attention-backend flashinfer, fa3 or fa4 for EAGLE_HARNESS.` | the draft backend cannot serve paged KV | set one of those backends |
diff --git a/eagle_harness/core/config.py b/eagle_harness/core/config.py
index 07930d4..b819847 100644
--- a/eagle_harness/core/config.py
+++ b/eagle_harness/core/config.py
@@ -242,14 +242,12 @@ def harness_config_from_dict(data: Mapping[str, Any]) -> HarnessConfig:
def parse_harness_config(value: Optional[str], *, default_data_path: Optional[str] = None) -> HarnessConfig:
- """Parse JSON or a JSON file, filling an absent data_path from a device-selected profile."""
+ """Parse JSON or a JSON file, filling an absent data_path from a device-selected profile.
+
+ No value is the default config (serve mode, costs measured on the served rounds).
+ """
if value is None or not value.strip():
- if default_data_path is not None:
- return harness_config_from_dict({"data_path": default_data_path})
- raise ValueError(
- "EAGLE_HARNESS needs --speculative-algorithm-config (JSON or a path to JSON), "
- 'e.g. {"data_path": "<calibration dir>"} or {"mode": "static"}.'
- )
+ return harness_config_from_dict({} if default_data_path is None else {"data_path": default_data_path})
text = value.strip()
if not text.startswith(("{", "[")):
with open(os.path.expanduser(text), encoding="utf-8") as f:
diff --git a/eagle_harness/sglang/algorithm.py b/eagle_harness/sglang/algorithm.py
index 9b0d145..c1fb8b5 100644
--- a/eagle_harness/sglang/algorithm.py
+++ b/eagle_harness/sglang/algorithm.py
@@ -57,6 +57,8 @@ logger = logging.getLogger(__name__)
ALGORITHM_NAME = "EAGLE_HARNESS"
# The draft attention backends the harness plans tree attention for.
DRAFT_ATTENTION_BACKENDS = ("flashinfer", "fa3", "fa4")
+# --speculative-num-steps, --speculative-eagle-topk, --speculative-num-draft-tokens when none is set.
+DEFAULT_TREE_SHAPE = (12, 4, 16)
# sglang's ROCm switches the measured AMD recipe depends on (read by sglang at import, never set here; the harness sets
# only aiter's GEMM table, eagle_harness.core.gemm_solutions.serve_bf16_gemms_through_torch).
ROCM_ENVIRONMENT = {
@@ -192,6 +194,18 @@ class EagleHarnessSpecAlgo(CustomSpecAlgo):
# Every spawned worker must see the same resolved path, without an environment overlay.
server_args.speculative_algorithm_config = json.dumps(config_to_dict(config))
logger.info("EAGLE_HARNESS: selected packaged MI355X cost profile %s", default_data_path)
+ shape = (
+ server_args.speculative_num_steps,
+ server_args.speculative_eagle_topk,
+ server_args.speculative_num_draft_tokens,
+ )
+ if shape == (None, None, None) and config.intensity is not ComputeIntensity.CHAIN:
+ # The release tree (depth 12, top-k 4, verify width 16), so a launch needs only the draft path.
+ (
+ server_args.speculative_num_steps,
+ server_args.speculative_eagle_topk,
+ server_args.speculative_num_draft_tokens,
+ ) = DEFAULT_TREE_SHAPE
if server_args.speculative_num_steps is None or server_args.speculative_eagle_topk is None:
raise ValueError(
"EAGLE_HARNESS needs --speculative-num-steps, --speculative-eagle-topk and "
diff --git a/eagle_harness/sglang/drafter.py b/eagle_harness/sglang/drafter.py
index e04c285..e177bbb 100644
--- a/eagle_harness/sglang/drafter.py
+++ b/eagle_harness/sglang/drafter.py
@@ -621,4 +621,7 @@ class DynamicDrafter:
seq_lens_cpu: Optional[torch.Tensor] = getattr(batch, "seq_lens_cpu", None)
if seq_lens_cpu is not None:
return sum(seq_lens_cpu.tolist()) // len(seq_lens_cpu)
- return int(batch.seq_lens_sum) // len(batch.seq_lens)
+ if batch.seq_lens_sum is not None:
+ return int(batch.seq_lens_sum) // len(batch.seq_lens)
+ # The overlap scheduler drops both host mirrors; the requests still hold their lengths (lagging one round).
+ return sum(req.seqlen for req in batch.reqs) // len(batch.reqs)
diff --git a/tests/test_config.py b/tests/test_config.py
index 61a16be..873838c 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -70,9 +70,9 @@ def test_options_removed_in_0_1_1_are_refused_by_name(raw, key):
parse_harness_config(json.dumps({"data_path": "/x", **raw}))
-def test_rejects_missing_and_non_object():
- with pytest.raises(ValueError, match="needs --speculative-algorithm-config"):
- parse_harness_config(None)
+def test_absent_config_is_the_default_and_bad_json_is_rejected():
+ assert parse_harness_config(None) == parse_harness_config("{}") == HarnessConfig()
+ assert parse_harness_config(" ") == HarnessConfig()
with pytest.raises(ValueError, match="JSON object"):
parse_harness_config("[1, 2]")
with pytest.raises(ValueError, match="not valid JSON"):
diff --git a/tests/test_drafter.py b/tests/test_drafter.py
index 02df392..ab9b48b 100644
--- a/tests/test_drafter.py
+++ b/tests/test_drafter.py
@@ -422,3 +422,12 @@ def test_a_bucket_the_cost_model_refuses_keeps_the_fixed_shape(caplog):
with caplog.at_level("WARNING"):
drafter._check_priced(1) # the model has no table for bucket 1
assert drafter._unpriced == {1} and "keeps the fixed draft shape" in caplog.text
+
+
+def test_average_context_reads_request_lengths_when_the_overlap_scheduler_drops_host_mirrors():
+ """Under the overlap scheduler a decode batch has neither seq_lens_cpu nor seq_lens_sum (first live-cost round)."""
+ reqs = [SimpleNamespace(seqlen=10), SimpleNamespace(seqlen=21)]
+ overlap = SimpleNamespace(reqs=reqs, seq_lens=torch.tensor([10, 21]), seq_lens_cpu=None, seq_lens_sum=None)
+ assert DynamicDrafter.average_context(overlap) == 15
+ synchronous = SimpleNamespace(reqs=reqs, seq_lens=torch.tensor([10, 21]), seq_lens_cpu=torch.tensor([12, 22]))
+ assert DynamicDrafter.average_context(synchronous) == 17
diff --git a/tests/test_sglang_plugin.py b/tests/test_sglang_plugin.py
index d4822d0..27f4a56 100644
--- a/tests/test_sglang_plugin.py
+++ b/tests/test_sglang_plugin.py
@@ -159,12 +159,22 @@ def test_server_args_accept_eagle_harness_and_config(registered, overlap_off):
@pytest.mark.parametrize(
"overrides, error, message",
[
- ({"speculative_algorithm_config": None}, ValueError, "needs --speculative-algorithm-config"),
({"speculative_algorithm_config": '{"data_path": "/x", "bogus": 1}'}, ValueError, "unknown keys"),
({"speculative_eagle_topk": 9}, NotImplementedError, "topk"),
({"speculative_num_draft_tokens": 65}, NotImplementedError, "verify_num"),
({"max_running_requests": 513}, ValueError, "batch limit 512"),
({"speculative_num_steps": None, "speculative_eagle_topk": None}, ValueError, "num-steps"),
+ ({"speculative_num_draft_tokens": None}, ValueError, "num-draft-tokens"),
+ (
+ {
+ "speculative_algorithm_config": '{"intensity": "chain_intensity"}',
+ "speculative_num_steps": None,
+ "speculative_eagle_topk": None,
+ "speculative_num_draft_tokens": None,
+ },
+ ValueError,
+ "num-steps",
+ ),
({"speculative_token_map": "/x/map.pt"}, NotImplementedError, "token-map"),
],
)
@@ -173,6 +183,22 @@ def test_server_args_rejections(registered, overrides, error, message):
handle_speculative_decoding(_server_args(**overrides))
+def test_draft_path_alone_gets_the_release_tree_and_live_costs(registered):
+ """With no config and no tree flags the launch serves the release recipe's tree with live costs."""
+ args = _server_args(
+ speculative_algorithm_config=None,
+ speculative_num_steps=None,
+ speculative_eagle_topk=None,
+ speculative_num_draft_tokens=None,
+ )
+ handle_speculative_decoding(args)
+ assert [
+ resolution_result(args, name)
+ for name in ("speculative_num_steps", "speculative_eagle_topk", "speculative_num_draft_tokens")
+ ] == [12, 4, 16]
+ assert parse_harness_config(args.speculative_algorithm_config).data_path is None
+
+
@pytest.mark.parametrize(
"vendor,backend,aiter_env,warns",
[