Official Inference Code of EAGLE-4
feat(sglang): serve with only the draft path by default
Seyfal Sultanov733fa938dbab
diff --git a/docs/configuration.md b/docs/configuration.mdindex 3721946..80a55cf 100644--- a/docs/configuration.md+++ b/docs/configuration.md@@ -1,8 +1,8 @@# Configuration referenceThe method takes one JSON object. On sglang it goes in `--speculative-algorithm-config`, inline or as a path to a-JSON file. On vLLM it goes in the `method_config` of `--speculative-config`. On TokenSpeed it goes in-`--speculative-algorithm-config` (inline JSON). The adapters share the core configuration parser and add+JSON file, and leaving the flag out means `{}`. On vLLM it goes in the `method_config` of `--speculative-config`.+On TokenSpeed it goes in `--speculative-algorithm-config` (inline JSON). The adapters share the core configuration parser and addengine-specific keys and restrictions described below. Unknown keys arerejected with the list of valid keys, so a typo cannot fall back to a default silently.diff --git a/docs/troubleshooting.md b/docs/troubleshooting.mdindex 58e4b44..165062d 100644--- a/docs/troubleshooting.md+++ b/docs/troubleshooting.md@@ -44,12 +44,11 @@ no supported host installed (one of: sglang, vllm, tokenspeed)| message | cause | fix ||---|---|---|| `Unknown speculative algorithm name: EAGLE_HARNESS` (sglang), vLLM rejects the method `eagle_harness`, or TokenSpeed does not know the algorithm `EAGLE_HARNESS` | the plugin did not register | run `eagle-harness-check` and read the error the engine logged at startup |-| `EAGLE_HARNESS needs --speculative-algorithm-config (JSON or a path to JSON), ...` | sglang got no config | pass `{}` (live costs), `{"data_path": "<calibration dir>"}` or `{"mode": "static"}` || `Cost tables under ... were measured on another setup (stored vs this server): {...}` | `data_path` tables keyed for another GPU, TP size, model pair, engine build or dtype | calibrate for this setup, or drop `data_path` to measure the costs on the served rounds || `TokenSpeed serve mode (the default) needs data_path or a complete explicit chain schedule ...` | serve mode, set explicitly or by default, with no cost tables and no complete chain schedule (TokenSpeed) | set `data_path` to matching TokenSpeed costs ([Calibration](calibration.md)); static mode is not a correctness-qualified substitute || `config: unknown keys ['intensty']; valid keys are [...]` | a misspelt key | use a key from the list ([Configuration](configuration.md)) || `mode: 'fast' is not one of static, serve, eval, calibrate.` | a bad value | use one of the listed values |-| `EAGLE_HARNESS needs --speculative-num-steps, --speculative-eagle-topk and --speculative-num-draft-tokens ...` | sglang got no tree shape | pass all three |+| `EAGLE_HARNESS needs --speculative-num-steps, --speculative-eagle-topk and --speculative-num-draft-tokens ...` | sglang got part of the tree shape, or a chain config without one | pass all three, or none of them for the default tree (depth 12, top-k 4, verify width 16) || `max_running_requests=... exceeds the tree draft batch limit 512.` | the explicit request cap exceeds the supported range | set `--max-running-requests` to at most 512, and check that KV and recurrent-state memory can hold the requested batch || `eagle-harness kernels are off: ... is older than NVIDIA sm_80 (A100).` | the GPU is outside the kernel tiers | use a supported GPU ([Kernels](kernels.md)) || `page_size > 1 needs --speculative-draft-attention-backend flashinfer, fa3 or fa4 for EAGLE_HARNESS.` | the draft backend cannot serve paged KV | set one of those backends |diff --git a/eagle_harness/core/config.py b/eagle_harness/core/config.pyindex 07930d4..b819847 100644--- a/eagle_harness/core/config.py+++ b/eagle_harness/core/config.py@@ -242,14 +242,12 @@ def harness_config_from_dict(data: Mapping[str, Any]) -> HarnessConfig:def parse_harness_config(value: Optional[str], *, default_data_path: Optional[str] = None) -> HarnessConfig:- """Parse JSON or a JSON file, filling an absent data_path from a device-selected profile."""+ """Parse JSON or a JSON file, filling an absent data_path from a device-selected profile.++ No value is the default config (serve mode, costs measured on the served rounds).+ """if value is None or not value.strip():- if default_data_path is not None:- return harness_config_from_dict({"data_path": default_data_path})- raise ValueError(- "EAGLE_HARNESS needs --speculative-algorithm-config (JSON or a path to JSON), "- 'e.g. {"data_path": "<calibration dir>"} or {"mode": "static"}.'- )+ return harness_config_from_dict({} if default_data_path is None else {"data_path": default_data_path})text = value.strip()if not text.startswith(("{", "[")):with open(os.path.expanduser(text), encoding="utf-8") as f:diff --git a/eagle_harness/sglang/algorithm.py b/eagle_harness/sglang/algorithm.pyindex 9b0d145..c1fb8b5 100644--- a/eagle_harness/sglang/algorithm.py+++ b/eagle_harness/sglang/algorithm.py@@ -57,6 +57,8 @@ logger = logging.getLogger(__name__)ALGORITHM_NAME = "EAGLE_HARNESS"# The draft attention backends the harness plans tree attention for.DRAFT_ATTENTION_BACKENDS = ("flashinfer", "fa3", "fa4")+# --speculative-num-steps, --speculative-eagle-topk, --speculative-num-draft-tokens when none is set.+DEFAULT_TREE_SHAPE = (12, 4, 16)# sglang's ROCm switches the measured AMD recipe depends on (read by sglang at import, never set here; the harness sets# only aiter's GEMM table, eagle_harness.core.gemm_solutions.serve_bf16_gemms_through_torch).ROCM_ENVIRONMENT = {@@ -192,6 +194,18 @@ class EagleHarnessSpecAlgo(CustomSpecAlgo):# Every spawned worker must see the same resolved path, without an environment overlay.server_args.speculative_algorithm_config = json.dumps(config_to_dict(config))logger.info("EAGLE_HARNESS: selected packaged MI355X cost profile %s", default_data_path)+ shape = (+ server_args.speculative_num_steps,+ server_args.speculative_eagle_topk,+ server_args.speculative_num_draft_tokens,+ )+ if shape == (None, None, None) and config.intensity is not ComputeIntensity.CHAIN:+ # The release tree (depth 12, top-k 4, verify width 16), so a launch needs only the draft path.+ (+ server_args.speculative_num_steps,+ server_args.speculative_eagle_topk,+ server_args.speculative_num_draft_tokens,+ ) = DEFAULT_TREE_SHAPEif server_args.speculative_num_steps is None or server_args.speculative_eagle_topk is None:raise ValueError("EAGLE_HARNESS needs --speculative-num-steps, --speculative-eagle-topk and "diff --git a/eagle_harness/sglang/drafter.py b/eagle_harness/sglang/drafter.pyindex e04c285..e177bbb 100644--- a/eagle_harness/sglang/drafter.py+++ b/eagle_harness/sglang/drafter.py@@ -621,4 +621,7 @@ class DynamicDrafter:seq_lens_cpu: Optional[torch.Tensor] = getattr(batch, "seq_lens_cpu", None)if seq_lens_cpu is not None:return sum(seq_lens_cpu.tolist()) // len(seq_lens_cpu)- return int(batch.seq_lens_sum) // len(batch.seq_lens)+ if batch.seq_lens_sum is not None:+ return int(batch.seq_lens_sum) // len(batch.seq_lens)+ # The overlap scheduler drops both host mirrors; the requests still hold their lengths (lagging one round).+ return sum(req.seqlen for req in batch.reqs) // len(batch.reqs)diff --git a/tests/test_config.py b/tests/test_config.pyindex 61a16be..873838c 100644--- a/tests/test_config.py+++ b/tests/test_config.py@@ -70,9 +70,9 @@ def test_options_removed_in_0_1_1_are_refused_by_name(raw, key):parse_harness_config(json.dumps({"data_path": "/x", **raw}))-def test_rejects_missing_and_non_object():- with pytest.raises(ValueError, match="needs --speculative-algorithm-config"):- parse_harness_config(None)+def test_absent_config_is_the_default_and_bad_json_is_rejected():+ assert parse_harness_config(None) == parse_harness_config("{}") == HarnessConfig()+ assert parse_harness_config(" ") == HarnessConfig()with pytest.raises(ValueError, match="JSON object"):parse_harness_config("[1, 2]")with pytest.raises(ValueError, match="not valid JSON"):diff --git a/tests/test_drafter.py b/tests/test_drafter.pyindex 02df392..ab9b48b 100644--- a/tests/test_drafter.py+++ b/tests/test_drafter.py@@ -422,3 +422,12 @@ def test_a_bucket_the_cost_model_refuses_keeps_the_fixed_shape(caplog):with caplog.at_level("WARNING"):drafter._check_priced(1) # the model has no table for bucket 1assert drafter._unpriced == {1} and "keeps the fixed draft shape" in caplog.text+++def test_average_context_reads_request_lengths_when_the_overlap_scheduler_drops_host_mirrors():+ """Under the overlap scheduler a decode batch has neither seq_lens_cpu nor seq_lens_sum (first live-cost round)."""+ reqs = [SimpleNamespace(seqlen=10), SimpleNamespace(seqlen=21)]+ overlap = SimpleNamespace(reqs=reqs, seq_lens=torch.tensor([10, 21]), seq_lens_cpu=None, seq_lens_sum=None)+ assert DynamicDrafter.average_context(overlap) == 15+ synchronous = SimpleNamespace(reqs=reqs, seq_lens=torch.tensor([10, 21]), seq_lens_cpu=torch.tensor([12, 22]))+ assert DynamicDrafter.average_context(synchronous) == 17diff --git a/tests/test_sglang_plugin.py b/tests/test_sglang_plugin.pyindex d4822d0..27f4a56 100644--- a/tests/test_sglang_plugin.py+++ b/tests/test_sglang_plugin.py@@ -159,12 +159,22 @@ def test_server_args_accept_eagle_harness_and_config(registered, overlap_off):@pytest.mark.parametrize("overrides, error, message",[- ({"speculative_algorithm_config": None}, ValueError, "needs --speculative-algorithm-config"),({"speculative_algorithm_config": '{"data_path": "/x", "bogus": 1}'}, ValueError, "unknown keys"),({"speculative_eagle_topk": 9}, NotImplementedError, "topk"),({"speculative_num_draft_tokens": 65}, NotImplementedError, "verify_num"),({"max_running_requests": 513}, ValueError, "batch limit 512"),({"speculative_num_steps": None, "speculative_eagle_topk": None}, ValueError, "num-steps"),+ ({"speculative_num_draft_tokens": None}, ValueError, "num-draft-tokens"),+ (+ {+ "speculative_algorithm_config": '{"intensity": "chain_intensity"}',+ "speculative_num_steps": None,+ "speculative_eagle_topk": None,+ "speculative_num_draft_tokens": None,+ },+ ValueError,+ "num-steps",+ ),({"speculative_token_map": "/x/map.pt"}, NotImplementedError, "token-map"),],)@@ -173,6 +183,22 @@ def test_server_args_rejections(registered, overrides, error, message):handle_speculative_decoding(_server_args(**overrides))+def test_draft_path_alone_gets_the_release_tree_and_live_costs(registered):+ """With no config and no tree flags the launch serves the release recipe's tree with live costs."""+ args = _server_args(+ speculative_algorithm_config=None,+ speculative_num_steps=None,+ speculative_eagle_topk=None,+ speculative_num_draft_tokens=None,+ )+ handle_speculative_decoding(args)+ assert [+ resolution_result(args, name)+ for name in ("speculative_num_steps", "speculative_eagle_topk", "speculative_num_draft_tokens")+ ] == [12, 4, 16]+ assert parse_harness_config(args.speculative_algorithm_config).data_path is None++@pytest.mark.parametrize("vendor,backend,aiter_env,warns",[