Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[Bugfix] Fix Sglang e2e UT issue (#2250)

Liang Lvd77c9eaa616b
diff --git a/test/e2e/test_cuda/test_sglang_throughput.py b/test/e2e/test_cuda/test_sglang_throughput.py
index aaf4d069f..3ed58a3ac 100644
--- a/test/e2e/test_cuda/test_sglang_throughput.py
+++ b/test/e2e/test_cuda/test_sglang_throughput.py
@@ -181,7 +181,16 @@ def _run_sglang_benchmark(
outputs = llm.generate(prompts, sampling_params)
total_time = time.perf_counter() - t0
- n_out_tokens = sum(len(o.get("meta_info", {}).get("output_ids", []) or []) for o in outputs)
+ # SGLang 0.5.x puts ``output_ids`` at the TOP level of each output dict,
+ # not inside ``meta_info``. ``completion_tokens`` in ``meta_info`` is
+ # always present, so fall back to it for robustness across versions.
+ def _count_output_tokens(o):
+ ids = o.get("output_ids") or (o.get("meta_info", {}) or {}).get("output_ids")
+ if ids:
+ return len(ids)
+ return (o.get("meta_info", {}) or {}).get("completion_tokens", 0) or 0
+
+ n_out_tokens = sum(_count_output_tokens(o) for o in outputs)
gen_tokens_per_s = n_out_tokens / max(total_time, 1e-6)
# SGLang's per-request meta_info exposes per-request decode throughput
@@ -376,10 +385,20 @@ def test_sglang_awq_format_via_cli(require_cuda):
rc = os.system(cmd)
assert rc == 0, f"awq-format quant via CLI failed (rc={rc})"
+ # The CLI derives a descriptive export sub-directory under
+ # ``output_dir`` (e.g. ``Qwen3-0.6B-w4g128/`` via ``_get_export_dir``),
+ # so the engine must load from that folder rather than ``out`` itself.
+ saved = [
+ os.path.join(out, d)
+ for d in os.listdir(out)
+ if os.path.isdir(os.path.join(out, d)) and os.path.isfile(os.path.join(out, d, "config.json"))
+ ]
+ assert len(saved) == 1, f"expected one quantized model dir under {out}, got {saved}"
+
# SGLang will JIT-compile the awq kernels during the first generate
# call, so we expect the first request to be slow but the second
# to be representative.
- llm = _build_sglang_engine(out, mem_fraction_static=0.5, context_len=1024)
+ llm = _build_sglang_engine(saved[0], mem_fraction_static=0.5, context_len=1024)
try:
outputs = llm.generate(["Hello, my name is"], {"max_new_tokens": 16, "temperature": 0.0})
text = outputs[0]["text"]