PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[Bugfix] Fix Sglang e2e UT issue (#2250)
Liang Lvd77c9eaa616b
diff --git a/test/e2e/test_cuda/test_sglang_throughput.py b/test/e2e/test_cuda/test_sglang_throughput.pyindex aaf4d069f..3ed58a3ac 100644--- a/test/e2e/test_cuda/test_sglang_throughput.py+++ b/test/e2e/test_cuda/test_sglang_throughput.py@@ -181,7 +181,16 @@ def _run_sglang_benchmark(outputs = llm.generate(prompts, sampling_params)total_time = time.perf_counter() - t0- n_out_tokens = sum(len(o.get("meta_info", {}).get("output_ids", []) or []) for o in outputs)+ # SGLang 0.5.x puts ``output_ids`` at the TOP level of each output dict,+ # not inside ``meta_info``. ``completion_tokens`` in ``meta_info`` is+ # always present, so fall back to it for robustness across versions.+ def _count_output_tokens(o):+ ids = o.get("output_ids") or (o.get("meta_info", {}) or {}).get("output_ids")+ if ids:+ return len(ids)+ return (o.get("meta_info", {}) or {}).get("completion_tokens", 0) or 0++ n_out_tokens = sum(_count_output_tokens(o) for o in outputs)gen_tokens_per_s = n_out_tokens / max(total_time, 1e-6)# SGLang's per-request meta_info exposes per-request decode throughput@@ -376,10 +385,20 @@ def test_sglang_awq_format_via_cli(require_cuda):rc = os.system(cmd)assert rc == 0, f"awq-format quant via CLI failed (rc={rc})"+ # The CLI derives a descriptive export sub-directory under+ # ``output_dir`` (e.g. ``Qwen3-0.6B-w4g128/`` via ``_get_export_dir``),+ # so the engine must load from that folder rather than ``out`` itself.+ saved = [+ os.path.join(out, d)+ for d in os.listdir(out)+ if os.path.isdir(os.path.join(out, d)) and os.path.isfile(os.path.join(out, d, "config.json"))+ ]+ assert len(saved) == 1, f"expected one quantized model dir under {out}, got {saved}"+# SGLang will JIT-compile the awq kernels during the first generate# call, so we expect the first request to be slow but the second# to be representative.- llm = _build_sglang_engine(out, mem_fraction_static=0.5, context_len=1024)+ llm = _build_sglang_engine(saved[0], mem_fraction_static=0.5, context_len=1024)try:outputs = llm.generate(["Hello, my name is"], {"max_new_tokens": 16, "temperature": 0.0})text = outputs[0]["text"]