PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""End-to-end throughput & latency tests for the **SGLang** inference engine.
These tests mirror :mod:`test_vllm_throughput` but drive the SGLang
``Engine`` instead. SGLang has slightly different defaults
(``mem_fraction_static`` instead of ``gpu_memory_utilization``) and
exposes its own ``token_usage`` and ``decode throughput`` fields, which
we record alongside the wall-clock measurement.
The test pipeline is identical:
1. ``auto-round`` Python API → quantized checkpoint.
2. ``sgl.Engine`` loads the checkpoint.
3. Warm up, then run a fixed prompt batch.
4. Record output tokens/s, decode-only tokens/s and TTFT.
Run a single test locally::
pytest test/e2e/test_cuda/test_sglang_throughput.py::TestSglangThroughput::test_quantize_and_serve \\
--e2e-model-preset=default -v -s
"""
import json
import os
import time
from test.e2e.test_cuda.conftest import ( # noqa: E402
BenchResult,
free_cuda,
make_bench_prompts,
quantize_and_save,
)
import pytest
import torch
# ---------------------------------------------------------------------------
# Output sink
# ---------------------------------------------------------------------------
_THIS_DIR = os.path.dirname(os.path.abspath(__file__))
_OUTPUT_DIR = os.path.join(_THIS_DIR, "..", "..", "output")
_OUTPUT_FILE = os.path.normpath(os.path.join(_OUTPUT_DIR, "sglang_throughput.jsonl"))
def _record(result: BenchResult) -> None:
"""Append a benchmark result to a JSONL file for trend tracking."""
os.makedirs(_OUTPUT_DIR, exist_ok=True)
with open(_OUTPUT_FILE, "a", encoding="utf-8") as f:
f.write(json.dumps(result.__dict__) + "\n")
def _server_arg_fields():
try:
import dataclasses
from sglang.srt.server_args import ServerArgs
return {f.name for f in dataclasses.fields(ServerArgs)}
except Exception:
return set()
# ---------------------------------------------------------------------------
# Skip markers
# ---------------------------------------------------------------------------
# sglang 0.2+ has a known multiprocessing.resource_tracker bug on Linux that
# manifests as ``[Errno 10] No child processes`` during teardown. We patch
# ``ResourceTracker._stop`` to swallow that exception, mirroring the
# workaround in ``test/integration/test_cuda/test_sglang.py``.
import gc # noqa: E402
import multiprocessing.resource_tracker # noqa: E402
_orig_stop = multiprocessing.resource_tracker.ResourceTracker._stop
def _patched_stop(self, *args, _orig=_orig_stop, **kwargs):
if _orig is not None:
try:
_orig(self, *args, **kwargs)
except ChildProcessError:
pass
multiprocessing.resource_tracker.ResourceTracker._stop = _patched_stop
def _shutdown_sglang_engine(llm) -> None:
"""Shut down an in-process sglang engine and reset torch inductor's compile pool.
``sgl.Engine.shutdown()`` calls ``kill_process_tree(os.getpid(), include_parent=False)``,
which also kills inductor's compile-worker sidecar spawned by the previous
``torch.compile``-enabled quantization. The dead pool stays cached in
``AsyncCompile.process_pool``, so the next quantization that needs a fresh
compile fails with ``RuntimeError: Attempting to use a closed pool``.
Dropping the cached pool lets inductor spawn a new one on demand.
"""
shutdown = getattr(llm, "shutdown", None)
if callable(shutdown):
try:
shutdown()
except Exception:
pass
try:
from torch._inductor.async_compile import shutdown_compile_workers
shutdown_compile_workers()
except Exception:
pass
pytestmark = [
pytest.mark.e2e,
pytest.mark.skipif(
not torch.cuda.is_available(),
reason="SGLang throughput tests require a CUDA GPU",
),
pytest.mark.enable_torch_compile,
]
# ---------------------------------------------------------------------------
# SGLang wrapper
# ---------------------------------------------------------------------------
def _build_sglang_engine(model_path: str, mem_fraction_static: float, context_len: int):
"""Construct a sglang engine configured for AutoRound-quantized checkpoints.
``disable_piecewise_cuda_graph=True`` and a small ``cuda_graph_bs`` list
avoid the gptq_marlin_repack JIT kernel tripping over Blackwell SM 12.x
when CUDA < 12.9 (the same constraint as the vLLM tests).
"""
try:
import sglang as sgl
except ImportError as e:
pytest.skip(f"sglang is not installed: {e}")
fields = _server_arg_fields()
engine_kwargs = {}
if "disable_piecewise_cuda_graph" in fields:
engine_kwargs["disable_piecewise_cuda_graph"] = True
if "cuda_graph_bs" in fields:
engine_kwargs["cuda_graph_bs"] = [1]
elif "disable_cuda_graph" in fields:
engine_kwargs["disable_cuda_graph"] = True
if "cuda_graph_bs_decode" in fields:
engine_kwargs["cuda_graph_bs_decode"] = [1]
return sgl.Engine(
model_path=model_path,
mem_fraction_static=mem_fraction_static,
context_length=context_len,
# Keep cuda-graphs conservative – AutoRound-int4 checkpoints don't
# benefit from large captured graphs on small workloads.
**engine_kwargs,
)
def _run_sglang_benchmark(
model_path: str,
max_new_tokens: int = 128,
num_prompts: int = 8,
mem_fraction_static: float = 0.7,
context_len: int = 2048,
warmup: int = 1,
) -> BenchResult:
"""End-to-end benchmark: load with SGLang, warm up, then time a prompt batch."""
from test.helpers import get_model_path
model_path = get_model_path(model_path) if "/" in model_path else model_path
llm = _build_sglang_engine(model_path, mem_fraction_static=mem_fraction_static, context_len=context_len)
try:
# SGLang's tokenizer is the HuggingFace tokenizer.
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
prompts = make_bench_prompts(tokenizer, num_prompts=num_prompts, target_input_tokens=64)
sampling_params = {
"temperature": 0.0, # greedy ⇒ deterministic output length
"top_p": 1.0,
"max_new_tokens": max_new_tokens,
}
# --- warm-up ---
for _ in range(max(0, warmup)):
llm.generate(prompts[:1], sampling_params)
# --- timed run ---
t0 = time.perf_counter()
outputs = llm.generate(prompts, sampling_params)
total_time = time.perf_counter() - t0
# SGLang 0.5.x puts ``output_ids`` at the TOP level of each output dict,
# not inside ``meta_info``. ``completion_tokens`` in ``meta_info`` is
# always present, so fall back to it for robustness across versions.
def _count_output_tokens(o):
ids = o.get("output_ids") or (o.get("meta_info", {}) or {}).get("output_ids")
if ids:
return len(ids)
return (o.get("meta_info", {}) or {}).get("completion_tokens", 0) or 0
n_out_tokens = sum(_count_output_tokens(o) for o in outputs)
gen_tokens_per_s = n_out_tokens / max(total_time, 1e-6)
# SGLang's per-request meta_info exposes per-request decode throughput
# and TTFT; we average over the batch for a single number.
decode_tps = []
ttfts = []
for o in outputs:
info = o.get("meta_info", {}) or {}
# ``completion_tokens`` and ``e2e_latency`` are always present.
comp = info.get("completion_tokens")
lat = info.get("e2e_latency")
if comp and lat and lat > 0:
decode_tps.append(comp / lat)
# TTFT is reported in ``prefill_latency`` for batch_size=1.
ttft = info.get("prefill_latency")
if ttft and ttft > 0:
ttfts.append(ttft)
gen_tokens_per_s_avg = sum(decode_tps) / len(decode_tps) if decode_tps else None
ttft_s = sum(ttfts) / len(ttfts) if ttfts else None
return BenchResult(
engine="sglang",
model=os.path.basename(model_path.rstrip("/")),
fmt=os.environ.get("_AR_E2E_FMT", "auto_round"),
bits=int(os.environ.get("_AR_E2E_BITS", "4")),
group_size=int(os.environ.get("_AR_E2E_GS", "128")),
num_prompts=num_prompts,
max_new_tokens=max_new_tokens,
total_time_s=total_time,
output_tokens_per_s=n_out_tokens / max(total_time, 1e-6),
gen_tokens_per_s=gen_tokens_per_s_avg,
ttft_s=ttft_s,
sample_output=outputs[0]["text"],
)
finally:
_shutdown_sglang_engine(llm)
del llm
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
torch.cuda.synchronize()
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestSglangThroughput:
"""Quantize-with-autoround + serve-with-sglang end-to-end suite."""
@pytest.mark.parametrize(
"model_case",
[
pytest.param(
__import__("test.e2e.test_cuda.conftest", fromlist=["ModelCase"]).ModelCase(
"Qwen/Qwen3-1.7B", 4, 128, True, "auto_round", min_gpu_gib=10
),
id="qwen3-1.7b-w4a16-auto_round",
),
pytest.param(
__import__("test.e2e.test_cuda.conftest", fromlist=["ModelCase"]).ModelCase(
"Qwen/Qwen3-1.7B", 4, 128, True, "auto_gptq", min_gpu_gib=10
),
id="qwen3-1.7b-w4a16-auto_gptq",
),
pytest.param(
__import__("test.e2e.test_cuda.conftest", fromlist=["ModelCase"]).ModelCase(
"Qwen/Qwen3-1.7B", 4, 128, True, "auto_awq", min_gpu_gib=10
),
id="qwen3-1.7b-w4a16-auto_awq",
),
],
)
def test_quantize_and_serve(self, model_case, tmp_path, require_gpu_memory):
save_dir = str(tmp_path / f"saved_{model_case.fmt}_w{model_case.bits}")
os.environ["_AR_E2E_FMT"] = model_case.fmt
os.environ["_AR_E2E_BITS"] = str(model_case.bits)
os.environ["_AR_E2E_GS"] = str(model_case.group_size)
saved = quantize_and_save(
model_id=model_case.hf_id,
bits=model_case.bits,
group_size=model_case.group_size,
sym=model_case.sym,
fmt=model_case.fmt,
output_dir=save_dir,
iters=200,
nsamples=128,
seqlen=2048,
)
result = _run_sglang_benchmark(saved, max_new_tokens=64, num_prompts=4, mem_fraction_static=0.7)
_record(result)
assert result.sample_output.strip(), "SGLang produced empty output"
assert "!!!" not in result.sample_output, "SGLang produced garbage output"
assert result.output_tokens_per_s > 0
assert (
result.output_tokens_per_s >= 1.0
), f"SGLang throughput suspiciously low: {result.output_tokens_per_s:.2f} tok/s"
print(
f"\n[SGLang] {model_case.hf_id} {model_case.fmt} w{model_case.bits} "
f"-> {result.output_tokens_per_s:.1f} tok/s "
f"(decode-only: {result.gen_tokens_per_s}, ttft: {result.ttft_s})"
)
class TestSglangLarge:
"""Heavier cases that need >=24 GiB; skipped on smaller GPUs."""
@pytest.mark.parametrize(
"model_case",
[
pytest.param(
__import__("test.e2e.test_cuda.conftest", fromlist=["ModelCase"]).ModelCase(
"Qwen/Qwen2.5-7B-Instruct", 4, 128, True, "auto_round", min_gpu_gib=18
),
id="qwen2.5-7b-w4a16-auto_round",
),
pytest.param(
__import__("test.e2e.test_cuda.conftest", fromlist=["ModelCase"]).ModelCase(
"meta-llama/Llama-3.2-3B-Instruct", 4, 128, True, "auto_round", min_gpu_gib=12
),
id="llama-3.2-3b-w4a16-auto_round",
),
],
)
def test_quantize_and_serve(self, model_case, tmp_path, require_gpu_memory):
save_dir = str(tmp_path / f"saved_{model_case.fmt}_w{model_case.bits}")
os.environ["_AR_E2E_FMT"] = model_case.fmt
os.environ["_AR_E2E_BITS"] = str(model_case.bits)
os.environ["_AR_E2E_GS"] = str(model_case.group_size)
saved = quantize_and_save(
model_id=model_case.hf_id,
bits=model_case.bits,
group_size=model_case.group_size,
sym=model_case.sym,
fmt=model_case.fmt,
output_dir=save_dir,
iters=200,
nsamples=128,
seqlen=2048,
)
result = _run_sglang_benchmark(saved, max_new_tokens=64, num_prompts=4, mem_fraction_static=0.75)
_record(result)
assert result.sample_output.strip()
assert "!!!" not in result.sample_output
assert result.output_tokens_per_s >= 1.0
print(
f"\n[SGLang-large] {model_case.hf_id} {model_case.fmt} w{model_case.bits} "
f"-> {result.output_tokens_per_s:.1f} tok/s"
)
# ---------------------------------------------------------------------------
# CLI / mixed-format regression
# ---------------------------------------------------------------------------
def test_sglang_awq_format_via_cli(require_cuda):
"""``auto-round --format auto_round:auto_awq`` → SGLang load.
This mirrors the existing ``test_ar_format_sglang`` in the
integration suite, but is parameterised over a real model on GPU so
that the same code path can be re-checked in the e2e pipeline.
"""
import sys
import tempfile
from test.helpers import get_model_path
model = get_model_path("Qwen/Qwen3-0.6B")
with tempfile.TemporaryDirectory() as out:
cmd = (
f"{sys.executable} -m auto_round --model {model} "
f"--scheme W4A16 --iters 0 --disable_opt_rtn --format auto_round:auto_awq "
f"--output_dir {out}"
)
rc = os.system(cmd)
assert rc == 0, f"awq-format quant via CLI failed (rc={rc})"
# The CLI derives a descriptive export sub-directory under
# ``output_dir`` (e.g. ``Qwen3-0.6B-w4g128/`` via ``_get_export_dir``),
# so the engine must load from that folder rather than ``out`` itself.
saved = [
os.path.join(out, d)
for d in os.listdir(out)
if os.path.isdir(os.path.join(out, d)) and os.path.isfile(os.path.join(out, d, "config.json"))
]
assert len(saved) == 1, f"expected one quantized model dir under {out}, got {saved}"
# SGLang will JIT-compile the awq kernels during the first generate
# call, so we expect the first request to be slow but the second
# to be representative.
llm = _build_sglang_engine(saved[0], mem_fraction_static=0.5, context_len=1024)
try:
outputs = llm.generate(["Hello, my name is"], {"max_new_tokens": 16, "temperature": 0.0})
text = outputs[0]["text"]
assert text.strip() and "!!!" not in text
finally:
_shutdown_sglang_engine(llm)
del llm
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()