Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

13.4 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""End-to-end throughput & latency tests for the **vLLM** inference engine.

Each test in this file exercises the full pipeline that a real user
would follow:

    1. Quantize a real LLM with ``auto-round`` (Python API).
    2. Save the checkpoint in a deployment-ready format
       (``auto_round``, ``auto_gptq``, ``auto_awq``, ``gguf``, ...).
    3. Load the checkpoint with the official ``vllm.LLM`` engine.
    4. Run a fixed prompt batch and measure wall-clock throughput
       (output tokens/s) and decode-only throughput.

The numbers are recorded in a JSON line at the end of each test under
``test/e2e/output/vllm_throughput.jsonl`` so weekly CI runs can be
diffed over time.

These tests are slow (a single 1.7B W4A16 case takes ~5 min on an
A100) and are expected to be scheduled weekly, not on every PR.

Run a single test locally::

    pytest test/e2e/test_cuda/test_vllm_throughput.py::TestVllmThroughput::test_quantize_and_serve \\
           --e2e-model-preset=default -v -s

Run the full (large) matrix::

    pytest test/e2e/test_cuda/test_vllm_throughput.py \\
           --e2e-model-preset=all -v -s
"""

import json
import os
import subprocess
import time
from test.e2e.test_cuda.conftest import (  # noqa: E402
    BenchResult,
    free_cuda,
    make_bench_prompts,
    quantize_and_save,
)

import pytest
import torch

# ---------------------------------------------------------------------------
# Output sink
# ---------------------------------------------------------------------------

_THIS_DIR = os.path.dirname(os.path.abspath(__file__))
_OUTPUT_DIR = os.path.join(_THIS_DIR, "..", "..", "output")
_OUTPUT_FILE = os.path.normpath(os.path.join(_OUTPUT_DIR, "vllm_throughput.jsonl"))


def _record(result: BenchResult) -> None:
    """Append a benchmark result to a JSONL file for trend tracking."""
    os.makedirs(_OUTPUT_DIR, exist_ok=True)
    with open(_OUTPUT_FILE, "a", encoding="utf-8") as f:
        f.write(json.dumps(result.__dict__) + "\n")


# ---------------------------------------------------------------------------
# Skip markers
# ---------------------------------------------------------------------------

pytestmark = [
    pytest.mark.e2e,
    pytest.mark.skipif(
        not torch.cuda.is_available(),
        reason="vLLM throughput tests require a CUDA GPU",
    ),
    pytest.mark.enable_torch_compile,
    # The "low" preset fits on a 24 GiB card; the "large" preset needs ~40+ GiB.
]


# ---------------------------------------------------------------------------
# vLLM-engine wrapper
# ---------------------------------------------------------------------------


def _build_vllm_engine(model_path: str, max_model_len: int, gpu_mem_util: float):
    """Construct a vLLM engine configured for AutoRound-quantized checkpoints."""
    try:
        from vllm import LLM
        from vllm.platforms import current_platform
    except ImportError as e:
        pytest.skip(f"vllm is not installed: {e}")

    if not (current_platform.is_cpu() or current_platform.is_xpu() or current_platform.is_cuda()):
        pytest.skip("vLLM tests only run on CPU/XPU/CUDA")

    # Let vLLM infer quantization backend from the exported checkpoint config.
    # Newer vLLM versions do not accept legacy aliases like "auto-round".
    return LLM(
        model=model_path,
        trust_remote_code=True,
        tensor_parallel_size=1,
        gpu_memory_utilization=gpu_mem_util,
        max_model_len=max_model_len,
        dtype="auto",
        allow_deprecated_quantization=True,
        enforce_eager=False,
    )


def _run_vllm_benchmark(
    model_path: str,
    max_new_tokens: int = 128,
    num_prompts: int = 8,
    gpu_mem_util: float = 0.85,
    max_model_len: int = 2048,
    warmup: int = 1,
) -> BenchResult:
    """End-to-end benchmark: load with vLLM, warm up, then time a prompt batch."""
    try:
        from vllm import SamplingParams
    except ImportError as e:
        pytest.skip(f"vllm is not installed: {e}")

    from test.helpers import get_model_path

    model_path = get_model_path(model_path) if "/" in model_path else model_path

    llm = _build_vllm_engine(model_path, max_model_len=max_model_len, gpu_mem_util=gpu_mem_util)
    try:
        tokenizer = llm.get_tokenizer()

        prompts = make_bench_prompts(tokenizer, num_prompts=num_prompts, target_input_tokens=64)
        sampling = SamplingParams(
            temperature=0.0,  # greedy ⇒ deterministic output length
            top_p=1.0,
            max_tokens=max_new_tokens,
        )

        # --- warm-up ---
        for _ in range(max(0, warmup)):
            llm.generate(prompts[:1], sampling)

        # --- timed run ---
        # Some vLLM versions populate metrics.* after generate; we use both
        # wall-clock time and vLLM's own counters for a robust measurement.
        t0 = time.perf_counter()
        outputs = llm.generate(prompts, sampling)
        total_time = time.perf_counter() - t0

        # Aggregate output token counts.
        n_out_tokens = sum(len(o.outputs[0].token_ids) for o in outputs)
        gen_tokens_per_s = n_out_tokens / max(total_time, 1e-6)

        # Try to pull vLLM's own decode-time stats for a second opinion.
        gen_tokens_per_s_vllm: float | None = None
        ttft_s: float | None = None
        try:
            metrics = llm.aggregate_metrics()  # vllm >= 0.6
            stats = getattr(metrics, "stats", None) or {}
            # vLLM reports ``prompt_tokens`` and ``generation_tokens``.
            gen_tok = float(stats.get("generation_tokens", 0.0))
            gen_time = float(stats.get("gen_time", 0.0))  # seconds, decode only
            prompt_tok = float(stats.get("prompt_tokens", 0.0))
            prompt_time = float(stats.get("prompt_time", 0.0))
            if gen_time > 0 and gen_tok > 0:
                gen_tokens_per_s_vllm = gen_tok / gen_time
            if prompt_time > 0 and prompt_tok > 0:
                ttft_s = prompt_time / max(1, len(prompts))
        except Exception:
            pass

        return BenchResult(
            engine="vllm",
            model=os.path.basename(model_path.rstrip("/")),
            fmt=os.environ.get("_AR_E2E_FMT", "auto_round"),
            bits=int(os.environ.get("_AR_E2E_BITS", "4")),
            group_size=int(os.environ.get("_AR_E2E_GS", "128")),
            num_prompts=num_prompts,
            max_new_tokens=max_new_tokens,
            total_time_s=total_time,
            output_tokens_per_s=n_out_tokens / max(total_time, 1e-6),
            gen_tokens_per_s=gen_tokens_per_s_vllm,
            ttft_s=ttft_s,
            sample_output=outputs[0].outputs[0].text,
        )
    finally:
        # vLLM 0.6+ supports .shutdown(); older versions fall back to del+gc.
        shutdown = getattr(llm, "shutdown", None)
        if callable(shutdown):
            try:
                shutdown()
            except Exception:
                pass
        del llm
        free_cuda()


# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------


class TestVllmThroughput:
    """Quantize-with-autoround + serve-with-vllm end-to-end suite."""

    @pytest.mark.parametrize(
        "model_case",
        [
            # (hf_id, bits, group_size, sym, fmt, min_gpu_gib)
            pytest.param(
                __import__("test.e2e.test_cuda.conftest", fromlist=["ModelCase"]).ModelCase(
                    "Qwen/Qwen3-1.7B", 4, 128, True, "auto_round", min_gpu_gib=10
                ),
                id="qwen3-1.7b-w4a16-auto_round",
            ),
            pytest.param(
                __import__("test.e2e.test_cuda.conftest", fromlist=["ModelCase"]).ModelCase(
                    "Qwen/Qwen3-1.7B", 4, 128, True, "auto_gptq", min_gpu_gib=10
                ),
                id="qwen3-1.7b-w4a16-auto_gptq",
            ),
            pytest.param(
                __import__("test.e2e.test_cuda.conftest", fromlist=["ModelCase"]).ModelCase(
                    "Qwen/Qwen3-1.7B", 4, 128, True, "auto_awq", min_gpu_gib=10
                ),
                id="qwen3-1.7b-w4a16-auto_awq",
            ),
        ],
    )
    def test_quantize_and_serve(self, model_case, tmp_path, require_gpu_memory):
        """Quantize → save → load with vLLM → measure throughput."""
        save_dir = str(tmp_path / f"saved_{model_case.fmt}_w{model_case.bits}")

        # Quantize with the python API (mirrors `auto-round --format ...`).
        os.environ["_AR_E2E_FMT"] = model_case.fmt
        os.environ["_AR_E2E_BITS"] = str(model_case.bits)
        os.environ["_AR_E2E_GS"] = str(model_case.group_size)

        saved = quantize_and_save(
            model_id=model_case.hf_id,
            bits=model_case.bits,
            group_size=model_case.group_size,
            sym=model_case.sym,
            fmt=model_case.fmt,
            output_dir=save_dir,
            iters=200,
            nsamples=128,
            seqlen=2048,
        )

        # Serve + benchmark.
        result = _run_vllm_benchmark(saved, max_new_tokens=64, num_prompts=4, gpu_mem_util=0.8)
        _record(result)

        # Sanity: the model produced non-empty output and didn't blow up.
        assert result.sample_output.strip(), "vLLM produced empty output"
        assert "!!!" not in result.sample_output, "vLLM produced garbage output"
        assert result.output_tokens_per_s > 0
        # Decoding throughput is bounded below by 1 tok/s on any modern GPU.
        # The number is intentionally loose – this is a regression check, not
        # a perf gate.
        assert (
            result.output_tokens_per_s >= 1.0
        ), f"vLLM throughput suspiciously low: {result.output_tokens_per_s:.2f} tok/s"

        print(
            f"\n[vLLM] {model_case.hf_id} {model_case.fmt} w{model_case.bits} "
            f"-> {result.output_tokens_per_s:.1f} tok/s "
            f"(decode-only: {result.gen_tokens_per_s}, ttft: {result.ttft_s})"
        )


class TestVllmLarge:
    """Heavier cases that need >=24 GiB; skipped on smaller GPUs."""

    @pytest.mark.parametrize(
        "model_case",
        [
            pytest.param(
                __import__("test.e2e.test_cuda.conftest", fromlist=["ModelCase"]).ModelCase(
                    "Qwen/Qwen2.5-7B-Instruct", 4, 128, True, "auto_round", min_gpu_gib=18
                ),
                id="qwen2.5-7b-w4a16-auto_round",
            ),
            pytest.param(
                __import__("test.e2e.test_cuda.conftest", fromlist=["ModelCase"]).ModelCase(
                    "meta-llama/Llama-3.2-3B-Instruct", 4, 128, True, "auto_round", min_gpu_gib=12
                ),
                id="llama-3.2-3b-w4a16-auto_round",
            ),
        ],
    )
    def test_quantize_and_serve(self, model_case, tmp_path, require_gpu_memory):
        save_dir = str(tmp_path / f"saved_{model_case.fmt}_w{model_case.bits}")

        os.environ["_AR_E2E_FMT"] = model_case.fmt
        os.environ["_AR_E2E_BITS"] = str(model_case.bits)
        os.environ["_AR_E2E_GS"] = str(model_case.group_size)

        saved = quantize_and_save(
            model_id=model_case.hf_id,
            bits=model_case.bits,
            group_size=model_case.group_size,
            sym=model_case.sym,
            fmt=model_case.fmt,
            output_dir=save_dir,
            iters=200,
            nsamples=128,
            seqlen=2048,
        )

        result = _run_vllm_benchmark(saved, max_new_tokens=64, num_prompts=4, gpu_mem_util=0.85)
        _record(result)

        assert result.sample_output.strip()
        assert "!!!" not in result.sample_output
        assert result.output_tokens_per_s >= 1.0

        print(
            f"\n[vLLM-large] {model_case.hf_id} {model_case.fmt} w{model_case.bits} "
            f"-> {result.output_tokens_per_s:.1f} tok/s"
        )


def test_vllm_offline_eval_backend(tmp_path, require_cuda):
    """Make sure ``--eval_backend vllm`` in the CLI still works.

    This complements the throughput tests above: rather than driving vLLM
    directly from Python, it spawns the full ``auto-round`` CLI exactly
    as an end user would.  It is also a useful canary for regressions in
    the CLI plumbing.
    """
    import sys
    from test.helpers import get_model_path

    model = get_model_path("Qwen/Qwen3-0.6B")
    output_dir = str(tmp_path / "cli_saved")
    cmd = (
        f"{sys.executable} -m auto_round --model {model} --scheme W4A16 --iters 0 "
        f"--disable_opt_rtn --tasks lambada_openai --eval_backend vllm --limit 4 "
        f"--eval_bs 4 --output_dir {output_dir} "
        f"--vllm_args tensor_parallel_size=1,gpu_memory_utilization=0.5,max_model_len=1024"
    )
    env = os.environ.copy()
    env["VLLM_SKIP_WARMUP"] = "true"
    env["NCCL_ASYNC_ERROR_HANDLING"] = "1"
    env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"

    completed = subprocess.run(cmd, shell=True, env=env, check=False)
    assert completed.returncode == 0, f"`auto-round --eval_backend vllm` failed (rc={completed.returncode})"