Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix CUDA UT issue (#2238)

Xin He9e030fd1ef91
diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.py
index 02151b6fc..27d28994f 100644
--- a/auto_round/eval/eval_cli.py
+++ b/auto_round/eval/eval_cli.py
@@ -270,7 +270,6 @@ def eval_with_vllm(args):
from lm_eval import evaluator # pylint: disable=E0401
from lm_eval.models.vllm_causallms import VLLM # pylint: disable=E0401
- from lm_eval.models.vllm_vlms import VLLM_VLM # pylint: disable=E0401
from lm_eval.utils import make_table # pylint: disable=E0401
st = time.time()
@@ -334,7 +333,12 @@ def eval_with_vllm(args):
logger.info(f"Set {env_name}={os.environ[env_name]}, tensor_parallel_size={tensor_parallel_size}")
- vllm_lm = VLLM_VLM(**vllm_kwargs) if args.mllm else VLLM(**vllm_kwargs)
+ if args.mllm:
+ from lm_eval.models.vllm_vlms import VLLM_VLM # pylint: disable=E0401
+
+ vllm_lm = VLLM_VLM(**vllm_kwargs)
+ else:
+ vllm_lm = VLLM(**vllm_kwargs)
res = evaluator.simple_evaluate(
model=vllm_lm,
tasks=tasks,
diff --git a/test/e2e/test_cuda/conftest.py b/test/e2e/test_cuda/conftest.py
index 1235cf996..7adf16464 100644
--- a/test/e2e/test_cuda/conftest.py
+++ b/test/e2e/test_cuda/conftest.py
@@ -222,8 +222,12 @@ def quantize_and_save(
--sym --format {fmt} --output_dir {output_dir} \\
--iters {iters} --nsamples {nsamples} --seqlen {seqlen}
"""
+ from test.helpers import get_model_path
+
from auto_round import AutoRound # local import: heavy module
+ model_id = get_model_path(model_id) if "/" in model_id else model_id
+
shutil.rmtree(output_dir, ignore_errors=True)
ar = AutoRound(
model=model_id,
diff --git a/test/e2e/test_cuda/test_vllm_throughput.py b/test/e2e/test_cuda/test_vllm_throughput.py
index d1367e4f8..9e06a3461 100644
--- a/test/e2e/test_cuda/test_vllm_throughput.py
+++ b/test/e2e/test_cuda/test_vllm_throughput.py
@@ -43,6 +43,7 @@ Run the full (large) matrix::
import json
import os
+import subprocess
import time
from test.e2e.test_cuda.conftest import ( # noqa: E402
BenchResult,
@@ -102,9 +103,8 @@ def _build_vllm_engine(model_path: str, max_model_len: int, gpu_mem_util: float)
if not (current_platform.is_cpu() or current_platform.is_xpu() or current_platform.is_cuda()):
pytest.skip("vLLM tests only run on CPU/XPU/CUDA")
- # ``auto-round`` is registered as a vLLM plugin via entrypoints. We still
- # pass ``quantization="auto-round"`` explicitly to make the dependency
- # obvious in CI logs and to be future-proof against entrypoint changes.
+ # Let vLLM infer quantization backend from the exported checkpoint config.
+ # Newer vLLM versions do not accept legacy aliases like "auto-round".
return LLM(
model=model_path,
quantization="auto-round",
@@ -113,6 +113,7 @@ def _build_vllm_engine(model_path: str, max_model_len: int, gpu_mem_util: float)
gpu_memory_utilization=gpu_mem_util,
max_model_len=max_model_len,
dtype="auto",
+ allow_deprecated_quantization=True,
enforce_eager=False,
)
@@ -356,5 +357,5 @@ def test_vllm_offline_eval_backend(tmp_path, require_cuda):
env["NCCL_ASYNC_ERROR_HANDLING"] = "1"
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
- rc = os.system(cmd)
- assert rc == 0, f"`auto-round --eval_backend vllm` failed (rc={rc})"
+ completed = subprocess.run(cmd, shell=True, env=env, check=False)
+ assert completed.returncode == 0, f"`auto-round --eval_backend vllm` failed (rc={completed.returncode})"