PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix CUDA UT issue (#2238)
Xin He9e030fd1ef91
diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.pyindex 02151b6fc..27d28994f 100644--- a/auto_round/eval/eval_cli.py+++ b/auto_round/eval/eval_cli.py@@ -270,7 +270,6 @@ def eval_with_vllm(args):from lm_eval import evaluator # pylint: disable=E0401from lm_eval.models.vllm_causallms import VLLM # pylint: disable=E0401- from lm_eval.models.vllm_vlms import VLLM_VLM # pylint: disable=E0401from lm_eval.utils import make_table # pylint: disable=E0401st = time.time()@@ -334,7 +333,12 @@ def eval_with_vllm(args):logger.info(f"Set {env_name}={os.environ[env_name]}, tensor_parallel_size={tensor_parallel_size}")- vllm_lm = VLLM_VLM(**vllm_kwargs) if args.mllm else VLLM(**vllm_kwargs)+ if args.mllm:+ from lm_eval.models.vllm_vlms import VLLM_VLM # pylint: disable=E0401++ vllm_lm = VLLM_VLM(**vllm_kwargs)+ else:+ vllm_lm = VLLM(**vllm_kwargs)res = evaluator.simple_evaluate(model=vllm_lm,tasks=tasks,diff --git a/test/e2e/test_cuda/conftest.py b/test/e2e/test_cuda/conftest.pyindex 1235cf996..7adf16464 100644--- a/test/e2e/test_cuda/conftest.py+++ b/test/e2e/test_cuda/conftest.py@@ -222,8 +222,12 @@ def quantize_and_save(--sym --format {fmt} --output_dir {output_dir} \\--iters {iters} --nsamples {nsamples} --seqlen {seqlen}"""+ from test.helpers import get_model_path+from auto_round import AutoRound # local import: heavy module+ model_id = get_model_path(model_id) if "/" in model_id else model_id+shutil.rmtree(output_dir, ignore_errors=True)ar = AutoRound(model=model_id,diff --git a/test/e2e/test_cuda/test_vllm_throughput.py b/test/e2e/test_cuda/test_vllm_throughput.pyindex d1367e4f8..9e06a3461 100644--- a/test/e2e/test_cuda/test_vllm_throughput.py+++ b/test/e2e/test_cuda/test_vllm_throughput.py@@ -43,6 +43,7 @@ Run the full (large) matrix::import jsonimport os+import subprocessimport timefrom test.e2e.test_cuda.conftest import ( # noqa: E402BenchResult,@@ -102,9 +103,8 @@ def _build_vllm_engine(model_path: str, max_model_len: int, gpu_mem_util: float)if not (current_platform.is_cpu() or current_platform.is_xpu() or current_platform.is_cuda()):pytest.skip("vLLM tests only run on CPU/XPU/CUDA")- # ``auto-round`` is registered as a vLLM plugin via entrypoints. We still- # pass ``quantization="auto-round"`` explicitly to make the dependency- # obvious in CI logs and to be future-proof against entrypoint changes.+ # Let vLLM infer quantization backend from the exported checkpoint config.+ # Newer vLLM versions do not accept legacy aliases like "auto-round".return LLM(model=model_path,quantization="auto-round",@@ -113,6 +113,7 @@ def _build_vllm_engine(model_path: str, max_model_len: int, gpu_mem_util: float)gpu_memory_utilization=gpu_mem_util,max_model_len=max_model_len,dtype="auto",+ allow_deprecated_quantization=True,enforce_eager=False,)@@ -356,5 +357,5 @@ def test_vllm_offline_eval_backend(tmp_path, require_cuda):env["NCCL_ASYNC_ERROR_HANDLING"] = "1"env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"- rc = os.system(cmd)- assert rc == 0, f"`auto-round --eval_backend vllm` failed (rc={rc})"+ completed = subprocess.run(cmd, shell=True, env=env, check=False)+ assert completed.returncode == 0, f"`auto-round --eval_backend vllm` failed (rc={completed.returncode})"