Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix cuda ut fail (#1370)

Heng Guo6cdef7a2cab5
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 8e6d1fbae..34418dc62 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -504,20 +504,18 @@ class BaseCompressor(object):
except (ImportError, ModuleNotFoundError):
logger.error("algorithm extension import error, fallback to default mode")
- def _gen_auto_scheme(
- self, model: torch.nn.Module, scheme: AutoScheme, dataset: str, device_map: Union[str, int, dict, torch.device]
- ) -> dict[str, dict]:
+ def _gen_auto_scheme(self) -> dict[str, dict]:
if self.mllm:
logger.info("AutoScheme is not yet supported for multimodal LLMs.")
sys.exit(-1)
- if is_quantized_input_module(model):
+ if is_quantized_input_module(self.model):
logger.info("AutoScheme does not currently support quantized input models (e.g., FP8).")
sys.exit(-1)
all_dtypes = []
all_gguf = True
- for option in scheme.options:
+ for option in self.orig_scheme.options:
# Resolve the quantization scheme or data type
dtype = "int"
if isinstance(option, str):
@@ -569,15 +567,15 @@ class BaseCompressor(object):
# mainly using quant_layers and fixed by users
from auto_round.auto_scheme.gen_auto_scheme import GenScheme
- if not self.enable_torch_compile and self.super_bits is None and not scheme.low_gpu_mem_usage:
+ if not self.enable_torch_compile and self.super_bits is None and not self.orig_scheme.low_gpu_mem_usage:
logger.warning("we strongly recommend to set `enable_torch_compile` to True for AutoScheme to save VRAM")
self.scheme_generator = GenScheme(
- scheme,
+ self.orig_scheme,
self.model,
quant_layer_names,
fixed_layer_scheme_new,
- dataset,
- device_map=device_map,
+ self.dataset,
+ device_map=self.device_map,
tokenizer=self.tokenizer,
enable_torch_compile=self.enable_torch_compile,
)
@@ -1543,7 +1541,7 @@ class BaseCompressor(object):
self.ignore_layers += "," + tmp_str
if self.is_auto_scheme:
- self.layer_config = self._gen_auto_scheme(self.model, self.orig_scheme, self.dataset, self.device_map)
+ self.layer_config = self._gen_auto_scheme()
fill_default_value = True
if self.is_auto_scheme:
diff --git a/test/test_cuda/models/test_get_block_name.py b/test/test_cuda/models/test_get_block_name.py
index 1850ec5c7..e8928d1f5 100644
--- a/test/test_cuda/models/test_get_block_name.py
+++ b/test/test_cuda/models/test_get_block_name.py
@@ -165,7 +165,7 @@ class TestAutoRound:
assert not is_pure_text_model(model)
def test_Mistral3(self):
- model_name = "/models/Mistral-Small-3.1-24B-Instruct-2503"
+ model_name = "/models/Mistral-Small-3.2-24B-Instruct-2506"
model = Mistral3ForConditionalGeneration.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
block_names = get_block_names(model)
self.check_block_names(block_names, ["model.language_model.layers"], [40])
diff --git a/test/test_cuda/models/test_support_vlms.py b/test/test_cuda/models/test_support_vlms.py
index 798674c83..67296925a 100644
--- a/test/test_cuda/models/test_support_vlms.py
+++ b/test/test_cuda/models/test_support_vlms.py
@@ -4,11 +4,14 @@ import sys
import pytest
import requests
+from packaging import version
from PIL import Image
from transformers import AutoRoundConfig # # must import for auto-round format
from auto_round.testing_utils import require_gptqmodel, require_package_version_ut, require_vlm_env
+from ...helpers import transformers_version
+
AUTO_ROUND_PATH = __file__.split("/")
AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])
@@ -177,7 +180,10 @@ class TestSupportVLMS:
print(response)
shutil.rmtree(quantized_model_path, ignore_errors=True)
- @require_package_version_ut("transformers", "<4.54.0")
+ @pytest.mark.skipif(
+ transformers_version >= version.parse("4.57.0"),
+ reason="transformers api changed",
+ )
def test_glm(self):
model_path = "/models/glm-4v-9b/"
## test tune
diff --git a/test/test_cuda/models/test_vlms.py b/test/test_cuda/models/test_vlms.py
index b0aa4a327..ad7fcaa11 100644
--- a/test/test_cuda/models/test_vlms.py
+++ b/test/test_cuda/models/test_vlms.py
@@ -136,7 +136,7 @@ class TestAutoRound:
"/models/Phi-3.5-vision-instruct",
"/models/Qwen2-VL-2B-Instruct",
"/models/SmolVLM-256M-Instruct",
- "/models/Mistral-Small-3.1-24B-Instruct-2503",
+ "/models/Mistral-Small-3.2-24B-Instruct-2506",
"/models/InternVL3-1B",
"/models/pixtral-12b",
]: