PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix cuda ut fail (#1370)
Heng Guo6cdef7a2cab5
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 8e6d1fbae..34418dc62 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -504,20 +504,18 @@ class BaseCompressor(object):except (ImportError, ModuleNotFoundError):logger.error("algorithm extension import error, fallback to default mode")- def _gen_auto_scheme(- self, model: torch.nn.Module, scheme: AutoScheme, dataset: str, device_map: Union[str, int, dict, torch.device]- ) -> dict[str, dict]:+ def _gen_auto_scheme(self) -> dict[str, dict]:if self.mllm:logger.info("AutoScheme is not yet supported for multimodal LLMs.")sys.exit(-1)- if is_quantized_input_module(model):+ if is_quantized_input_module(self.model):logger.info("AutoScheme does not currently support quantized input models (e.g., FP8).")sys.exit(-1)all_dtypes = []all_gguf = True- for option in scheme.options:+ for option in self.orig_scheme.options:# Resolve the quantization scheme or data typedtype = "int"if isinstance(option, str):@@ -569,15 +567,15 @@ class BaseCompressor(object):# mainly using quant_layers and fixed by usersfrom auto_round.auto_scheme.gen_auto_scheme import GenScheme- if not self.enable_torch_compile and self.super_bits is None and not scheme.low_gpu_mem_usage:+ if not self.enable_torch_compile and self.super_bits is None and not self.orig_scheme.low_gpu_mem_usage:logger.warning("we strongly recommend to set `enable_torch_compile` to True for AutoScheme to save VRAM")self.scheme_generator = GenScheme(- scheme,+ self.orig_scheme,self.model,quant_layer_names,fixed_layer_scheme_new,- dataset,- device_map=device_map,+ self.dataset,+ device_map=self.device_map,tokenizer=self.tokenizer,enable_torch_compile=self.enable_torch_compile,)@@ -1543,7 +1541,7 @@ class BaseCompressor(object):self.ignore_layers += "," + tmp_strif self.is_auto_scheme:- self.layer_config = self._gen_auto_scheme(self.model, self.orig_scheme, self.dataset, self.device_map)+ self.layer_config = self._gen_auto_scheme()fill_default_value = Trueif self.is_auto_scheme:diff --git a/test/test_cuda/models/test_get_block_name.py b/test/test_cuda/models/test_get_block_name.pyindex 1850ec5c7..e8928d1f5 100644--- a/test/test_cuda/models/test_get_block_name.py+++ b/test/test_cuda/models/test_get_block_name.py@@ -165,7 +165,7 @@ class TestAutoRound:assert not is_pure_text_model(model)def test_Mistral3(self):- model_name = "/models/Mistral-Small-3.1-24B-Instruct-2503"+ model_name = "/models/Mistral-Small-3.2-24B-Instruct-2506"model = Mistral3ForConditionalGeneration.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)block_names = get_block_names(model)self.check_block_names(block_names, ["model.language_model.layers"], [40])diff --git a/test/test_cuda/models/test_support_vlms.py b/test/test_cuda/models/test_support_vlms.pyindex 798674c83..67296925a 100644--- a/test/test_cuda/models/test_support_vlms.py+++ b/test/test_cuda/models/test_support_vlms.py@@ -4,11 +4,14 @@ import sysimport pytestimport requests+from packaging import versionfrom PIL import Imagefrom transformers import AutoRoundConfig # # must import for auto-round formatfrom auto_round.testing_utils import require_gptqmodel, require_package_version_ut, require_vlm_env+from ...helpers import transformers_version+AUTO_ROUND_PATH = __file__.split("/")AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])@@ -177,7 +180,10 @@ class TestSupportVLMS:print(response)shutil.rmtree(quantized_model_path, ignore_errors=True)- @require_package_version_ut("transformers", "<4.54.0")+ @pytest.mark.skipif(+ transformers_version >= version.parse("4.57.0"),+ reason="transformers api changed",+ )def test_glm(self):model_path = "/models/glm-4v-9b/"## test tunediff --git a/test/test_cuda/models/test_vlms.py b/test/test_cuda/models/test_vlms.pyindex b0aa4a327..ad7fcaa11 100644--- a/test/test_cuda/models/test_vlms.py+++ b/test/test_cuda/models/test_vlms.py@@ -136,7 +136,7 @@ class TestAutoRound:"/models/Phi-3.5-vision-instruct","/models/Qwen2-VL-2B-Instruct","/models/SmolVLM-256M-Instruct",- "/models/Mistral-Small-3.1-24B-Instruct-2503",+ "/models/Mistral-Small-3.2-24B-Instruct-2506","/models/InternVL3-1B","/models/pixtral-12b",]: