Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix new arch bug for llmc (#1781)

Heng Guoaa29325a5ade
diff --git a/auto_round/compressors_new/base.py b/auto_round/compressors_new/base.py
index e8dc9d135..5a6d07d70 100644
--- a/auto_round/compressors_new/base.py
+++ b/auto_round/compressors_new/base.py
@@ -192,7 +192,7 @@ class BaseCompressor(object):
logger.warning("The static kv is experimental and currently has limited support.")
if kwargs:
- logger.warning(
+ logger.warning_once(
f"unrecognized keys {list(kwargs.keys())} were passed. "
"Please check them. If you use old api, just ignore this warning."
)
@@ -609,16 +609,18 @@ class BaseCompressor(object):
# On HPU, we rely on torch.compile to speed up the model execution.
if self.enable_torch_compile and is_raw_fp8 and not is_hpex_available():
self.enable_torch_compile = False
- logger.warning("reset enable_torch_compile to `False` as fp8 is enabled")
+ logger.warning_once("reset enable_torch_compile to `False` as fp8 is enabled")
# TODO: fix https://github.com/intel/auto-round/issues/1109
if self.enable_torch_compile and is_raw_nv_fp:
self.enable_torch_compile = False
- logger.warning("reset enable_torch_compile to `False` as nvfp4 is enabled")
+ logger.warning_once("reset enable_torch_compile to `False` as nvfp4 is enabled")
super_group_size = getattr(cfg, "super_group_size", None)
enable_alg_ext = getattr(cfg, "enable_alg_ext", False)
if self.enable_torch_compile and super_group_size is not None and enable_alg_ext:
self.enable_torch_compile = False
- logger.warning("reset enable_torch_compile to `False` as super_group_size is set for algorithm extension")
+ logger.warning_once(
+ "reset enable_torch_compile to `False` as super_group_size is set for algorithm extension"
+ )
def _get_calibration_dataset(self) -> str:
"""Resolve calibration dataset: self.dataset > AutoScheme.dataset > default."""
diff --git a/auto_round/compressors_new/calib.py b/auto_round/compressors_new/calib.py
index 67c77c058..4041f9379 100644
--- a/auto_round/compressors_new/calib.py
+++ b/auto_round/compressors_new/calib.py
@@ -788,15 +788,22 @@ class CalibCompressor(BaseCompressor):
``enable_quanted_input`` is ``False``), and *reference_output* is the
full-precision reference output collected before optimization.
"""
- assert not self.mllm and not self.diffusion, (
- f"Currently, {self.__class__.__name__} does not support quantize_block " "for MLLM / diffusion models."
- )
+ assert (
+ not self.diffusion
+ ), f"Currently, {self.__class__.__name__} does not support quantize_block for diffusion models."
# Ensure post_init has been called (sets up model_context, compress_context,
# quantizer, layer_config, etc.).
if not self._post_init_done:
self.post_init()
+ # When called from LLM-Compressor, `wrapped_model` is a single decoder layer
+ # (not the full VL model), so it must not be treated as an MLLM regardless of
+ # whether the original model had multimodal assets. Force is_mllm=False for
+ # the duration of this call to stay on the standard LLM quantize_block path.
+ orig_is_mllm = self.model_context.is_mllm
+ self.model_context.is_mllm = False
+
self.normalize_decoding_layer_inputs_(inputs)
block_inputs = self.inputs[self.quant_block_list[0][0]]
input_ids, input_others = self._preprocess_block_inputs(block_inputs, "hidden_states")
@@ -881,6 +888,7 @@ class CalibCompressor(BaseCompressor):
accelerate.hooks.remove_hook_from_submodules(block)
mv_module_from_gpu(block)
+ self.model_context.is_mllm = orig_is_mllm
return q_outputs, reference_output
def _quantize_blocks(
diff --git a/auto_round/compressors_new/entry.py b/auto_round/compressors_new/entry.py
index e26691528..1a9f984fd 100644
--- a/auto_round/compressors_new/entry.py
+++ b/auto_round/compressors_new/entry.py
@@ -505,13 +505,14 @@ class AutoRoundCompatible:
num_inference_steps = kwargs.pop("num_inference_steps", 50)
generator_seed = kwargs.pop("generator_seed", None)
- # Check model type for logging
+ # Check model type for logging (use warning_once to avoid repeating for every block
+ # when called from LLM-Compressor which instantiates AutoRound per block)
if is_mllm_model(model, platform=platform):
- logger.info("Using MLLM mode for multimodal model (new architecture).")
+ logger.warning_once("Using MLLM mode for multimodal model (new architecture).")
elif is_diffusion_model(model):
- logger.info("Using Diffusion mode for diffusion model (new architecture).")
+ logger.warning_once("Using Diffusion mode for diffusion model (new architecture).")
else:
- logger.info("Using LLM mode (new architecture).")
+ logger.warning_once("Using LLM mode (new architecture).")
# Create AutoRound instance using new architecture
compressor = AutoRound(
diff --git a/auto_round/compressors_new/zero_shot.py b/auto_round/compressors_new/zero_shot.py
index 977244639..31ccc2494 100644
--- a/auto_round/compressors_new/zero_shot.py
+++ b/auto_round/compressors_new/zero_shot.py
@@ -90,9 +90,9 @@ class ZeroShotCompressor(BaseCompressor):
Returns:
tuple: ``(None, None)`` — RTN does not produce reference outputs.
"""
- assert not self.mllm and not self.diffusion, (
- f"Currently, {self.__class__.__name__} does not support quantize_block " "for MLLM / diffusion models."
- )
+ assert (
+ not self.diffusion
+ ), f"Currently, {self.__class__.__name__} does not support quantize_block for diffusion models."
if not self._post_init_done:
self.post_init()
diff --git a/auto_round/context/base.py b/auto_round/context/base.py
index e3f75fb8f..82c5d9770 100644
--- a/auto_round/context/base.py
+++ b/auto_round/context/base.py
@@ -45,7 +45,7 @@ class BaseContext(metaclass=AutoSkipInitMeta):
_instances = {}
def __init__(self):
- logger.info(f"{self.__class__.__name__} context initialized.")
+ logger.debug(f"{self.__class__.__name__} context initialized.")
@classmethod
def get_context(cls):
diff --git a/auto_round/context/model.py b/auto_round/context/model.py
index 49648997e..9bb46152c 100644
--- a/auto_round/context/model.py
+++ b/auto_round/context/model.py
@@ -119,7 +119,7 @@ class ModelContext(BaseContext):
if self.model.dtype != torch.bfloat16:
self.model = self.model.to(torch.bfloat16)
else:
- logger.info(f"using {self.model.dtype} for quantization tuning")
+ logger.debug(f"using {self.model.dtype} for quantization tuning")
# Reclaim C heap fragmentation left by model/tokenizer loading so
# that the quantize loop starts from a tighter RSS baseline.
diff --git a/auto_round/utils/device.py b/auto_round/utils/device.py
index c05a17e79..2be868452 100644
--- a/auto_round/utils/device.py
+++ b/auto_round/utils/device.py
@@ -1709,11 +1709,17 @@ class MemoryMonitor:
if str(device) == "cpu":
continue
if torch.cuda.is_available():
- current_vram = torch.cuda.memory_reserved(device) / 1024**3 # GB
+ try:
+ current_vram = torch.cuda.memory_reserved(device) / 1024**3 # GB
+ except (RuntimeError, Exception):
+ continue # Skip devices that are not initialized or out of range
if device == "cuda":
device = "0"
elif torch.xpu.is_available():
- current_vram = torch.xpu.memory_reserved(device) / 1024**3 # GB
+ try:
+ current_vram = torch.xpu.memory_reserved(device) / 1024**3 # GB
+ except (RuntimeError, Exception):
+ continue # Skip devices that are not initialized or out of range
if device == "xpu":
device = "0"
elif is_hpex_available():
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 575b7ada0..004a619c5 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -759,10 +759,18 @@ def is_pure_text_model(model):
return True
+_is_mllm_model_cache: dict = {}
+
+
def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = None):
from auto_round.utils.common import MM_KEYS
model_path = model_or_path if isinstance(model_or_path, str) else model_or_path.name_or_path
+
+ # Fast path: return cached result for already-seen paths
+ if model_path in _is_mllm_model_cache:
+ return _is_mllm_model_cache[model_path]
+
# For dummy model, model_path could be "".
# Only try to download if the path looks like a HF repo id (not a local filesystem path).
# Skip download for absolute paths or relative paths that contain current/parent dir markers.
@@ -770,24 +778,30 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = No
if model_path and not os.path.isdir(model_path) and not _is_local_path:
model_path = download_or_get_path(model_path, platform=platform)
+ result = False
if isinstance(model_path, str):
if os.path.exists(os.path.join(model_path, "preprocessor_config.json")):
- return True
- if os.path.exists(os.path.join(model_path, "processor_config.json")):
- return True
- if os.path.exists(os.path.join(model_path, "config.json")):
+ result = True
+ elif os.path.exists(os.path.join(model_path, "processor_config.json")):
+ result = True
+ elif os.path.exists(os.path.join(model_path, "config.json")):
with open(os.path.join(model_path, "config.json")) as f:
config = json.load(f)
for key in config.keys():
if any([k in key for k in MM_KEYS]):
- return True
+ result = True
+ break
- if isinstance(model_or_path, torch.nn.Module):
+ if not result and isinstance(model_or_path, torch.nn.Module):
for name, module in model_or_path.named_modules():
if any([k in name for k in MM_KEYS]):
- return True
+ result = True
+ break
- return False
+ # Cache by the original path key (model_path may have been resolved above)
+ original_key = model_or_path if isinstance(model_or_path, str) else model_or_path.name_or_path
+ _is_mllm_model_cache[original_key] = result
+ return result
def is_gguf_model(model_path: Union[str, torch.nn.Module]) -> bool: