PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix xpu oom checker (#705)
Heng Guo32d2cd512bd1
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 5f2e57955..8ea8fb70f 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -63,6 +63,7 @@ from auto_round.utils import (llm_load_model,logger,mv_module_from_gpu,+ out_of_vram,reset_params,set_module,to_device,@@ -784,7 +785,7 @@ class AutoRound(object):**{k: config[k] for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]},)except RuntimeError as e:- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ if out_of_vram(e):logger.info("out of VRAM, falling back to CPU.")weight, scale, zp = quant_func(module.weight.to("cpu"),@@ -921,7 +922,7 @@ class AutoRound(object):cnt = 1cnt += 1except RuntimeError as e:- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ if out_of_vram(e):try:if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:import accelerate@@ -933,7 +934,7 @@ class AutoRound(object):clear_memory()self.quantize_via_rtn_blockwise(all_to_quantized_module_names)except RuntimeError as e:- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ if out_of_vram(e):# Final fallback: warn and use CPU-only quantizationlogger.warning("Fallback to CPU. "@@ -1085,7 +1086,7 @@ class AutoRound(object):m = m.unwrapper({})m.to("cpu")except RuntimeError as e:- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ if out_of_vram(e):logger.warning("Out of VRAM, falling back to CPU.")m.to("cpu")m = WrapperLinear(@@ -1799,7 +1800,7 @@ class AutoRound(object):block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name)except RuntimeError as e:- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ if out_of_vram(e):logger.info("switch to cpu to cache block inputs")if self.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":logger.warning(@@ -2153,6 +2154,7 @@ class AutoRound(object):logger.info(dump_info)def register_act_max_hook(self, model):+def get_act_max_hook(module, input, output):if isinstance(input, (tuple, list)):input = input[0]diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex f47b60ecc..d1c55f5fd 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -30,6 +30,7 @@ from ..utils import (get_block_names,logger,mllm_load_model,+ out_of_vram,to_device,to_dtype,)@@ -57,7 +58,7 @@ def _only_text_test(model, tokenizer, device, model_type):model(**inputs)return Trueexcept RuntimeError as e:- if "CUDA out of memory" in str(e):+ if out_of_vram(e):model = model.to("cpu")inputs = inputs.to("cpu")try:diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 12b267d13..748b6ef69 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -38,6 +38,7 @@ from auto_round.utils import (get_fp_layer_names,get_model_dtype,infer_bits_by_data_type,+ out_of_vram,set_cuda_visible_devices,str2bool,)@@ -842,7 +843,7 @@ def eval_task_by_task()breakexcept Exception as e:- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):+ if out_of_vram(e):ori_batch_sizes = hflm.batch_sizes if hflm.batch_sizes else {"0": 64}try:for k, v in hflm.batch_sizes.items():diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 36f6c70cf..24d1b99a3 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -36,6 +36,7 @@ SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")class SupportedFormats:+def __init__(self):self._support_format = ("auto_round",@@ -967,7 +968,6 @@ TORCH_VERSION_AT_LEAST_2_6 = torch_version_at_least("2.6.0")TORCH_VERSION_AT_LEAST_2_5 = torch_version_at_least("2.5.0")TORCH_VERSION_AT_LEAST_2_4 = torch_version_at_least("2.4.0")-# Note on HPU usage:# There are two modes available for enabling auto-round on HPU:# 1. Compile Mode@@ -2053,3 +2053,17 @@ def check_need_act_calibration(is_act_dynamic, act_data_type=None):if act_data_type is not None and "static" in act_data_type:return Truereturn False+++def out_of_vram(error_msg):+ error_msg = str(error_msg)+ # CUDA+ if "CUDA out of memory" in error_msg:+ return True+ # gaudi+ if "MODULE:PT_DEVMEM" in error_msg:+ return True+ # XPU+ if "UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY" in error_msg:+ return True+ return False