Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix xpu oom checker (#705)

Heng Guo32d2cd512bd1
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 5f2e57955..8ea8fb70f 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -63,6 +63,7 @@ from auto_round.utils import (
llm_load_model,
logger,
mv_module_from_gpu,
+ out_of_vram,
reset_params,
set_module,
to_device,
@@ -784,7 +785,7 @@ class AutoRound(object):
**{k: config[k] for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]},
)
except RuntimeError as e:
- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
+ if out_of_vram(e):
logger.info("out of VRAM, falling back to CPU.")
weight, scale, zp = quant_func(
module.weight.to("cpu"),
@@ -921,7 +922,7 @@ class AutoRound(object):
cnt = 1
cnt += 1
except RuntimeError as e:
- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
+ if out_of_vram(e):
try:
if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
import accelerate
@@ -933,7 +934,7 @@ class AutoRound(object):
clear_memory()
self.quantize_via_rtn_blockwise(all_to_quantized_module_names)
except RuntimeError as e:
- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
+ if out_of_vram(e):
# Final fallback: warn and use CPU-only quantization
logger.warning(
"Fallback to CPU. "
@@ -1085,7 +1086,7 @@ class AutoRound(object):
m = m.unwrapper({})
m.to("cpu")
except RuntimeError as e:
- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
+ if out_of_vram(e):
logger.warning("Out of VRAM, falling back to CPU.")
m.to("cpu")
m = WrapperLinear(
@@ -1799,7 +1800,7 @@ class AutoRound(object):
block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
)
except RuntimeError as e:
- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
+ if out_of_vram(e):
logger.info("switch to cpu to cache block inputs")
if self.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":
logger.warning(
@@ -2153,6 +2154,7 @@ class AutoRound(object):
logger.info(dump_info)
def register_act_max_hook(self, model):
+
def get_act_max_hook(module, input, output):
if isinstance(input, (tuple, list)):
input = input[0]
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index f47b60ecc..d1c55f5fd 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -30,6 +30,7 @@ from ..utils import (
get_block_names,
logger,
mllm_load_model,
+ out_of_vram,
to_device,
to_dtype,
)
@@ -57,7 +58,7 @@ def _only_text_test(model, tokenizer, device, model_type):
model(**inputs)
return True
except RuntimeError as e:
- if "CUDA out of memory" in str(e):
+ if out_of_vram(e):
model = model.to("cpu")
inputs = inputs.to("cpu")
try:
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 12b267d13..748b6ef69 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -38,6 +38,7 @@ from auto_round.utils import (
get_fp_layer_names,
get_model_dtype,
infer_bits_by_data_type,
+ out_of_vram,
set_cuda_visible_devices,
str2bool,
)
@@ -842,7 +843,7 @@ def eval_task_by_task(
)
break
except Exception as e:
- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):
+ if out_of_vram(e):
ori_batch_sizes = hflm.batch_sizes if hflm.batch_sizes else {"0": 64}
try:
for k, v in hflm.batch_sizes.items():
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 36f6c70cf..24d1b99a3 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -36,6 +36,7 @@ SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")
class SupportedFormats:
+
def __init__(self):
self._support_format = (
"auto_round",
@@ -967,7 +968,6 @@ TORCH_VERSION_AT_LEAST_2_6 = torch_version_at_least("2.6.0")
TORCH_VERSION_AT_LEAST_2_5 = torch_version_at_least("2.5.0")
TORCH_VERSION_AT_LEAST_2_4 = torch_version_at_least("2.4.0")
-
# Note on HPU usage:
# There are two modes available for enabling auto-round on HPU:
# 1. Compile Mode
@@ -2053,3 +2053,17 @@ def check_need_act_calibration(is_act_dynamic, act_data_type=None):
if act_data_type is not None and "static" in act_data_type:
return True
return False
+
+
+def out_of_vram(error_msg):
+ error_msg = str(error_msg)
+ # CUDA
+ if "CUDA out of memory" in error_msg:
+ return True
+ # gaudi
+ if "MODULE:PT_DEVMEM" in error_msg:
+ return True
+ # XPU
+ if "UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY" in error_msg:
+ return True
+ return False