Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix KeyError when GPU is missing from accelerate max_memory (#1457)

Liang Lvb697e1d2dde4
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index b1e1b62be..3fbd07ab3 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -2182,10 +2182,25 @@ class BaseCompressor(object):
device = 0
else:
raise ValueError(f"Unsupported device {device} in device_map: {self.device_map}")
+ if device not in max_memory:
+ # Skip devices that are not reported by accelerate's max_memory.
+ # This is expected when a device is unavailable or cannot provide memory info.
+ continue
# Use 90% of the reported max memory to leave headroom for activations,
# temporary tensors, other processes, and allocator fragmentation, reducing
# the chance of runtime OOM while still utilizing most available memory.
new_max_memory[device] = max_memory[device] * 0.9
+
+ # If non-CPU devices were requested but none survived, fall back to CPU caching
+ # via the OOM handler below, avoiding unnecessary dispatch overhead.
+ requested_non_cpu = any((d != "cpu") for d in devices)
+ has_non_cpu_memory = any((k != "cpu") for k in new_max_memory)
+ if requested_non_cpu and not has_non_cpu_memory:
+ raise torch.OutOfMemoryError(
+ "No non-CPU device available in accelerate's reported memory. "
+ "Falling back to CPU caching."
+ )
+
new_max_memory = get_balanced_memory(
self.model,
max_memory=new_max_memory,
@@ -2228,15 +2243,13 @@ class BaseCompressor(object):
cuda_error_msg = traceback.format_exc()
try:
logger.info("switch to cpu to cache block inputs")
+ self.cache_device = torch.device("cpu")
if self.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":
logger.warning(
"we recommend using more GPUs in calibration."
" Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy."
)
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(
- self.model
- ) # self.model.hf_device_map has not been changed
+ accelerate.hooks.remove_hook_from_submodules(self.model)
self.model = mv_module_from_gpu(self.model)
clear_memory(device_list=self.device_list)
# Important change after v0.51, on cpu, we use rtn mode for layers in layer_names
diff --git a/auto_round/compressors/mllm/compressor.py b/auto_round/compressors/mllm/compressor.py
index 905277bed..165d4f3d3 100644
--- a/auto_round/compressors/mllm/compressor.py
+++ b/auto_round/compressors/mllm/compressor.py
@@ -65,20 +65,47 @@ def _only_text_test(model, tokenizer, device, model_type):
new_tokenizer.pad_token = new_tokenizer.eos_token
inputs = new_tokenizer(text, return_tensors="pt", padding=True, truncation=True)
- try:
- inputs = inputs.to(device)
- model = model.to(device)
- model(**inputs)
- return True
- except RuntimeError as e:
- model = model.to("cpu")
- inputs = inputs.to("cpu")
+ # Estimate model size and check if it fits in GPU memory.
+ # When the model is too large, we skip the GPU transfer and test on CPU only.
+ use_gpu = True if device != "cpu" else False
+ if device != "cpu" and torch.cuda.is_available():
+ model_bytes = sum(p.numel() * p.element_size() for p in model.parameters())
+ dev_idx_str = device.split(":")[-1] if ":" in device else "0"
+ if dev_idx_str.isdigit():
+ dev_idx = int(dev_idx_str)
+ if 0 <= dev_idx < torch.cuda.device_count():
+ free_bytes, _ = torch.cuda.mem_get_info(dev_idx)
+ if model_bytes > free_bytes * 0.9:
+ use_gpu = False
+
+ if use_gpu:
try:
+ inputs = inputs.to(device)
+ model = model.to(device)
model(**inputs)
- except:
+ return True
+ except RuntimeError as e:
+ model = model.to("cpu")
+ inputs = inputs.to("cpu")
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ try:
+ model(**inputs)
+ except Exception:
+ return False
+ return True
+ except Exception as e:
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
return False
+
+ # Model too large for GPU — run the test on CPU directly
+ try:
+ inputs = inputs.to("cpu")
+ model = model.to("cpu")
+ model(**inputs)
return True
- except Exception as e:
+ except Exception:
return False
diff --git a/auto_round/compressors/mllm/template.py b/auto_round/compressors/mllm/template.py
index b3f9802ba..75190a091 100644
--- a/auto_round/compressors/mllm/template.py
+++ b/auto_round/compressors/mllm/template.py
@@ -123,6 +123,7 @@ _register_template("mllama", default_dataset="liuhaotian/llava", processor=PROCE
_register_template("deepseek_vl_v2", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["deepseek_v2"])
_register_template("mistral3", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])
_register_template("mistral3_2", default_dataset="liuhaotian/llava", processor=PROCESSORS["mistral3_2"])
+_register_template("gemma3", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])
def load_template(path: str):
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 15594938d..5a0b16bb7 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -36,6 +36,7 @@ SUPPORT_ONLY_TEXT_MODELS = [
"internvl_chat",
"glm4v_moe",
"qwen3_vl_moe",
+ "gemma3",
]
NOT_SUPPORT_ONLY_TEXT_MODELS = ["mllama", "mistral3_2"]
diff --git a/auto_round/utils/device.py b/auto_round/utils/device.py
index ed3ab4d6c..78f595745 100644
--- a/auto_round/utils/device.py
+++ b/auto_round/utils/device.py
@@ -1474,6 +1474,13 @@ def parse_available_devices(device_map: Union[str, torch.device, int, dict, None
# Numeric → assign to first available device type
device_type = device_types[0]
parsed.append(f"{device_type}:{p}" if device_type != "cpu" else "cpu")
+ elif p in device_types and ":" not in p:
+ # Bare device type string (e.g., "cuda", "xpu") → normalize to "type:0".
+ # Special-case CPU: keep it as "cpu" without an index.
+ if p.lower() == "cpu":
+ parsed.append("cpu")
+ else:
+ parsed.append(f"{p}:0")
else:
parsed.append(p)
return list(set(parsed))