PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix KeyError when GPU is missing from accelerate max_memory (#1457)
Liang Lvb697e1d2dde4
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex b1e1b62be..3fbd07ab3 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -2182,10 +2182,25 @@ class BaseCompressor(object):device = 0else:raise ValueError(f"Unsupported device {device} in device_map: {self.device_map}")+ if device not in max_memory:+ # Skip devices that are not reported by accelerate's max_memory.+ # This is expected when a device is unavailable or cannot provide memory info.+ continue# Use 90% of the reported max memory to leave headroom for activations,# temporary tensors, other processes, and allocator fragmentation, reducing# the chance of runtime OOM while still utilizing most available memory.new_max_memory[device] = max_memory[device] * 0.9++ # If non-CPU devices were requested but none survived, fall back to CPU caching+ # via the OOM handler below, avoiding unnecessary dispatch overhead.+ requested_non_cpu = any((d != "cpu") for d in devices)+ has_non_cpu_memory = any((k != "cpu") for k in new_max_memory)+ if requested_non_cpu and not has_non_cpu_memory:+ raise torch.OutOfMemoryError(+ "No non-CPU device available in accelerate's reported memory. "+ "Falling back to CPU caching."+ )+new_max_memory = get_balanced_memory(self.model,max_memory=new_max_memory,@@ -2228,15 +2243,13 @@ class BaseCompressor(object):cuda_error_msg = traceback.format_exc()try:logger.info("switch to cpu to cache block inputs")+ self.cache_device = torch.device("cpu")if self.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":logger.warning("we recommend using more GPUs in calibration."" Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy.")- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- accelerate.hooks.remove_hook_from_submodules(- self.model- ) # self.model.hf_device_map has not been changed+ accelerate.hooks.remove_hook_from_submodules(self.model)self.model = mv_module_from_gpu(self.model)clear_memory(device_list=self.device_list)# Important change after v0.51, on cpu, we use rtn mode for layers in layer_namesdiff --git a/auto_round/compressors/mllm/compressor.py b/auto_round/compressors/mllm/compressor.pyindex 905277bed..165d4f3d3 100644--- a/auto_round/compressors/mllm/compressor.py+++ b/auto_round/compressors/mllm/compressor.py@@ -65,20 +65,47 @@ def _only_text_test(model, tokenizer, device, model_type):new_tokenizer.pad_token = new_tokenizer.eos_tokeninputs = new_tokenizer(text, return_tensors="pt", padding=True, truncation=True)- try:- inputs = inputs.to(device)- model = model.to(device)- model(**inputs)- return True- except RuntimeError as e:- model = model.to("cpu")- inputs = inputs.to("cpu")+ # Estimate model size and check if it fits in GPU memory.+ # When the model is too large, we skip the GPU transfer and test on CPU only.+ use_gpu = True if device != "cpu" else False+ if device != "cpu" and torch.cuda.is_available():+ model_bytes = sum(p.numel() * p.element_size() for p in model.parameters())+ dev_idx_str = device.split(":")[-1] if ":" in device else "0"+ if dev_idx_str.isdigit():+ dev_idx = int(dev_idx_str)+ if 0 <= dev_idx < torch.cuda.device_count():+ free_bytes, _ = torch.cuda.mem_get_info(dev_idx)+ if model_bytes > free_bytes * 0.9:+ use_gpu = False++ if use_gpu:try:+ inputs = inputs.to(device)+ model = model.to(device)model(**inputs)- except:+ return True+ except RuntimeError as e:+ model = model.to("cpu")+ inputs = inputs.to("cpu")+ if torch.cuda.is_available():+ torch.cuda.empty_cache()+ try:+ model(**inputs)+ except Exception:+ return False+ return True+ except Exception as e:+ if torch.cuda.is_available():+ torch.cuda.empty_cache()return False++ # Model too large for GPU — run the test on CPU directly+ try:+ inputs = inputs.to("cpu")+ model = model.to("cpu")+ model(**inputs)return True- except Exception as e:+ except Exception:return Falsediff --git a/auto_round/compressors/mllm/template.py b/auto_round/compressors/mllm/template.pyindex b3f9802ba..75190a091 100644--- a/auto_round/compressors/mllm/template.py+++ b/auto_round/compressors/mllm/template.py@@ -123,6 +123,7 @@ _register_template("mllama", default_dataset="liuhaotian/llava", processor=PROCE_register_template("deepseek_vl_v2", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["deepseek_v2"])_register_template("mistral3", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])_register_template("mistral3_2", default_dataset="liuhaotian/llava", processor=PROCESSORS["mistral3_2"])+_register_template("gemma3", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])def load_template(path: str):diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 15594938d..5a0b16bb7 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -36,6 +36,7 @@ SUPPORT_ONLY_TEXT_MODELS = ["internvl_chat","glm4v_moe","qwen3_vl_moe",+ "gemma3",]NOT_SUPPORT_ONLY_TEXT_MODELS = ["mllama", "mistral3_2"]diff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex ed3ab4d6c..78f595745 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -1474,6 +1474,13 @@ def parse_available_devices(device_map: Union[str, torch.device, int, dict, None# Numeric → assign to first available device typedevice_type = device_types[0]parsed.append(f"{device_type}:{p}" if device_type != "cpu" else "cpu")+ elif p in device_types and ":" not in p:+ # Bare device type string (e.g., "cuda", "xpu") → normalize to "type:0".+ # Special-case CPU: keep it as "cpu" without an index.+ if p.lower() == "cpu":+ parsed.append("cpu")+ else:+ parsed.append(f"{p}:0")else:parsed.append(p)return list(set(parsed))