PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Refine CPU RAM offload strategy (#1544)
Liang Lvc37add634f67
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 0a2352e2a..421396571 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1431,7 +1431,7 @@ class BaseCompressor(object):shard_writer(self, name=block_name)block.to("meta")if self.low_cpu_mem_usage and not self.is_immediate_saving:- self._offloader.offload(self.model, block_name)+ self._offloader(self.model, block_name)clear_memory(device_list=self.device_list)memory_monitor.log_summary()pbar.update(1)@@ -1595,7 +1595,7 @@ class BaseCompressor(object):mv_module_from_gpu(block)if self.low_cpu_mem_usage and not self.is_immediate_saving:- self._offloader.offload(self.model, block_name)+ self._offloader(self.model, block_name)if block_name == block_names[-1]:clear_memory(input_ids, device_list=self.device_list)else:@@ -1698,14 +1698,6 @@ class BaseCompressor(object):if self.low_cpu_mem_usage and self.is_immediate_packing:self.is_immediate_saving = True- if self.low_cpu_mem_usage and not self.is_immediate_packing:- logger.info(- "`low_cpu_mem_usage` is only supported when `immediate_packing` is True. "- "Setting `low_cpu_mem_usage` to False."- )- self.low_cpu_mem_usage = False- self.is_immediate_saving = False-if self.low_cpu_mem_usage and self.is_immediate_packing:if formats[0].is_gguf():logger.warning(@@ -1819,7 +1811,12 @@ class BaseCompressor(object):clear_memory(device_list=self.device_list)logger.info("caching done")if self.low_cpu_mem_usage:- self._offloader.offload(self.model, all_blocks, clear_memory=True, device_list=self.device_list)+ if self.is_model_patched and not self.is_immediate_saving:+ self._offloader(self.model, all_blocks, clear_memory=True, device_list=self.device_list)+ if not self._offloader.enabled:+ self.low_cpu_mem_usage = False+ else:+ self.low_cpu_mem_usage = Falseif len(all_blocks) > 1:pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))else:@@ -3299,10 +3296,10 @@ class BaseCompressor(object):if self.low_cpu_mem_usage and not self.is_immediate_saving:if nblocks == 1:- self._offloader.offload(model, n, overwrite=True)+ self._offloader(model, n, overwrite=True)else:for name in names:- self._offloader.offload(model, name, overwrite=True)+ self._offloader(model, name, overwrite=True)if pbar is not None:pbar.update(1)diff --git a/auto_round/envs.py b/auto_round/envs.pyindex 448644500..b4576dece 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -30,6 +30,7 @@ environment_variables: dict[str, Callable[[], Any]] = {"AR_ENABLE_UNIFY_MOE_INPUT_SCALE": lambda: os.getenv("AR_ENABLE_UNIFY_MOE_INPUT_SCALE", "False").lower()in ["1", "true"],"AR_OMP_NUM_THREADS": lambda: os.getenv("AR_OMP_NUM_THREADS", None),+ "AR_DISABLE_OFFLOAD": lambda: os.getenv("AR_DISABLE_OFFLOAD", "0").lower() in ("1", "true", "yes"),}diff --git a/auto_round/utils/offload.py b/auto_round/utils/offload.pyindex 1ce64d13c..a95399be0 100644--- a/auto_round/utils/offload.py+++ b/auto_round/utils/offload.py@@ -30,8 +30,8 @@ Usage patterns**Compressor (offload mode)**::offloader = OffloadManager(mode="offload")- offloader.offload(model, "model.layers.0") # save + clear one module- offloader.offload(model, block_names, clear_memory=True) # save + clear many + gc+ offloader(model, "model.layers.0") # save + clear one module+ offloader(model, block_names, clear_memory=True) # save + clear many + gcoffloader.reload(model, "model.layers.0") # load back + auto-clean temp file**AutoScheme (clean mode with hooks)**::@@ -262,7 +262,9 @@ class OffloadManager:offload_dir_prefix: str = "ar_offload",cache_numel: bool = False,):- self.enabled = enabled+ from auto_round import envs++ self.enabled = enabled and not envs.AR_DISABLE_OFFLOADself.mode = modeself.model_dir = model_dirself.cache_numel = cache_numel@@ -298,6 +300,26 @@ class OffloadManager:except Exception:pass+ def __call__(+ self,+ model: torch.nn.Module,+ names: Union[str, list[str], list[list[str]]],+ *,+ skip_if_saved: bool = False,+ overwrite: bool = False,+ clear_memory: bool = False,+ device_list=None,+ ) -> float:+ """Alias for offload() so the manager can be invoked directly."""+ return self.offload(+ model,+ names,+ skip_if_saved=skip_if_saved,+ overwrite=overwrite,+ clear_memory=clear_memory,+ device_list=device_list,+ )+# ------------------------------------------------------------------# Core API# ------------------------------------------------------------------@@ -338,6 +360,9 @@ class OffloadManager:"""if not self.enabled:return 0.0+ if self.mode == "offload" and not self._check_disk_space(model, names):+ self.enabled = False+ return 0.0if isinstance(names, str):self._offload(model, names, skip_if_saved=skip_if_saved, overwrite=overwrite)return 0.0@@ -360,6 +385,51 @@ class OffloadManager:logger.info(f"offload done, freed {total_gb:.2f} GB")return total_gb+ def _check_disk_space(self, model: torch.nn.Module, names: Union[str, list[str], list[list[str]]]) -> bool:+ """Check whether there is enough disk space to offload the given modules.++ Args:+ model: The root model containing the module(s).+ names: Module name(s) to check.++ Returns:+ True if sufficient disk space is available, False otherwise.+ """+ if isinstance(names, str):+ flat_names = [names]+ else:+ flat_names = self._flatten_names(names)+ total_bytes = 0+ for name in flat_names:+ module = get_module(model, name)+ if module is None:+ continue+ # Estimate size based on state_dict (parameters + buffers), excluding meta tensors,+ # to match what is actually written by the offload logic.+ state_dict = module.state_dict()+ for tensor in state_dict.values():+ if not isinstance(tensor, torch.Tensor):+ continue+ if tensor.is_meta or tensor.numel() == 0:+ continue+ total_bytes += tensor.numel() * tensor.element_size()+ # torch.save adds serialization overhead; use 1.2x safety margin+ required_bytes = int(total_bytes * 1.2)+ from auto_round import envs++ target_dir = os.path.join(envs.AR_WORK_SPACE, "offload")+ os.makedirs(target_dir, exist_ok=True)+ free_bytes = shutil.disk_usage(target_dir).free+ if free_bytes < required_bytes:+ required_gb = required_bytes / (1024**3)+ free_gb = free_bytes / (1024**3)+ logger.warning(+ f"Insufficient disk space for offloading: need ~{required_gb:.2f} GB "+ f"(including safety margin) but only {free_gb:.2f} GB available at {target_dir}. Skipping offload."+ )+ return False+ return True+def _offload(self, model: torch.nn.Module, name: str, *, skip_if_saved: bool = False, overwrite: bool = False) -> None:diff --git a/docs/environments.md b/docs/environments.mdindex 14428f922..78e71eab7 100644--- a/docs/environments.md+++ b/docs/environments.md@@ -108,6 +108,16 @@ else:print("AR_LOG_LEVEL is using default value")```+### AR_DISABLE_OFFLOAD+- **Description**: Forcibly disables the weight offloading feature in `OffloadManager`. Useful during development and debugging to skip all offload/reload overhead.+- **Default**: `False` (equivalent to `"0"`)+- **Valid Values**: `"1"`, `"true"`, `"yes"` (case-insensitive) for disabling offload; any other value keeps the default behavior+- **Usage**: Set this to bypass offloading entirely++```bash+export AR_DISABLE_OFFLOAD=1+```+## Configuration Best Practices1. **Development Environment**: Set `AR_LOG_LEVEL=TRACE` or `AR_LOG_LEVEL=DEBUG` for detailed logging during developmentdiff --git a/test/test_cpu/core/test_low_cpu_mem_options.py b/test/test_cpu/core/test_low_cpu_mem_options.pyindex 543a9a515..b56ff1cea 100644--- a/test/test_cpu/core/test_low_cpu_mem_options.py+++ b/test/test_cpu/core/test_low_cpu_mem_options.py@@ -64,7 +64,7 @@ class TestLowCpuMemUsage:)assert autoround.low_cpu_mem_usage is False# offload with list should be a no-op when offloader is disabled- autoround._offloader.offload(+ autoround._offloader(autoround.model, [["model.layers.0"]], clear_memory=True, device_list=autoround.device_list)assert autoround._offloader._saved == {}@@ -89,7 +89,7 @@ class TestLowCpuMemUsage:# Force the offloader to think it has a tempdir alreadyautoround._offloader._tempdir = str(tmp_path)- autoround._offloader.offload(+ autoround._offloader(autoround.model, [["model.layers.0"]], clear_memory=True, device_list=autoround.device_list)assert autoround._offloader.has("model.layers.0")