Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Refine CPU RAM offload strategy (#1544)

Liang Lvc37add634f67
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 0a2352e2a..421396571 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -1431,7 +1431,7 @@ class BaseCompressor(object):
shard_writer(self, name=block_name)
block.to("meta")
if self.low_cpu_mem_usage and not self.is_immediate_saving:
- self._offloader.offload(self.model, block_name)
+ self._offloader(self.model, block_name)
clear_memory(device_list=self.device_list)
memory_monitor.log_summary()
pbar.update(1)
@@ -1595,7 +1595,7 @@ class BaseCompressor(object):
mv_module_from_gpu(block)
if self.low_cpu_mem_usage and not self.is_immediate_saving:
- self._offloader.offload(self.model, block_name)
+ self._offloader(self.model, block_name)
if block_name == block_names[-1]:
clear_memory(input_ids, device_list=self.device_list)
else:
@@ -1698,14 +1698,6 @@ class BaseCompressor(object):
if self.low_cpu_mem_usage and self.is_immediate_packing:
self.is_immediate_saving = True
- if self.low_cpu_mem_usage and not self.is_immediate_packing:
- logger.info(
- "`low_cpu_mem_usage` is only supported when `immediate_packing` is True. "
- "Setting `low_cpu_mem_usage` to False."
- )
- self.low_cpu_mem_usage = False
- self.is_immediate_saving = False
-
if self.low_cpu_mem_usage and self.is_immediate_packing:
if formats[0].is_gguf():
logger.warning(
@@ -1819,7 +1811,12 @@ class BaseCompressor(object):
clear_memory(device_list=self.device_list)
logger.info("caching done")
if self.low_cpu_mem_usage:
- self._offloader.offload(self.model, all_blocks, clear_memory=True, device_list=self.device_list)
+ if self.is_model_patched and not self.is_immediate_saving:
+ self._offloader(self.model, all_blocks, clear_memory=True, device_list=self.device_list)
+ if not self._offloader.enabled:
+ self.low_cpu_mem_usage = False
+ else:
+ self.low_cpu_mem_usage = False
if len(all_blocks) > 1:
pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))
else:
@@ -3299,10 +3296,10 @@ class BaseCompressor(object):
if self.low_cpu_mem_usage and not self.is_immediate_saving:
if nblocks == 1:
- self._offloader.offload(model, n, overwrite=True)
+ self._offloader(model, n, overwrite=True)
else:
for name in names:
- self._offloader.offload(model, name, overwrite=True)
+ self._offloader(model, name, overwrite=True)
if pbar is not None:
pbar.update(1)
diff --git a/auto_round/envs.py b/auto_round/envs.py
index 448644500..b4576dece 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -30,6 +30,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
"AR_ENABLE_UNIFY_MOE_INPUT_SCALE": lambda: os.getenv("AR_ENABLE_UNIFY_MOE_INPUT_SCALE", "False").lower()
in ["1", "true"],
"AR_OMP_NUM_THREADS": lambda: os.getenv("AR_OMP_NUM_THREADS", None),
+ "AR_DISABLE_OFFLOAD": lambda: os.getenv("AR_DISABLE_OFFLOAD", "0").lower() in ("1", "true", "yes"),
}
diff --git a/auto_round/utils/offload.py b/auto_round/utils/offload.py
index 1ce64d13c..a95399be0 100644
--- a/auto_round/utils/offload.py
+++ b/auto_round/utils/offload.py
@@ -30,8 +30,8 @@ Usage patterns
**Compressor (offload mode)**::
offloader = OffloadManager(mode="offload")
- offloader.offload(model, "model.layers.0") # save + clear one module
- offloader.offload(model, block_names, clear_memory=True) # save + clear many + gc
+ offloader(model, "model.layers.0") # save + clear one module
+ offloader(model, block_names, clear_memory=True) # save + clear many + gc
offloader.reload(model, "model.layers.0") # load back + auto-clean temp file
**AutoScheme (clean mode with hooks)**::
@@ -262,7 +262,9 @@ class OffloadManager:
offload_dir_prefix: str = "ar_offload",
cache_numel: bool = False,
):
- self.enabled = enabled
+ from auto_round import envs
+
+ self.enabled = enabled and not envs.AR_DISABLE_OFFLOAD
self.mode = mode
self.model_dir = model_dir
self.cache_numel = cache_numel
@@ -298,6 +300,26 @@ class OffloadManager:
except Exception:
pass
+ def __call__(
+ self,
+ model: torch.nn.Module,
+ names: Union[str, list[str], list[list[str]]],
+ *,
+ skip_if_saved: bool = False,
+ overwrite: bool = False,
+ clear_memory: bool = False,
+ device_list=None,
+ ) -> float:
+ """Alias for offload() so the manager can be invoked directly."""
+ return self.offload(
+ model,
+ names,
+ skip_if_saved=skip_if_saved,
+ overwrite=overwrite,
+ clear_memory=clear_memory,
+ device_list=device_list,
+ )
+
# ------------------------------------------------------------------
# Core API
# ------------------------------------------------------------------
@@ -338,6 +360,9 @@ class OffloadManager:
"""
if not self.enabled:
return 0.0
+ if self.mode == "offload" and not self._check_disk_space(model, names):
+ self.enabled = False
+ return 0.0
if isinstance(names, str):
self._offload(model, names, skip_if_saved=skip_if_saved, overwrite=overwrite)
return 0.0
@@ -360,6 +385,51 @@ class OffloadManager:
logger.info(f"offload done, freed {total_gb:.2f} GB")
return total_gb
+ def _check_disk_space(self, model: torch.nn.Module, names: Union[str, list[str], list[list[str]]]) -> bool:
+ """Check whether there is enough disk space to offload the given modules.
+
+ Args:
+ model: The root model containing the module(s).
+ names: Module name(s) to check.
+
+ Returns:
+ True if sufficient disk space is available, False otherwise.
+ """
+ if isinstance(names, str):
+ flat_names = [names]
+ else:
+ flat_names = self._flatten_names(names)
+ total_bytes = 0
+ for name in flat_names:
+ module = get_module(model, name)
+ if module is None:
+ continue
+ # Estimate size based on state_dict (parameters + buffers), excluding meta tensors,
+ # to match what is actually written by the offload logic.
+ state_dict = module.state_dict()
+ for tensor in state_dict.values():
+ if not isinstance(tensor, torch.Tensor):
+ continue
+ if tensor.is_meta or tensor.numel() == 0:
+ continue
+ total_bytes += tensor.numel() * tensor.element_size()
+ # torch.save adds serialization overhead; use 1.2x safety margin
+ required_bytes = int(total_bytes * 1.2)
+ from auto_round import envs
+
+ target_dir = os.path.join(envs.AR_WORK_SPACE, "offload")
+ os.makedirs(target_dir, exist_ok=True)
+ free_bytes = shutil.disk_usage(target_dir).free
+ if free_bytes < required_bytes:
+ required_gb = required_bytes / (1024**3)
+ free_gb = free_bytes / (1024**3)
+ logger.warning(
+ f"Insufficient disk space for offloading: need ~{required_gb:.2f} GB "
+ f"(including safety margin) but only {free_gb:.2f} GB available at {target_dir}. Skipping offload."
+ )
+ return False
+ return True
+
def _offload(
self, model: torch.nn.Module, name: str, *, skip_if_saved: bool = False, overwrite: bool = False
) -> None:
diff --git a/docs/environments.md b/docs/environments.md
index 14428f922..78e71eab7 100644
--- a/docs/environments.md
+++ b/docs/environments.md
@@ -108,6 +108,16 @@ else:
print("AR_LOG_LEVEL is using default value")
```
+### AR_DISABLE_OFFLOAD
+- **Description**: Forcibly disables the weight offloading feature in `OffloadManager`. Useful during development and debugging to skip all offload/reload overhead.
+- **Default**: `False` (equivalent to `"0"`)
+- **Valid Values**: `"1"`, `"true"`, `"yes"` (case-insensitive) for disabling offload; any other value keeps the default behavior
+- **Usage**: Set this to bypass offloading entirely
+
+```bash
+export AR_DISABLE_OFFLOAD=1
+```
+
## Configuration Best Practices
1. **Development Environment**: Set `AR_LOG_LEVEL=TRACE` or `AR_LOG_LEVEL=DEBUG` for detailed logging during development
diff --git a/test/test_cpu/core/test_low_cpu_mem_options.py b/test/test_cpu/core/test_low_cpu_mem_options.py
index 543a9a515..b56ff1cea 100644
--- a/test/test_cpu/core/test_low_cpu_mem_options.py
+++ b/test/test_cpu/core/test_low_cpu_mem_options.py
@@ -64,7 +64,7 @@ class TestLowCpuMemUsage:
)
assert autoround.low_cpu_mem_usage is False
# offload with list should be a no-op when offloader is disabled
- autoround._offloader.offload(
+ autoround._offloader(
autoround.model, [["model.layers.0"]], clear_memory=True, device_list=autoround.device_list
)
assert autoround._offloader._saved == {}
@@ -89,7 +89,7 @@ class TestLowCpuMemUsage:
# Force the offloader to think it has a tempdir already
autoround._offloader._tempdir = str(tmp_path)
- autoround._offloader.offload(
+ autoround._offloader(
autoround.model, [["model.layers.0"]], clear_memory=True, device_list=autoround.device_list
)
assert autoround._offloader.has("model.layers.0")