Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Optimize CPU RAM peak memory during quantization (#1386)

Liang Lv0d3721511da9
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index 58b2bae3c..ea99fa511 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -15,7 +15,7 @@
import copy
from dataclasses import asdict
from functools import wraps
-from typing import Iterable, Union
+from typing import Iterable, Optional, Union
import torch
from accelerate import dispatch_model
@@ -55,6 +55,8 @@ from auto_round.utils import (
set_non_auto_device_map,
to_device,
)
+from auto_round.utils.device import MemoryMonitor
+from auto_round.utils.offload import OffloadManager
from auto_round.wrapper import WrapperLinear
__all__ = ["gen_layer_config"]
@@ -488,6 +490,7 @@ def get_score_for_scheme(
major_device="cpu",
batch_size=1,
disable_opt_rtn=True,
+ offload_context: Optional[OffloadManager] = None,
):
scores_dict = {} # Key=name,Val=[quant_total_bits, loss]
for n, m in model.named_modules():
@@ -506,6 +509,8 @@ def get_score_for_scheme(
break
for name in quant_layer_names:
+ if offload_context is not None:
+ offload_context.ensure_loaded(model, name)
if name in fixed_layer_scheme.keys():
continue
m = get_module(model, name)
@@ -547,6 +552,8 @@ def get_score_for_scheme(
disable_opt_rtn=disable_opt_rtn,
)
set_module(model, name, new_m)
+ if offload_context is not None:
+ offload_context.flush_loaded(model)
if low_gpu_mem_usage:
dataloader = get_dataloader(tokenizer, seqlen, dataset_name=dataset, seed=42, bs=batch_size, nsamples=nsamples)
@@ -658,6 +665,21 @@ def _gen_layer_config(
major_device="cpu",
device_list=None,
):
+ # Initialize memory tracking for AutoScheme
+ memory_monitor = MemoryMonitor()
+ memory_monitor.reset()
+ memory_monitor.update_cpu()
+
+ # Create offload context for CPU RAM optimization
+ # Note: low_cpu_mem_usage only works when low_gpu_mem_usage is also enabled,
+ # because it requires layer-by-layer processing
+ offload_context = None
+ if auto_scheme.low_cpu_mem_usage and auto_scheme.low_gpu_mem_usage:
+ _model_dir = model_name
+ if _model_dir is None and hasattr(model, "config"):
+ _model_dir = getattr(model.config, "_name_or_path", None)
+ offload_context = OffloadManager(enabled=True, mode="clean", model_dir=_model_dir, cache_numel=True)
+
target_bits = auto_scheme.avg_bits
model.eval()
@@ -762,6 +784,11 @@ def _gen_layer_config(
cal_imatrix(model, dataloader)
logger.info("finish calculating imatrix")
+ # Register hooks and clear all block weights before the scheme loop.
+ # Hooks will transparently reload weights on demand during forward passes.
+ if offload_context is not None:
+ offload_context.add_offload_hooks(model, block_name)
+
pbar = tqdm(total=pbar_cnt, desc="Generating AutoScheme")
for index, scheme in enumerate(schemes):
apply_quant_scheme(
@@ -792,7 +819,11 @@ def _gen_layer_config(
major_device=major_device,
batch_size=batch_size,
disable_opt_rtn=auto_scheme.disable_opt_rtn,
+ offload_context=offload_context,
)
+ # Track peak RAM after each scheme scoring
+ memory_monitor.update_cpu()
+
new_scores = {}
for share_layer in shared_layers:
param_bits = 0
@@ -817,10 +848,17 @@ def _gen_layer_config(
options_scores.append(options_total_loss)
clear_memory(device_list=device_list)
+ # Remove hooks and restore original weights from disk for final bit-budget computations
+ if offload_context is not None:
+ offload_context.remove_offload_hooks(model, block_name)
+
total_params = 0
for n, m in model.named_modules():
if n in quant_layer_names + embedding_layers_names:
- total_params += m.weight.numel()
+ n_param = m.weight.numel()
+ if n_param == 0 and hasattr(m, "_cached_weight_numel"):
+ n_param = m._cached_weight_numel
+ total_params += n_param
target_params_cnt = int(total_params * target_bits)
sorted_indices = sorted(range(len(options_scores)), key=lambda i: options_scores[i])
@@ -902,6 +940,12 @@ def _gen_layer_config(
global last_grad_input
last_grad_input = None
clear_memory(device_list=device_list)
+
+ # Log AutoScheme memory usage
+ memory_monitor.update_cpu()
+ low_cpu_str = "enabled" if auto_scheme.low_cpu_mem_usage else "disabled"
+ memory_monitor.log_summary(f"AutoScheme complete (low_cpu_mem_usage={low_cpu_str})")
+
pbar.close()
return layer_config
diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.py
index 8572a920a..55ec4e205 100644
--- a/auto_round/auto_scheme/gen_auto_scheme.py
+++ b/auto_round/auto_scheme/gen_auto_scheme.py
@@ -40,6 +40,7 @@ class AutoScheme:
enable_torch_compile: Optional[bool] = None
disable_opt_rtn: bool = True
low_gpu_mem_usage: bool = True
+ low_cpu_mem_usage: bool = True
def __post_init__(self):
if isinstance(self.options, str):
diff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.py
index c04780ff6..df1783d83 100644
--- a/auto_round/auto_scheme/utils.py
+++ b/auto_round/auto_scheme/utils.py
@@ -105,7 +105,10 @@ def compute_avg_bits_for_scheme(
# continue
if not hasattr(module, "weight"):
continue
- total_params += module.weight.numel()
+ n_param = module.weight.numel()
+ if n_param == 0 and hasattr(module, "_cached_weight_numel"):
+ n_param = module._cached_weight_numel
+ total_params += n_param
layer_bits, _ = compute_layer_bits(module, ignore_scale_zp_bits)
total_quantized_bits += layer_bits
avg_bits = float(total_quantized_bits) / total_params
@@ -133,7 +136,10 @@ def compute_avg_bits_for_model(model: torch.nn.Module, ignore_scale_zp_bits: boo
continue
if not hasattr(module, "weight"):
continue
- total_params += module.weight.numel()
+ n_param = module.weight.numel()
+ if n_param == 0 and hasattr(module, "_cached_weight_numel"):
+ n_param = module._cached_weight_numel
+ total_params += n_param
layer_bits, _ = compute_layer_bits(module, ignore_scale_zp_bits)
total_quantized_bits += layer_bits
@@ -157,6 +163,9 @@ def compute_layer_bits(
"""
weight = layer.weight
n_param = weight.numel()
+ # Use cached numel when weight has been cleared to an empty tensor (low_cpu_mem_usage offload)
+ if n_param == 0 and hasattr(layer, "_cached_weight_numel"):
+ n_param = layer._cached_weight_numel
weight_bits = getattr(layer, "bits", 16)
group_size = getattr(layer, "group_size", 128)
data_type = getattr(layer, "data_type", "int")
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 5472c373d..afcc489cd 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -14,6 +14,7 @@
import copy
import os
+import re
import sys
import time
import traceback
@@ -113,6 +114,7 @@ from auto_round.utils.device import (
set_non_auto_device_map,
)
from auto_round.utils.distributed import setup_ddp_if_needed_
+from auto_round.utils.offload import OffloadManager
from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_block
SERIALIZATION_KEYS = (
@@ -371,6 +373,7 @@ class BaseCompressor(object):
self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
self.scale_dtype = convert_dtype_str2torch(scale_dtype)
self.low_cpu_mem_usage = low_cpu_mem_usage
+ self._offloader = OffloadManager(enabled=low_cpu_mem_usage, mode="offload", offload_dir_prefix="compressor")
if kwargs:
logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
@@ -1259,6 +1262,10 @@ class BaseCompressor(object):
m = get_module(self.model, name)
m.to("cpu")
shard_writer(self, m, name, False)
+ # Free RAM immediately: the data is now in the shard-writer buffer
+ # (and will be flushed to disk). Keeping it also in the model tree
+ # causes linear RAM growth for large models.
+ m.to("meta")
def _immediate_pack(self, name: str):
if not self.is_immediate_packing:
@@ -1331,8 +1338,16 @@ class BaseCompressor(object):
for handle in hook_handles:
handle.remove()
else:
- # By default, we go with layer-wise way if no replacement happened
+ # By default, we go with layer-wise way if no replacement happened.
+ # In RTN mode (iters == 0), force blockwise quantization to avoid
+ # full-model materialization and linear CPU RAM growth.
use_blockwise_quantization = global_state.replaced_module_count > 0
+ if self.iters == 0 and not use_blockwise_quantization:
+ logger.info(
+ "RTN mode detected (iters=0): force blockwise quantization to avoid "
+ "layer-wise full-model materialization."
+ )
+ use_blockwise_quantization = True
tied_weights_keys = getattr(self.model, "_tied_weights_keys", [])
if tied_weights_keys is None:
tied_weights_keys = []
@@ -1370,8 +1385,22 @@ class BaseCompressor(object):
and self.is_immediate_saving
):
set_module(self.model, m.global_name, copy.deepcopy(m))
- shard_writer(self, name=m.global_name)
+ if self.is_immediate_saving:
+ shard_writer(self, name=m.global_name)
+ copied_m = get_module(self.model, m.global_name)
+ copied_m.to("meta")
m.to("meta")
+ # Move remaining GPU tensors to CPU; offload to disk if low_cpu_mem_usage.
+ # This mirrors _quantize_via_rtn_blockwise's post-block cleanup.
+ if not self.is_immediate_saving:
+ mv_module_from_gpu(block)
+ else:
+ # Save once at block scope to capture tensors that are not saved
+ # in per-layer branch (e.g., custom module-level params/buffers).
+ shard_writer(self, name=block_name)
+ block.to("meta")
+ if self.low_cpu_mem_usage and not self.is_immediate_saving:
+ self._offloader.offload(self.model, block_name)
clear_memory(device_list=self.device_list)
memory_monitor.log_summary()
pbar.update(1)
@@ -1413,6 +1442,8 @@ class BaseCompressor(object):
# Convert remaining fp8
convert_module_to_hp_if_necessary(self.model, self.amp_dtype, self.device)
+ if self.low_cpu_mem_usage:
+ self._offloader.reload(self.model)
if self.is_immediate_saving:
shard_writer(self, is_finalize=True)
@@ -1531,9 +1562,9 @@ class BaseCompressor(object):
self._quantize_layer_via_rtn(m.global_name, to_cpu=self.low_gpu_mem_usage)
all_to_quantized_module_names.remove(m.global_name)
- if not self.is_immediate_saving:
- # some modules may have been flushed and set to meta, so we could not move to gpu
- mv_module_from_gpu(block)
+ mv_module_from_gpu(block)
+ if self.low_cpu_mem_usage and not self.is_immediate_saving:
+ self._offloader.offload(self.model, block_name)
if block_name == block_names[-1]:
clear_memory(input_ids, device_list=self.device_list)
else:
@@ -1644,14 +1675,18 @@ class BaseCompressor(object):
self.is_immediate_saving = False
if self.low_cpu_mem_usage and self.is_immediate_packing:
- if self.has_qlayer_outside_block and self.disable_opt_rtn and self.iters == 0:
+ if formats[0].is_gguf():
logger.warning(
- "`low_cpu_mem_usage` is not fully supported "
- "when there are quantized layers outside blocks and optimized RTN is disabled. "
+ "`low_cpu_mem_usage` is not fully supported for gguf format. "
"Setting `low_cpu_mem_usage` to False."
)
self.low_cpu_mem_usage = False
self.is_immediate_saving = False
+ elif self.has_qlayer_outside_block and self.disable_opt_rtn and self.iters == 0:
+ logger.info(
+ "Keeping `low_cpu_mem_usage` enabled in RTN mode (iters=0): "
+ "RTN path uses blockwise quantization and supports per-block offloading."
+ )
elif self.has_qlayer_outside_block and self.iters > 0:
logger.warning(
"`low_cpu_mem_usage` is not fully supported "
@@ -1660,12 +1695,6 @@ class BaseCompressor(object):
)
self.low_cpu_mem_usage = False
self.is_immediate_saving = False
- elif formats[0].is_gguf():
- logger.warning(
- "`low_cpu_mem_usage` is not fully supported for gguf format" "Setting `low_cpu_mem_usage `to False."
- )
- self.low_cpu_mem_usage = False
- self.is_immediate_saving = False
if self.is_immediate_saving and "int" not in self.data_type:
logger.warning("immediate_saving is only supported for int quantization, set to False")
@@ -1700,6 +1729,9 @@ class BaseCompressor(object):
self.configure_layer_config(enable_gguf_official_mixed=enable_gguf_official_mixed)
+ if self.low_cpu_mem_usage:
+ self._offloader.reset()
+
def _should_disable_inplace_due_to_layers_outside_block() -> bool:
return self.has_qlayer_outside_block and (self.iters != 0 or (self.iters == 0 and not self.disable_opt_rtn))
@@ -1747,11 +1779,15 @@ class BaseCompressor(object):
all_q_inputs = self.try_cache_inter_data_gpucpu(
all_first_block_names, self.nsamples, layer_names=layer_names
)
- self.model = safe_device_move_with_meta_handling(self.model, "cpu")
- clear_memory(device_list=self.device_list)
+ # Remove accelerate dispatch hooks before moving parameters.
+ # hf_device_map is kept for reference but hooks are no longer needed.
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(self.model) # self.model.hf_device_map has not been changed
+ accelerate.hooks.remove_hook_from_submodules(self.model)
+ self.model = mv_module_from_gpu(self.model)
+ clear_memory(device_list=self.device_list)
logger.info("caching done")
+ if self.low_cpu_mem_usage:
+ self._offloader.offload(self.model, all_blocks, clear_memory=True, device_list=self.device_list)
if len(all_blocks) > 1:
pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))
else:
@@ -1797,6 +1833,9 @@ class BaseCompressor(object):
if self.is_immediate_saving:
shard_writer(self, is_finalize=True)
+ if self.low_cpu_mem_usage:
+ self._offloader.reload(self.model)
+
end_time = time.time()
cost_time = end_time - start_time
logger.info(f"quantization tuning time {cost_time}")
@@ -2923,7 +2962,7 @@ class BaseCompressor(object):
if hook_handles:
self._get_block_outputs(
block,
- q_input,
+ q_input if q_input is not None else input_ids,
input_others,
self.batch_size * self.infer_bs_coeff,
device,
@@ -3201,6 +3240,12 @@ class BaseCompressor(object):
modules = [get_module(model, n) for n in names]
m = WrapperMultiblock(modules)
+ if self.low_cpu_mem_usage:
+ if nblocks == 1:
+ self._offloader.reload(model, n)
+ else:
+ self._offloader.reload(model, names)
+
m.config = model.config if hasattr(model, "config") else None
q_input, input_ids = self._quantize_block(
m,
@@ -3219,6 +3264,13 @@ class BaseCompressor(object):
if self.is_immediate_saving:
shard_writer(self, m, is_finalize=False)
+
+ if self.low_cpu_mem_usage and not self.is_immediate_saving:
+ if nblocks == 1:
+ self._offloader.offload(model, n, overwrite=True)
+ else:
+ for name in names:
+ self._offloader.offload(model, name, overwrite=True)
if pbar is not None:
pbar.update(1)
diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.py
index 62362bff3..990c5c387 100644
--- a/auto_round/compressors/shard_writer.py
+++ b/auto_round/compressors/shard_writer.py
@@ -58,6 +58,7 @@ class ShardWriter:
# Stats
self.total_param_elems = 0
self.total_param_size_bytes = 0
+ self.skipped_meta_tensors = []
# Directory Setup
self.output_dir = os.path.join(rounder._get_save_folder_name(rounder.formats[0]), "")
@@ -95,6 +96,9 @@ class ShardWriter:
self._add_tensor(param_name, v)
def _add_tensor(self, name: str, tensor: torch.Tensor):
+ if isinstance(tensor, torch.Tensor) and tensor.device.type == "meta":
+ self.skipped_meta_tensors.append(name)
+ return
t_size = tensor.nbytes
self.total_param_elems += tensor.numel()
self.total_param_size_bytes += t_size
@@ -162,6 +166,7 @@ class ShardWriter:
tie_word_embeddings = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)
all_saved_names = {p for meta in self.shard_meta for p in meta["params"]}
+ finalize_skipped_meta_tensors = []
for pname, tensor in full_sd.items():
if pname in all_saved_names:
continue
@@ -176,6 +181,10 @@ class ShardWriter:
self._flush_shard()
+ total_skipped = len(self.skipped_meta_tensors) + len(finalize_skipped_meta_tensors)
+ if total_skipped > 0:
+ examples = (self.skipped_meta_tensors + finalize_skipped_meta_tensors)[:5]
+
# 2. Rename temp files to HF standard and map weights
if self.shard_counter == 0:
logger.warning("No tensors saved.")
diff --git a/auto_round/utils/device.py b/auto_round/utils/device.py
index 78f595745..d605268a2 100644
--- a/auto_round/utils/device.py
+++ b/auto_round/utils/device.py
@@ -11,16 +11,19 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import ctypes
import functools
import gc
import os
import re
+import shutil
import sys
+import tempfile
from contextlib import ContextDecorator, contextmanager
from functools import lru_cache
from itertools import combinations
from threading import Lock
-from typing import Callable, Union
+from typing import Any, Callable, Optional, Union
import cpuinfo
import psutil
@@ -513,6 +516,7 @@ def _clear_memory_for_cpu_and_cuda(
tensor[i] = None
tensor = None
gc.collect()
+ _maybe_trim_malloc()
# ------------------------
# Normalize device_list
@@ -556,6 +560,40 @@ def _clear_memory_for_cpu_and_cuda(
torch.xpu.empty_cache()
+_malloc_trim_counter = 0
+
+
+def _maybe_trim_malloc() -> None:
+ """Optionally release glibc heap pages back to OS on Linux.
+
+ Controlled by environment variables:
+ - AR_ENABLE_MALLOC_TRIM: default "1" (enabled)
+ - AR_MALLOC_TRIM_EVERY: default "10" (trigger every N clear_memory calls)
+ """
+ global _malloc_trim_counter
+
+ if os.name != "posix":
+ return
+ if os.environ.get("AR_ENABLE_MALLOC_TRIM", "1") != "1":
+ return
+
+ try:
+ every = int(os.environ.get("AR_MALLOC_TRIM_EVERY", "10"))
+ except ValueError:
+ every = 10
+ every = max(1, every)
+
+ _malloc_trim_counter += 1
+ if _malloc_trim_counter % every != 0:
+ return
+
+ try:
+ libc = ctypes.CDLL("libc.so.6")
+ libc.malloc_trim(0)
+ except Exception:
+ pass
+
+
class ClearMemory:
def __init__(self, device_list: list | tuple | None = None):
self.device_list = device_list
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 0e8bb5938..91f2b7e89 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -1164,13 +1164,27 @@ def mv_module_from_gpu(module):
The module on the specified device.
"""
if hasattr(module, "device"):
- target_device = "cpu"
- if module.device.type == target_device:
+ if module.device.type in ("cpu", "meta"):
return module
- else:
- return module.to(target_device)
- else:
- return module.to("cpu")
+
+ has_meta = any(p.device.type == "meta" for p in module.parameters())
+ if not has_meta:
+ has_meta = any(b.device.type == "meta" for b in module.buffers())
+
+ if has_meta:
+ for _, child in module.named_children():
+ mv_module_from_gpu(child)
+ for attr_name in list(module._parameters.keys()):
+ p = module._parameters[attr_name]
+ if p is not None and p.device.type != "meta" and p.device.type != "cpu":
+ module._parameters[attr_name] = p.to("cpu")
+ for attr_name in list(module._buffers.keys()):
+ b = module._buffers[attr_name]
+ if b is not None and b.device.type != "meta" and b.device.type != "cpu":
+ module._buffers[attr_name] = b.to("cpu")
+ return module
+
+ return module.to("cpu")
def safe_device_move_with_meta_handling(
diff --git a/auto_round/utils/offload.py b/auto_round/utils/offload.py
new file mode 100644
index 000000000..b39d2fbc9
--- /dev/null
+++ b/auto_round/utils/offload.py
@@ -0,0 +1,692 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Offloading module for CPU RAM optimization.
+
+This module provides :class:`OffloadManager`, a single class that can
+**offload** any named module (block, layer, or sub-module) to free CPU RAM
+and **reload** it on demand. Two operating modes are supported:
+
+* ``"offload"`` -- saves the module's ``state_dict`` to a temporary directory
+ before clearing. Best when the original weights may have been modified
+ (e.g. after quantization) and need to be restored later.
+* ``"clean"`` -- simply clears weights without saving; reloads from the
+ original HuggingFace model checkpoint files. Zero extra disk writes.
+
+Usage patterns
+--------------
+
+**Compressor (offload mode)**::
+
+ offloader = OffloadManager(mode="offload")
+ offloader.offload(model, "model.layers.0") # save + clear one module
+ offloader.offload(model, block_names, clear_memory=True) # save + clear many + gc
+ offloader.reload(model, "model.layers.0") # load back + auto-clean temp file
+
+**AutoScheme (clean mode with hooks)**::
+
+ offloader = OffloadManager(mode="clean",
+ model_dir="/path/to/model")
+ offloader.add_offload_hooks(model, block_names) # clear all + register hooks
+ # ... forward passes transparently reload on demand ...
+ offloader.remove_offload_hooks(model, block_names) # remove hooks + restore all
+"""
+
+import gc
+import json
+import os
+import shutil
+import tempfile
+from collections import defaultdict
+from functools import partial
+from typing import Any, Optional, Union
+
+import torch
+
+from auto_round.logger import logger
+from auto_round.utils.model import get_module
+
+__all__ = ["OffloadManager"]
+
+# =====================================================================
+# Low-level helpers
+# =====================================================================
+
+
+def _load_state_dict_into_module(state_dict: dict, module: torch.nn.Module) -> None:
+ """Assign every key in *state_dict* to the corresponding sub-module.
+
+ Handles cleared parameters (empty tensors) and wrapper objects that store
+ the original layer in an ``orig_layer`` attribute.
+ """
+ for name, param in state_dict.items():
+ parts = name.split(".")
+ target = module
+ try:
+ for part in parts[:-1]:
+ target = getattr(target, part)
+ except AttributeError:
+ continue
+ if hasattr(target, "orig_layer"):
+ target = target.orig_layer
+ param_name = parts[-1]
+ if hasattr(target, param_name):
+ old_param = getattr(target, param_name)
+ if isinstance(old_param, torch.nn.Parameter):
+ param = param.to(dtype=old_param.dtype, device=old_param.device)
+ setattr(target, param_name, torch.nn.Parameter(param, requires_grad=old_param.requires_grad))
+ else:
+ setattr(target, param_name, param)
+ del state_dict
+
+
+def _clear_module_weights(module: torch.nn.Module, cache_numel: bool = False) -> None:
+ """Replace a single module's weight/bias with empty tensors.
+
+ Args:
+ module: The leaf module to clear.
+ cache_numel: If *True*, store ``_cached_weight_numel`` and
+ ``_cached_weight_shape`` before clearing.
+ """
+ if module is None:
+ return
+ if hasattr(module, "orig_layer"):
+ return
+
+ with torch.no_grad():
+ for name, param in list(module.named_parameters(recurse=False)):
+ if param is None or param.numel() == 0:
+ continue
+ if cache_numel and name == "weight":
+ module._cached_weight_numel = param.numel()
+ module._cached_weight_shape = tuple(param.shape)
+ setattr(
+ module,
+ name,
+ torch.nn.Parameter(torch.empty(0, dtype=param.dtype, device="cpu"), requires_grad=param.requires_grad),
+ )
+ for name, buf in list(module.named_buffers(recurse=False)):
+ if buf is None or buf.numel() == 0:
+ continue
+ module.register_buffer(name, torch.empty(0, dtype=buf.dtype, device="cpu"))
+
+
+# =====================================================================
+# Checkpoint file helpers
+# =====================================================================
+
+
+def _resolve_model_dir(model_dir: str) -> str:
+ """Resolve a model name/path to a local directory containing weight files."""
+ if os.path.isdir(model_dir):
+ return model_dir
+ try:
+ from huggingface_hub import snapshot_download
+
+ return snapshot_download(model_dir, local_files_only=True)
+ except Exception:
+ return model_dir
+
+
+def _build_weight_map(model_dir: str) -> dict[str, str]:
+ """Build ``{tensor_name: shard_filename}`` from the model directory."""
+ index_path = os.path.join(model_dir, "model.safetensors.index.json")
+ if os.path.exists(index_path):
+ with open(index_path) as f:
+ return json.load(f)["weight_map"]
+
+ single_path = os.path.join(model_dir, "model.safetensors")
+ if os.path.exists(single_path):
+ from safetensors import safe_open
+
+ with safe_open(single_path, framework="pt") as f:
+ return {k: "model.safetensors" for k in f.keys()}
+
+ bin_index_path = os.path.join(model_dir, "pytorch_model.bin.index.json")
+ if os.path.exists(bin_index_path):
+ with open(bin_index_path) as f:
+ return json.load(f)["weight_map"]
+
+ single_bin = os.path.join(model_dir, "pytorch_model.bin")
+ if os.path.exists(single_bin):
+ state_dict = torch.load(single_bin, map_location="cpu")
+ return {k: "pytorch_model.bin" for k in state_dict.keys()}
+
+ raise FileNotFoundError(
+ f"Could not find model weight files in {model_dir}. "
+ "Expected model.safetensors or pytorch_model.bin (with optional index.json)."
+ )
+
+
+def load_block_from_model_files(model_dir: str, block_name: str, block: torch.nn.Module) -> None:
+ """Reload a module's weights directly from the original model checkpoint.
+
+ Selectively loads only tensors belonging to *block_name* without loading
+ the entire model into memory.
+
+ Args:
+ model_dir: Path to the model directory (local or HuggingFace hub name).
+ block_name: The module name prefix (e.g. ``"model.layers.0"``).
+ block: The ``nn.Module`` to load weights into.
+ """
+ model_dir = _resolve_model_dir(model_dir)
+ weight_map = _build_weight_map(model_dir)
+
+ prefix = block_name + "."
+ matching = {k: v for k, v in weight_map.items() if k.startswith(prefix)}
+ if not matching:
+ logger.warning(f"No tensors found for {block_name} in {model_dir}")
+ return
+
+ shard_to_tensors: dict[str, list[str]] = defaultdict(list)
+ for tensor_name, shard_file in matching.items():
+ shard_to_tensors[shard_file].append(tensor_name)
+
+ state_dict = {}
+ for shard_file, tensor_names in shard_to_tensors.items():
+ shard_path = os.path.join(model_dir, shard_file)
+ if shard_file.endswith(".safetensors"):
+ from safetensors import safe_open
+
+ with safe_open(shard_path, framework="pt", device="cpu") as f:
+ for name in tensor_names:
+ state_dict[name[len(prefix) :]] = f.get_tensor(name)
+ else:
+ full_state = torch.load(shard_path, map_location="cpu")
+ for name in tensor_names:
+ if name in full_state:
+ state_dict[name[len(prefix) :]] = full_state[name]
+ del full_state
+
+ _load_state_dict_into_module(state_dict, block)
+
+
+# =====================================================================
+# OffloadManager -- the offload manager class
+# =====================================================================
+
+
+class OffloadManager:
+ """Module offloader for CPU RAM optimization.
+
+ Provides two core operations:
+
+ * `offload` -- save/clear a named module's weights to free CPU RAM.
+ * `reload` -- restore the weights back into the module.
+
+ Works at any granularity (block, layer, individual module) and supports
+ two operating modes:
+
+ * ``"offload"`` -- saves ``state_dict`` to a temp directory on disk
+ before clearing. Supports ``overwrite=True`` for re-saving modified
+ (e.g. quantized) weights.
+ * ``"clean"`` -- simply clears weights; reloads from the original
+ HuggingFace model checkpoint files. Zero extra disk writes but
+ cannot persist modifications.
+
+ The manager tracks its own state and supports automatic cleanup via
+ context manager (``with`` statement) or explicit `cleanup`.
+
+ Parameters
+ ----------
+ enabled : bool
+ When *False*, every public method is a no-op.
+ mode : str
+ ``"offload"`` or ``"clean"``.
+ model_dir : str, optional
+ Path to the model checkpoint directory. Required for ``"clean"``
+ mode; optional for ``"offload"``.
+ offload_dir_prefix : str
+ Prefix for the temp directory name (``"offload"`` mode only).
+ cache_numel : bool
+ If *True*, cache original ``weight.numel()`` before clearing so
+ downstream code can still determine the original size.
+ """
+
+ def __init__(
+ self,
+ enabled: bool = True,
+ mode: str = "offload",
+ model_dir: Optional[str] = None,
+ offload_dir_prefix: str = "ar_offload",
+ cache_numel: bool = False,
+ ):
+ self.enabled = enabled
+ self.mode = mode
+ self.model_dir = model_dir
+ self.cache_numel = cache_numel
+ self._prefix = offload_dir_prefix
+
+ # Disk state (offload mode)
+ self._tempdir: Optional[str] = None
+ self._saved: dict[str, dict] = {} # name -> {"save_path": str}
+
+ # Hook state (for add_offload_hooks/remove_offload_hooks transparent offloading)
+ self._hook_handles: list = []
+ self._model_ref: Optional[torch.nn.Module] = None
+ self._module_names: list[str] = []
+ self._last_loaded: Optional[str] = None
+
+ # Ensure-style state (for wrapping loops)
+ self._current_loaded: Optional[str] = None
+
+ # ------------------------------------------------------------------
+ # Context manager
+ # ------------------------------------------------------------------
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.cleanup()
+ return False
+
+ def __del__(self):
+ try:
+ self.cleanup(_skip_reload=True)
+ except Exception:
+ pass
+
+ # ------------------------------------------------------------------
+ # Core API
+ # ------------------------------------------------------------------
+
+ def offload(
+ self,
+ model: torch.nn.Module,
+ names: Union[str, list[str], list[list[str]]],
+ *,
+ skip_if_saved: bool = False,
+ overwrite: bool = False,
+ clear_memory: bool = False,
+ device_list=None,
+ ) -> float:
+ """Offload one or more named modules' weights to free CPU RAM.
+
+ Accepts a single module name, a flat list, or a nested list of names.
+ For ``"offload"`` mode: saves state_dict to disk, then clears weights.
+ For ``"clean"`` mode: simply clears weights (will reload from
+ model files later).
+
+ Args:
+ model: The root model containing the module(s).
+ names: Module name(s) -- a single string, a list of strings,
+ or a nested list (e.g. ``all_blocks``).
+ skip_if_saved: If *True* and a name was already saved, skip it.
+ overwrite: If *True*, discard any previous save and re-save the
+ current (possibly modified) weights before clearing. Useful
+ after quantization when updated weights must be persisted.
+ Only meaningful for ``"offload"`` mode.
+ clear_memory: If *True*, call ``clear_memory()`` after offloading
+ all modules and log the total freed size.
+ device_list: Device list passed to ``clear_memory`` (only used
+ when *clear_memory* is *True*).
+
+ Returns:
+ Total offloaded size in GB (non-zero only when *clear_memory* is True).
+ """
+ if not self.enabled:
+ return 0.0
+ if isinstance(names, str):
+ self._offload(model, names, skip_if_saved=skip_if_saved, overwrite=overwrite)
+ return 0.0
+ flat_names = self._flatten_names(names)
+ if clear_memory:
+ logger.info("offloading module weights...")
+ total_gb = 0.0
+ for name in flat_names:
+ if skip_if_saved and self.has(name):
+ continue
+ if clear_memory:
+ module = get_module(model, name)
+ if module is not None:
+ total_gb += self.estimate_module_size_gb(module)
+ self._offload(model, name, skip_if_saved=skip_if_saved, overwrite=overwrite)
+ if clear_memory:
+ from auto_round.utils import clear_memory as _clear_memory
+
+ _clear_memory(device_list=device_list)
+ logger.info(f"offload done, freed {total_gb:.2f} GB")
+ return total_gb
+
+ def _offload(
+ self, model: torch.nn.Module, name: str, *, skip_if_saved: bool = False, overwrite: bool = False
+ ) -> None:
+ """Offload a single named module (internal helper)."""
+ self._model_ref = model
+ module = get_module(model, name)
+ if module is None:
+ return
+ if self.mode == "offload":
+ if overwrite:
+ old_meta = self._saved.pop(name, None)
+ if old_meta:
+ old_path = old_meta.get("save_path")
+ if old_path and os.path.exists(old_path):
+ try:
+ os.remove(old_path)
+ except Exception as e:
+ logger.warning(f"OffloadManager: could not remove {old_path}: {e}")
+ elif skip_if_saved and name in self._saved:
+ return
+ self._save_to_disk(name, module)
+ self._clear(module)
+
+ def reload(self, model: torch.nn.Module, names: Union[str, list[str], None] = None) -> None:
+ """Reload previously offloaded module(s).
+
+ For ``"offload"`` mode: loads from the temp directory, then
+ removes the temp file. When all offloaded modules have been
+ reloaded the temp directory is automatically deleted.
+ For ``"clean"`` mode: loads from original model files.
+
+ Args:
+ model: The root model containing the module(s).
+ names: A single module name, a list of names, or *None*
+ to reload all tracked modules.
+ """
+ if not self.enabled:
+ return
+ if names is None:
+ if self.mode == "offload":
+ names = list(self._saved.keys())
+ else:
+ names = list(self._module_names)
+ if isinstance(names, str):
+ self._reload(model, names)
+ return
+ for name in names:
+ self._reload(model, name)
+
+ def _reload(self, model: torch.nn.Module, name: str) -> None:
+ """Reload a single named module (internal helper)."""
+ module = get_module(model, name)
+ if module is None:
+ return
+ if self.mode == "offload":
+ self._load_from_disk(name, module)
+ self._remove_saved_entry(name)
+ else:
+ if self.model_dir is None:
+ logger.warning("OffloadManager: model_dir is required for clean mode")
+ return
+ load_block_from_model_files(self.model_dir, name, module)
+
+ # ------------------------------------------------------------------
+ # Hook-based transparent offloading
+ # ------------------------------------------------------------------
+
+ def add_offload_hooks(self, model: torch.nn.Module, names: list[str]) -> None:
+ """Clear all named modules and register pre-forward hooks for
+ transparent reload-on-demand.
+
+ Args:
+ model: The root model.
+ names: List of module names to manage.
+ """
+ if not self.enabled:
+ return
+ self._model_ref = model
+ self._module_names = list(names)
+ # Register hooks
+ for name in names:
+ module = get_module(model, name)
+ if module is None:
+ continue
+ handle = module.register_forward_pre_hook(partial(self._pre_forward_hook, name=name))
+ self._hook_handles.append(handle)
+ # Clear all
+ logger.info("clearing module weights to free RAM...")
+ for name in names:
+ self._offload(model, name, skip_if_saved=True)
+ gc.collect()
+ from auto_round.utils import clear_memory
+
+ clear_memory()
+ logger.info("module weights cleared")
+
+ def remove_offload_hooks(self, model: torch.nn.Module, names: Optional[list[str]] = None) -> None:
+ """Remove hooks and reload all managed modules.
+
+ Args:
+ model: The root model.
+ names: Module names to reload. If None, uses the names from
+ `add_offload_hooks`.
+ """
+ self._remove_hooks()
+ if names is None:
+ names = self._module_names
+ self.reload(model, names)
+ self._model_ref = None
+
+ def _pre_forward_hook(self, module: torch.nn.Module, args, *, name: str) -> None:
+ """Pre-forward hook: reload this module if cleared, clear previous."""
+ if not self._needs_loading(module):
+ return
+ # Clear previous to keep only one module loaded at a time
+ if self._last_loaded is not None and self._last_loaded != name:
+ prev = get_module(self._model_ref, self._last_loaded)
+ if prev is not None:
+ self._clear(prev)
+ self.reload(self._model_ref, name)
+ self._last_loaded = name
+
+ def _remove_hooks(self) -> None:
+ for handle in self._hook_handles:
+ handle.remove()
+ self._hook_handles.clear()
+ self._last_loaded = None
+
+ # ------------------------------------------------------------------
+ # Ensure-style API (for wrapping loops)
+ # ------------------------------------------------------------------
+
+ def ensure_loaded(self, model: torch.nn.Module, layer_name: str) -> None:
+ """Ensure the module containing *layer_name* is loaded.
+
+ When the loop moves to a layer in a different module, the previous
+ module is automatically cleared. Call `flush_loaded` after
+ the loop to release the last module.
+
+ Args:
+ model: The root model.
+ layer_name: Full layer name (e.g. ``"model.layers.0.attn.q"``).
+ """
+ if not self.enabled or not self._module_names:
+ return
+ target = None
+ for mn in self._module_names:
+ if layer_name.startswith(mn + "."):
+ target = mn
+ break
+ if target is None:
+ return
+ if target == self._current_loaded:
+ return
+ # Clear previous
+ if self._current_loaded is not None:
+ module = get_module(model, self._current_loaded)
+ if module is not None:
+ self._clear(module)
+ # Load new
+ self.reload(model, target)
+ self._current_loaded = target
+
+ def flush_loaded(self, model: torch.nn.Module) -> None:
+ """Clear the last module loaded by `ensure_loaded`."""
+ if self._current_loaded is not None:
+ module = get_module(model, self._current_loaded)
+ if module is not None:
+ self._clear(module)
+ self._current_loaded = None
+
+ # ------------------------------------------------------------------
+ # State queries
+ # ------------------------------------------------------------------
+
+ def has(self, name: str) -> bool:
+ """Return *True* if *name* has been offloaded."""
+ if self.mode == "offload":
+ return name in self._saved
+ return False
+
+ def reset(self) -> None:
+ """Clear tracking state (temp directory is kept)."""
+ self._saved = {}
+ self._current_loaded = None
+ self._last_loaded = None
+
+ def cleanup(self, _skip_reload: bool = False) -> None:
+ """Reload remaining modules and remove temp files.
+
+ Normally this is called automatically -- ``reload`` deletes each
+ temp file as it goes and removes the temp directory when done.
+ Explicit calls are usually unnecessary; the context-manager exit
+ and ``__del__`` use this as a fallback.
+ """
+ self._remove_hooks()
+ # Auto-reload offloaded modules if we still have a model reference
+ if not _skip_reload and self._model_ref is not None and self._saved:
+ try:
+ self.reload(self._model_ref)
+ except Exception as e:
+ logger.warning(f"OffloadManager: auto-reload during cleanup failed: {e}")
+ self._cleanup_tempdir()
+ self._saved = {}
+ self._current_loaded = None
+ self._module_names = []
+ self._model_ref = None
+
+ # ------------------------------------------------------------------
+ # Static estimation helpers
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def estimate_tensor_size_gb(tensor: Any) -> float:
+ """Estimate size of a tensor (or nested container) in GB."""
+ if tensor is None:
+ return 0.0
+ if isinstance(tensor, torch.Tensor):
+ return tensor.numel() * tensor.element_size() / (1024**3)
+ if isinstance(tensor, list):
+ return sum(OffloadManager.estimate_tensor_size_gb(t) for t in tensor)
+ if isinstance(tensor, dict):
+ return sum(OffloadManager.estimate_tensor_size_gb(v) for v in tensor.values())
+ return 0.0
+
+ @staticmethod
+ def estimate_inputs_size_gb(all_inputs: dict) -> float:
+ """Estimate total size of calibration inputs in GB."""
+ return sum(OffloadManager.estimate_tensor_size_gb(v) for v in all_inputs.values())
+
+ @staticmethod
+ def estimate_model_size_gb(model: torch.nn.Module) -> float:
+ """Estimate model weights size in GB."""
+ return sum(p.numel() * p.element_size() / (1024**3) for p in model.parameters() if p.numel() > 0)
+
+ @staticmethod
+ def estimate_module_size_gb(module: torch.nn.Module) -> float:
+ """Estimate a module's parameter size in GB."""
+ return sum(p.numel() * p.element_size() / (1024**3) for p in module.parameters() if p.numel() > 0)
+
+ # ------------------------------------------------------------------
+ # Internal: disk operations (offload mode)
+ # ------------------------------------------------------------------
+
+ def _ensure_dir(self) -> str:
+ if self._tempdir is None:
+ self._tempdir = tempfile.mkdtemp(prefix=f"{self._prefix}_")
+ logger.info(f"OffloadManager ({self._prefix}): tempdir = {self._tempdir}")
+ return self._tempdir
+
+ def _save_to_disk(self, name: str, module: torch.nn.Module) -> None:
+ tmpdir = self._ensure_dir()
+ safe_name = name.replace(".", "_")
+ save_path = os.path.join(tmpdir, f"{safe_name}.pt")
+ try:
+ # Skip meta tensors: they contain no real data (e.g. quantized weights
+ # already flushed to disk by an immediate-saving shard writer).
+ state_dict = {k: v.cpu().contiguous() for k, v in module.state_dict().items() if v.device.type != "meta"}
+ torch.save(state_dict, save_path)
+ self._saved[name] = {"save_path": save_path}
+ del state_dict
+ except Exception as e:
+ logger.warning(f"OffloadManager: failed to save {name}: {e}")
+
+ def _load_from_disk(self, name: str, module: torch.nn.Module) -> None:
+ metadata = self._saved.get(name)
+ if not metadata:
+ return
+ save_path = metadata["save_path"]
+ if not os.path.exists(save_path):
+ logger.warning(f"OffloadManager: file not found {save_path}")
+ return
+ try:
+ state_dict = torch.load(save_path, map_location="cpu")
+ _load_state_dict_into_module(state_dict, module)
+ except Exception as e:
+ logger.warning(f"OffloadManager: failed to load {name}: {e}")
+
+ def _remove_saved_entry(self, name: str) -> None:
+ """Remove a single saved entry and its temp file; clean tempdir if empty."""
+ meta = self._saved.pop(name, None)
+ if meta:
+ path = meta.get("save_path")
+ if path and os.path.exists(path):
+ try:
+ os.remove(path)
+ except Exception as e:
+ logger.warning(f"OffloadManager: could not remove {path}: {e}")
+ if not self._saved:
+ self._cleanup_tempdir()
+
+ def _cleanup_tempdir(self) -> None:
+ """Remove the temp directory if it exists."""
+ if self._tempdir and os.path.isdir(self._tempdir):
+ try:
+ shutil.rmtree(self._tempdir)
+ except Exception as e:
+ logger.warning(f"OffloadManager: cleanup failed for {self._tempdir}: {e}")
+ self._tempdir = None
+
+ # ------------------------------------------------------------------
+ # Internal: clearing
+ # ------------------------------------------------------------------
+
+ def _clear(self, module: torch.nn.Module) -> None:
+ """Clear all weight/bias tensors in *module* and its sub-modules."""
+ for submodule in module.modules():
+ _clear_module_weights(submodule, cache_numel=self.cache_numel)
+
+ @staticmethod
+ def _needs_loading(module: torch.nn.Module) -> bool:
+ """Return *True* if any parameter in *module* has been cleared."""
+ for submodule in module.modules():
+ if hasattr(submodule, "orig_layer"):
+ submodule = submodule.orig_layer
+ for param in submodule.parameters(recurse=False):
+ if param is not None and param.numel() == 0:
+ return True
+ return False
+
+ @staticmethod
+ def _flatten_names(names: Union[list[str], list[list[str]]]) -> list[str]:
+ """Flatten a potentially nested list of names."""
+ flat = []
+ for item in names:
+ if isinstance(item, list):
+ flat.extend(item)
+ else:
+ flat.append(item)
+ return flat
diff --git a/test/test_cpu/core/test_low_cpu_mem_options.py b/test/test_cpu/core/test_low_cpu_mem_options.py
new file mode 100644
index 000000000..543a9a515
--- /dev/null
+++ b/test/test_cpu/core/test_low_cpu_mem_options.py
@@ -0,0 +1,95 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Unit tests for low_cpu_mem_usage option:
+When enabled, block weights are offloaded to disk upfront and loaded on demand
+during quantization, significantly reducing peak CPU RAM usage.
+"""
+
+import torch
+
+from auto_round import AutoRound
+from auto_round.utils import offload as offload_module
+
+
+class TestLowCpuMemUsage:
+ """Tests for low_cpu_mem_usage block offloading."""
+
+ def test_option_stored_correctly(self, tiny_opt_model_path):
+ """Test that low_cpu_mem_usage option is stored correctly."""
+ autoround = AutoRound(
+ tiny_opt_model_path,
+ bits=4,
+ low_cpu_mem_usage=True,
+ iters=0,
+ disable_opt_rtn=True,
+ nsamples=1,
+ seqlen=32,
+ )
+ assert autoround.low_cpu_mem_usage is True
+
+ autoround2 = AutoRound(
+ tiny_opt_model_path,
+ bits=4,
+ low_cpu_mem_usage=False,
+ iters=0,
+ disable_opt_rtn=True,
+ nsamples=1,
+ seqlen=32,
+ )
+ assert autoround2.low_cpu_mem_usage is False
+
+ def test_offloader_disabled_when_low_cpu_mem_false(self, tiny_opt_model_path):
+ """Test that the offloader is disabled when low_cpu_mem_usage=False."""
+ autoround = AutoRound(
+ tiny_opt_model_path,
+ bits=4,
+ low_cpu_mem_usage=False,
+ iters=0,
+ disable_opt_rtn=True,
+ nsamples=1,
+ seqlen=32,
+ )
+ assert autoround.low_cpu_mem_usage is False
+ # offload with list should be a no-op when offloader is disabled
+ autoround._offloader.offload(
+ autoround.model, [["model.layers.0"]], clear_memory=True, device_list=autoround.device_list
+ )
+ assert autoround._offloader._saved == {}
+
+ def test_offload_all_records_blocks(self, tiny_opt_model_path, tmp_path, monkeypatch):
+ """Test that offload(list) records offloaded blocks when enabled."""
+ autoround = AutoRound(
+ tiny_opt_model_path,
+ bits=4,
+ low_cpu_mem_usage=True,
+ iters=0,
+ disable_opt_rtn=True,
+ nsamples=1,
+ seqlen=32,
+ )
+
+ dummy_block = torch.nn.Linear(4, 4)
+ # Monkeypatch get_module used by offload in offload.py
+
+ monkeypatch.setattr(offload_module, "get_module", lambda _model, _name: dummy_block)
+ monkeypatch.setattr(torch, "save", lambda *args, **kwargs: None)
+
+ # Force the offloader to think it has a tempdir already
+ autoround._offloader._tempdir = str(tmp_path)
+ autoround._offloader.offload(
+ autoround.model, [["model.layers.0"]], clear_memory=True, device_list=autoround.device_list
+ )
+ assert autoround._offloader.has("model.layers.0")
diff --git a/test/test_cpu/schemes/test_auto_scheme_low_cpu_mem.py b/test/test_cpu/schemes/test_auto_scheme_low_cpu_mem.py
new file mode 100644
index 000000000..4776db7c0
--- /dev/null
+++ b/test/test_cpu/schemes/test_auto_scheme_low_cpu_mem.py
@@ -0,0 +1,361 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Unit tests for AutoScheme CPU RAM optimization (low_cpu_mem_usage option).
+
+This tests the disk offload mechanism for AutoScheme that reduces CPU RAM usage
+by offloading block weights to disk during gradient computation.
+"""
+
+import shutil
+
+import pytest
+import torch
+
+from auto_round import AutoRound, AutoScheme
+from auto_round.auto_scheme.utils import compute_layer_bits
+from auto_round.utils import get_block_names, get_module
+from auto_round.utils.offload import OffloadManager, _clear_module_weights
+
+
+class TestOffloadManager:
+ """Tests for OffloadManager class."""
+
+ def test_context_init_disabled(self):
+ """Test that manager is properly initialized when disabled."""
+ mgr = OffloadManager(enabled=False)
+ assert mgr.enabled is False
+
+ def test_context_init_enabled(self):
+ """Test that manager is properly initialized when enabled."""
+ mgr = OffloadManager(enabled=True, mode="clean", model_dir="/tmp/fake")
+ assert mgr.enabled is True
+ assert mgr.model_dir == "/tmp/fake"
+ assert mgr.mode == "clean"
+
+ def test_clear_clears_weights(self):
+ """Test that _clear clears weights from a module."""
+ mgr = OffloadManager(enabled=True, mode="clean")
+ module = torch.nn.Linear(4, 4)
+ mgr._clear(module)
+
+ # Verify weight was cleared
+ assert module.weight.numel() == 0
+
+ def test_add_hooks_noop_when_disabled(self):
+ """Test that add_offload_hooks does nothing when disabled."""
+ mgr = OffloadManager(enabled=False)
+ model = torch.nn.Sequential(torch.nn.Linear(4, 4))
+ original_numel = model[0].weight.numel()
+ mgr.add_offload_hooks(model, ["0"])
+ assert model[0].weight.numel() == original_numel # unchanged
+
+ def test_ensure_loaded_noop_when_disabled(self):
+ """Test that ensure_loaded does nothing when disabled."""
+ mgr = OffloadManager(enabled=False)
+ model = torch.nn.Sequential(torch.nn.Linear(4, 4))
+ mgr.ensure_loaded(model, "0.weight") # should not raise
+
+ def test_cleanup_resets_state(self):
+ """Test that cleanup resets internal state."""
+ mgr = OffloadManager(enabled=True, mode="clean")
+ module = torch.nn.Linear(4, 4)
+ mgr._clear(module)
+ mgr._module_names = ["test"]
+ assert len(mgr._module_names) == 1
+
+ mgr.cleanup()
+ assert len(mgr._module_names) == 0
+
+ def test_reset_clears_tracking(self):
+ """Test that reset clears tracking state."""
+ mgr = OffloadManager(enabled=True, mode="offload")
+ mgr._saved["b"] = {"save_path": "/tmp/x"}
+ assert len(mgr._saved) == 1
+
+ mgr.reset()
+ assert len(mgr._saved) == 0
+
+ def test_model_dir_property(self):
+ """Test model_dir getter/setter."""
+ mgr = OffloadManager(enabled=True, mode="clean", model_dir="/path/a")
+ assert mgr.model_dir == "/path/a"
+ mgr.model_dir = "/path/b"
+ assert mgr.model_dir == "/path/b"
+
+ def test_context_manager(self):
+ """Test OffloadManager as context manager."""
+ with OffloadManager(enabled=True, mode="offload") as mgr:
+ assert mgr.enabled is True
+ # After exit, state should be cleaned up
+ assert mgr._saved == {}
+
+ def test_reload_auto_cleans_temp_file(self):
+ """Test that reload() removes the temp file and auto-cleans tempdir."""
+ model = torch.nn.Sequential(torch.nn.Linear(4, 4))
+ original_weight = model[0].weight.data.clone()
+ mgr = OffloadManager(enabled=True, mode="offload")
+ mgr.offload(model, "0")
+ assert model[0].weight.numel() == 0
+ assert mgr._tempdir is not None
+ tempdir = mgr._tempdir
+
+ # reload restores weights and auto-cleans temp file + tempdir
+ mgr.reload(model, "0")
+ assert model[0].weight.numel() > 0
+ assert torch.allclose(model[0].weight.data, original_weight)
+ assert len(mgr._saved) == 0
+ assert mgr._tempdir is None # auto-cleaned
+
+ def test_reload_all_auto_cleans(self):
+ """Test that reload() with no names auto-cleans all temp files."""
+ model = torch.nn.Sequential(
+ torch.nn.Linear(4, 4),
+ torch.nn.Linear(4, 4),
+ )
+ mgr = OffloadManager(enabled=True, mode="offload")
+ mgr.offload(model, "0")
+ mgr.offload(model, "1")
+ assert len(mgr._saved) == 2
+
+ mgr.reload(model)
+ assert len(mgr._saved) == 0
+ assert mgr._tempdir is None # auto-cleaned
+
+ def test_context_manager_auto_reloads(self):
+ """Test that context manager exit auto-reloads offloaded modules."""
+ model = torch.nn.Sequential(torch.nn.Linear(4, 4))
+ original_weight = model[0].weight.data.clone()
+ with OffloadManager(enabled=True, mode="offload") as mgr:
+ mgr.offload(model, "0")
+ assert model[0].weight.numel() == 0
+ # After exiting context, weights should be restored
+ assert model[0].weight.numel() > 0
+ assert torch.allclose(model[0].weight.data, original_weight)
+
+
+class TestClearModuleWeights:
+ """Tests for _clear_module_weights helper function."""
+
+ def test_clear_linear_weights(self):
+ """Test clearing weights from a Linear module."""
+ module = torch.nn.Linear(16, 8)
+ _clear_module_weights(module)
+
+ assert module.weight.numel() == 0
+ assert module.bias.numel() == 0
+
+ def test_clear_caches_numel(self):
+ """Test that clearing weights caches the original numel."""
+ module = torch.nn.Linear(32, 16)
+ expected = 32 * 16
+ _clear_module_weights(module, cache_numel=True)
+ assert hasattr(module, "_cached_weight_numel")
+ assert module._cached_weight_numel == expected
+
+ def test_compute_layer_bits_with_empty_weight(self):
+ """Test compute_layer_bits works after weight is cleared."""
+ layer = torch.nn.Linear(64, 128)
+ layer.bits = 4
+ layer.group_size = 32
+ layer.data_type = "int"
+ layer.sym = True
+ layer.super_group_size = None
+ layer.super_bits = None
+
+ bits_before, _ = compute_layer_bits(layer, False)
+ _clear_module_weights(layer, cache_numel=True)
+ bits_after, _ = compute_layer_bits(layer, False)
+ assert bits_before == bits_after
+
+ def test_clear_module_none(self):
+ """Test clearing weights with None module doesn't crash."""
+ _clear_module_weights(None) # Should not raise
+
+ def test_clear_module_no_weights(self):
+ """Test clearing module without weights doesn't crash."""
+ module = torch.nn.ReLU()
+ _clear_module_weights(module) # Should not raise
+
+
+class TestAutoSchemeDataclassLowCpuMem:
+ """Tests for low_cpu_mem_usage parameter in AutoScheme dataclass."""
+
+ def test_auto_scheme_default_low_cpu_mem_usage(self):
+ """Test that low_cpu_mem_usage defaults to True."""
+ scheme = AutoScheme(avg_bits=4, options="W4A16")
+ assert scheme.low_cpu_mem_usage is True
+
+ def test_auto_scheme_low_cpu_mem_usage_enabled(self):
+ """Test that low_cpu_mem_usage can be enabled."""
+ scheme = AutoScheme(avg_bits=4, options="W4A16", low_cpu_mem_usage=True)
+ assert scheme.low_cpu_mem_usage is True
+
+ def test_auto_scheme_low_cpu_mem_usage_with_low_gpu_mem_usage(self):
+ """Test that both low_cpu_mem_usage and low_gpu_mem_usage can be set."""
+ scheme = AutoScheme(
+ avg_bits=4,
+ options="W4A16",
+ low_cpu_mem_usage=True,
+ low_gpu_mem_usage=True,
+ )
+ assert scheme.low_cpu_mem_usage is True
+ assert scheme.low_gpu_mem_usage is True
+
+
+class TestAutoSchemeIntegration:
+ """Integration tests for AutoScheme with low_cpu_mem_usage."""
+
+ save_dir = "./saved"
+
+ @pytest.fixture(autouse=True, scope="class")
+ def setup_and_teardown_class(self):
+ yield
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_auto_scheme_with_low_cpu_mem_disabled(self, tiny_opt_model_path):
+ """Test AutoScheme works normally with low_cpu_mem_usage disabled."""
+ model_name = tiny_opt_model_path
+ scheme = AutoScheme(
+ avg_bits=4,
+ options="W2A16,W4A16",
+ nsamples=1,
+ ignore_scale_zp_bits=True,
+ low_cpu_mem_usage=False,
+ low_gpu_mem_usage=True,
+ )
+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)
+ _, layer_config = ar.quantize()
+ assert layer_config is not None
+ assert len(layer_config) > 0
+
+ def test_auto_scheme_with_low_cpu_mem_enabled(self, tiny_opt_model_path):
+ """Test AutoScheme works with low_cpu_mem_usage enabled."""
+ model_name = tiny_opt_model_path
+ scheme = AutoScheme(
+ avg_bits=4,
+ options="W2A16,W4A16",
+ nsamples=1,
+ ignore_scale_zp_bits=True,
+ low_cpu_mem_usage=True,
+ low_gpu_mem_usage=True,
+ )
+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)
+ _, layer_config = ar.quantize()
+ assert layer_config is not None
+ assert len(layer_config) > 0
+
+ def test_auto_scheme_low_cpu_mem_results_consistent(self, tiny_opt_model_path):
+ """Test that results are consistent with and without low_cpu_mem_usage."""
+ model_name = tiny_opt_model_path
+
+ # Without low_cpu_mem_usage
+ scheme1 = AutoScheme(
+ avg_bits=4,
+ options="W4A16",
+ nsamples=1,
+ ignore_scale_zp_bits=True,
+ low_cpu_mem_usage=False,
+ low_gpu_mem_usage=True,
+ )
+ ar1 = AutoRound(model=model_name, scheme=scheme1, iters=0, nsamples=1, seed=42)
+ _, layer_config1 = ar1.quantize()
+
+ # With low_cpu_mem_usage
+ scheme2 = AutoScheme(
+ avg_bits=4,
+ options="W4A16",
+ nsamples=1,
+ ignore_scale_zp_bits=True,
+ low_cpu_mem_usage=True,
+ low_gpu_mem_usage=True,
+ )
+ ar2 = AutoRound(model=model_name, scheme=scheme2, iters=0, nsamples=1, seed=42)
+ _, layer_config2 = ar2.quantize()
+
+ # Layer configs should have same keys
+ assert set(layer_config1.keys()) == set(layer_config2.keys())
+
+
+class TestOffloadManagerWithModel:
+ """Tests for OffloadManager with actual model blocks."""
+
+ def test_clear_and_load_model_block(self, tiny_opt_model_path):
+ """Test clearing and reloading an actual model block from checkpoint files."""
+ from transformers import AutoModelForCausalLM
+
+ from auto_round.utils.offload import load_block_from_model_files
+
+ model = AutoModelForCausalLM.from_pretrained(tiny_opt_model_path, torch_dtype=torch.float32)
+ block_names = get_block_names(model)[0]
+
+ if len(block_names) == 0:
+ pytest.skip("Model has no blocks")
+
+ block_name = block_names[0]
+ block = get_module(model, block_name)
+
+ mgr = OffloadManager(enabled=True, mode="clean", model_dir=tiny_opt_model_path, cache_numel=True)
+
+ # Get original weights
+ original_params = sum(p.numel() for p in block.parameters())
+ original_weight = next(block.parameters()).data.clone()
+
+ # Clear
+ mgr._clear(block)
+ current_params = sum(p.numel() for p in block.parameters())
+ assert current_params < original_params
+
+ # Load back from model files
+ load_block_from_model_files(tiny_opt_model_path, block_name, block)
+ restored_params = sum(p.numel() for p in block.parameters())
+ assert restored_params == original_params
+
+ # Verify values match
+ restored_weight = next(block.parameters()).data
+ assert torch.allclose(restored_weight, original_weight)
+
+ def test_attach_and_detach(self, tiny_opt_model_path):
+ """Test clearing all model blocks via attach and loading them back via detach."""
+ from transformers import AutoModelForCausalLM
+
+ model = AutoModelForCausalLM.from_pretrained(tiny_opt_model_path, torch_dtype=torch.float32)
+ block_names = get_block_names(model)[0]
+
+ if len(block_names) == 0:
+ pytest.skip("Model has no blocks")
+
+ mgr = OffloadManager(enabled=True, mode="clean", model_dir=tiny_opt_model_path, cache_numel=True)
+
+ # Save original param counts
+ original_counts = {}
+ for bn in block_names:
+ blk = get_module(model, bn)
+ original_counts[bn] = sum(p.numel() for p in blk.parameters())
+
+ # add_offload_hooks clears all blocks and registers hooks
+ mgr.add_offload_hooks(model, block_names)
+ for bn in block_names:
+ blk = get_module(model, bn)
+ block_params = sum(p.numel() for p in blk.parameters())
+ assert block_params == 0
+
+ # remove_offload_hooks removes hooks and reloads all blocks
+ mgr.remove_offload_hooks(model, block_names)
+ for bn in block_names:
+ blk = get_module(model, bn)
+ block_params = sum(p.numel() for p in blk.parameters())
+ assert block_params == original_counts[bn]