PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Optimize CPU RAM peak memory during quantization (#1386)
Liang Lv0d3721511da9
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex 58b2bae3c..ea99fa511 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -15,7 +15,7 @@import copyfrom dataclasses import asdictfrom functools import wraps-from typing import Iterable, Union+from typing import Iterable, Optional, Unionimport torchfrom accelerate import dispatch_model@@ -55,6 +55,8 @@ from auto_round.utils import (set_non_auto_device_map,to_device,)+from auto_round.utils.device import MemoryMonitor+from auto_round.utils.offload import OffloadManagerfrom auto_round.wrapper import WrapperLinear__all__ = ["gen_layer_config"]@@ -488,6 +490,7 @@ def get_score_for_scheme(major_device="cpu",batch_size=1,disable_opt_rtn=True,+ offload_context: Optional[OffloadManager] = None,):scores_dict = {} # Key=name,Val=[quant_total_bits, loss]for n, m in model.named_modules():@@ -506,6 +509,8 @@ def get_score_for_scheme(breakfor name in quant_layer_names:+ if offload_context is not None:+ offload_context.ensure_loaded(model, name)if name in fixed_layer_scheme.keys():continuem = get_module(model, name)@@ -547,6 +552,8 @@ def get_score_for_scheme(disable_opt_rtn=disable_opt_rtn,)set_module(model, name, new_m)+ if offload_context is not None:+ offload_context.flush_loaded(model)if low_gpu_mem_usage:dataloader = get_dataloader(tokenizer, seqlen, dataset_name=dataset, seed=42, bs=batch_size, nsamples=nsamples)@@ -658,6 +665,21 @@ def _gen_layer_config(major_device="cpu",device_list=None,):+ # Initialize memory tracking for AutoScheme+ memory_monitor = MemoryMonitor()+ memory_monitor.reset()+ memory_monitor.update_cpu()++ # Create offload context for CPU RAM optimization+ # Note: low_cpu_mem_usage only works when low_gpu_mem_usage is also enabled,+ # because it requires layer-by-layer processing+ offload_context = None+ if auto_scheme.low_cpu_mem_usage and auto_scheme.low_gpu_mem_usage:+ _model_dir = model_name+ if _model_dir is None and hasattr(model, "config"):+ _model_dir = getattr(model.config, "_name_or_path", None)+ offload_context = OffloadManager(enabled=True, mode="clean", model_dir=_model_dir, cache_numel=True)+target_bits = auto_scheme.avg_bitsmodel.eval()@@ -762,6 +784,11 @@ def _gen_layer_config(cal_imatrix(model, dataloader)logger.info("finish calculating imatrix")+ # Register hooks and clear all block weights before the scheme loop.+ # Hooks will transparently reload weights on demand during forward passes.+ if offload_context is not None:+ offload_context.add_offload_hooks(model, block_name)+pbar = tqdm(total=pbar_cnt, desc="Generating AutoScheme")for index, scheme in enumerate(schemes):apply_quant_scheme(@@ -792,7 +819,11 @@ def _gen_layer_config(major_device=major_device,batch_size=batch_size,disable_opt_rtn=auto_scheme.disable_opt_rtn,+ offload_context=offload_context,)+ # Track peak RAM after each scheme scoring+ memory_monitor.update_cpu()+new_scores = {}for share_layer in shared_layers:param_bits = 0@@ -817,10 +848,17 @@ def _gen_layer_config(options_scores.append(options_total_loss)clear_memory(device_list=device_list)+ # Remove hooks and restore original weights from disk for final bit-budget computations+ if offload_context is not None:+ offload_context.remove_offload_hooks(model, block_name)+total_params = 0for n, m in model.named_modules():if n in quant_layer_names + embedding_layers_names:- total_params += m.weight.numel()+ n_param = m.weight.numel()+ if n_param == 0 and hasattr(m, "_cached_weight_numel"):+ n_param = m._cached_weight_numel+ total_params += n_paramtarget_params_cnt = int(total_params * target_bits)sorted_indices = sorted(range(len(options_scores)), key=lambda i: options_scores[i])@@ -902,6 +940,12 @@ def _gen_layer_config(global last_grad_inputlast_grad_input = Noneclear_memory(device_list=device_list)++ # Log AutoScheme memory usage+ memory_monitor.update_cpu()+ low_cpu_str = "enabled" if auto_scheme.low_cpu_mem_usage else "disabled"+ memory_monitor.log_summary(f"AutoScheme complete (low_cpu_mem_usage={low_cpu_str})")+pbar.close()return layer_configdiff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.pyindex 8572a920a..55ec4e205 100644--- a/auto_round/auto_scheme/gen_auto_scheme.py+++ b/auto_round/auto_scheme/gen_auto_scheme.py@@ -40,6 +40,7 @@ class AutoScheme:enable_torch_compile: Optional[bool] = Nonedisable_opt_rtn: bool = Truelow_gpu_mem_usage: bool = True+ low_cpu_mem_usage: bool = Truedef __post_init__(self):if isinstance(self.options, str):diff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.pyindex c04780ff6..df1783d83 100644--- a/auto_round/auto_scheme/utils.py+++ b/auto_round/auto_scheme/utils.py@@ -105,7 +105,10 @@ def compute_avg_bits_for_scheme(# continueif not hasattr(module, "weight"):continue- total_params += module.weight.numel()+ n_param = module.weight.numel()+ if n_param == 0 and hasattr(module, "_cached_weight_numel"):+ n_param = module._cached_weight_numel+ total_params += n_paramlayer_bits, _ = compute_layer_bits(module, ignore_scale_zp_bits)total_quantized_bits += layer_bitsavg_bits = float(total_quantized_bits) / total_params@@ -133,7 +136,10 @@ def compute_avg_bits_for_model(model: torch.nn.Module, ignore_scale_zp_bits: boocontinueif not hasattr(module, "weight"):continue- total_params += module.weight.numel()+ n_param = module.weight.numel()+ if n_param == 0 and hasattr(module, "_cached_weight_numel"):+ n_param = module._cached_weight_numel+ total_params += n_paramlayer_bits, _ = compute_layer_bits(module, ignore_scale_zp_bits)total_quantized_bits += layer_bits@@ -157,6 +163,9 @@ def compute_layer_bits("""weight = layer.weightn_param = weight.numel()+ # Use cached numel when weight has been cleared to an empty tensor (low_cpu_mem_usage offload)+ if n_param == 0 and hasattr(layer, "_cached_weight_numel"):+ n_param = layer._cached_weight_numelweight_bits = getattr(layer, "bits", 16)group_size = getattr(layer, "group_size", 128)data_type = getattr(layer, "data_type", "int")diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 5472c373d..afcc489cd 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -14,6 +14,7 @@import copyimport os+import reimport sysimport timeimport traceback@@ -113,6 +114,7 @@ from auto_round.utils.device import (set_non_auto_device_map,)from auto_round.utils.distributed import setup_ddp_if_needed_+from auto_round.utils.offload import OffloadManagerfrom auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_blockSERIALIZATION_KEYS = (@@ -371,6 +373,7 @@ class BaseCompressor(object):self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPESself.scale_dtype = convert_dtype_str2torch(scale_dtype)self.low_cpu_mem_usage = low_cpu_mem_usage+ self._offloader = OffloadManager(enabled=low_cpu_mem_usage, mode="offload", offload_dir_prefix="compressor")if kwargs:logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")@@ -1259,6 +1262,10 @@ class BaseCompressor(object):m = get_module(self.model, name)m.to("cpu")shard_writer(self, m, name, False)+ # Free RAM immediately: the data is now in the shard-writer buffer+ # (and will be flushed to disk). Keeping it also in the model tree+ # causes linear RAM growth for large models.+ m.to("meta")def _immediate_pack(self, name: str):if not self.is_immediate_packing:@@ -1331,8 +1338,16 @@ class BaseCompressor(object):for handle in hook_handles:handle.remove()else:- # By default, we go with layer-wise way if no replacement happened+ # By default, we go with layer-wise way if no replacement happened.+ # In RTN mode (iters == 0), force blockwise quantization to avoid+ # full-model materialization and linear CPU RAM growth.use_blockwise_quantization = global_state.replaced_module_count > 0+ if self.iters == 0 and not use_blockwise_quantization:+ logger.info(+ "RTN mode detected (iters=0): force blockwise quantization to avoid "+ "layer-wise full-model materialization."+ )+ use_blockwise_quantization = Truetied_weights_keys = getattr(self.model, "_tied_weights_keys", [])if tied_weights_keys is None:tied_weights_keys = []@@ -1370,8 +1385,22 @@ class BaseCompressor(object):and self.is_immediate_saving):set_module(self.model, m.global_name, copy.deepcopy(m))- shard_writer(self, name=m.global_name)+ if self.is_immediate_saving:+ shard_writer(self, name=m.global_name)+ copied_m = get_module(self.model, m.global_name)+ copied_m.to("meta")m.to("meta")+ # Move remaining GPU tensors to CPU; offload to disk if low_cpu_mem_usage.+ # This mirrors _quantize_via_rtn_blockwise's post-block cleanup.+ if not self.is_immediate_saving:+ mv_module_from_gpu(block)+ else:+ # Save once at block scope to capture tensors that are not saved+ # in per-layer branch (e.g., custom module-level params/buffers).+ shard_writer(self, name=block_name)+ block.to("meta")+ if self.low_cpu_mem_usage and not self.is_immediate_saving:+ self._offloader.offload(self.model, block_name)clear_memory(device_list=self.device_list)memory_monitor.log_summary()pbar.update(1)@@ -1413,6 +1442,8 @@ class BaseCompressor(object):# Convert remaining fp8convert_module_to_hp_if_necessary(self.model, self.amp_dtype, self.device)+ if self.low_cpu_mem_usage:+ self._offloader.reload(self.model)if self.is_immediate_saving:shard_writer(self, is_finalize=True)@@ -1531,9 +1562,9 @@ class BaseCompressor(object):self._quantize_layer_via_rtn(m.global_name, to_cpu=self.low_gpu_mem_usage)all_to_quantized_module_names.remove(m.global_name)- if not self.is_immediate_saving:- # some modules may have been flushed and set to meta, so we could not move to gpu- mv_module_from_gpu(block)+ mv_module_from_gpu(block)+ if self.low_cpu_mem_usage and not self.is_immediate_saving:+ self._offloader.offload(self.model, block_name)if block_name == block_names[-1]:clear_memory(input_ids, device_list=self.device_list)else:@@ -1644,14 +1675,18 @@ class BaseCompressor(object):self.is_immediate_saving = Falseif self.low_cpu_mem_usage and self.is_immediate_packing:- if self.has_qlayer_outside_block and self.disable_opt_rtn and self.iters == 0:+ if formats[0].is_gguf():logger.warning(- "`low_cpu_mem_usage` is not fully supported "- "when there are quantized layers outside blocks and optimized RTN is disabled. "+ "`low_cpu_mem_usage` is not fully supported for gguf format. ""Setting `low_cpu_mem_usage` to False.")self.low_cpu_mem_usage = Falseself.is_immediate_saving = False+ elif self.has_qlayer_outside_block and self.disable_opt_rtn and self.iters == 0:+ logger.info(+ "Keeping `low_cpu_mem_usage` enabled in RTN mode (iters=0): "+ "RTN path uses blockwise quantization and supports per-block offloading."+ )elif self.has_qlayer_outside_block and self.iters > 0:logger.warning("`low_cpu_mem_usage` is not fully supported "@@ -1660,12 +1695,6 @@ class BaseCompressor(object):)self.low_cpu_mem_usage = Falseself.is_immediate_saving = False- elif formats[0].is_gguf():- logger.warning(- "`low_cpu_mem_usage` is not fully supported for gguf format" "Setting `low_cpu_mem_usage `to False."- )- self.low_cpu_mem_usage = False- self.is_immediate_saving = Falseif self.is_immediate_saving and "int" not in self.data_type:logger.warning("immediate_saving is only supported for int quantization, set to False")@@ -1700,6 +1729,9 @@ class BaseCompressor(object):self.configure_layer_config(enable_gguf_official_mixed=enable_gguf_official_mixed)+ if self.low_cpu_mem_usage:+ self._offloader.reset()+def _should_disable_inplace_due_to_layers_outside_block() -> bool:return self.has_qlayer_outside_block and (self.iters != 0 or (self.iters == 0 and not self.disable_opt_rtn))@@ -1747,11 +1779,15 @@ class BaseCompressor(object):all_q_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names=layer_names)- self.model = safe_device_move_with_meta_handling(self.model, "cpu")- clear_memory(device_list=self.device_list)+ # Remove accelerate dispatch hooks before moving parameters.+ # hf_device_map is kept for reference but hooks are no longer needed.if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:- accelerate.hooks.remove_hook_from_submodules(self.model) # self.model.hf_device_map has not been changed+ accelerate.hooks.remove_hook_from_submodules(self.model)+ self.model = mv_module_from_gpu(self.model)+ clear_memory(device_list=self.device_list)logger.info("caching done")+ if self.low_cpu_mem_usage:+ self._offloader.offload(self.model, all_blocks, clear_memory=True, device_list=self.device_list)if len(all_blocks) > 1:pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))else:@@ -1797,6 +1833,9 @@ class BaseCompressor(object):if self.is_immediate_saving:shard_writer(self, is_finalize=True)+ if self.low_cpu_mem_usage:+ self._offloader.reload(self.model)+end_time = time.time()cost_time = end_time - start_timelogger.info(f"quantization tuning time {cost_time}")@@ -2923,7 +2962,7 @@ class BaseCompressor(object):if hook_handles:self._get_block_outputs(block,- q_input,+ q_input if q_input is not None else input_ids,input_others,self.batch_size * self.infer_bs_coeff,device,@@ -3201,6 +3240,12 @@ class BaseCompressor(object):modules = [get_module(model, n) for n in names]m = WrapperMultiblock(modules)+ if self.low_cpu_mem_usage:+ if nblocks == 1:+ self._offloader.reload(model, n)+ else:+ self._offloader.reload(model, names)+m.config = model.config if hasattr(model, "config") else Noneq_input, input_ids = self._quantize_block(m,@@ -3219,6 +3264,13 @@ class BaseCompressor(object):if self.is_immediate_saving:shard_writer(self, m, is_finalize=False)++ if self.low_cpu_mem_usage and not self.is_immediate_saving:+ if nblocks == 1:+ self._offloader.offload(model, n, overwrite=True)+ else:+ for name in names:+ self._offloader.offload(model, name, overwrite=True)if pbar is not None:pbar.update(1)diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pyindex 62362bff3..990c5c387 100644--- a/auto_round/compressors/shard_writer.py+++ b/auto_round/compressors/shard_writer.py@@ -58,6 +58,7 @@ class ShardWriter:# Statsself.total_param_elems = 0self.total_param_size_bytes = 0+ self.skipped_meta_tensors = []# Directory Setupself.output_dir = os.path.join(rounder._get_save_folder_name(rounder.formats[0]), "")@@ -95,6 +96,9 @@ class ShardWriter:self._add_tensor(param_name, v)def _add_tensor(self, name: str, tensor: torch.Tensor):+ if isinstance(tensor, torch.Tensor) and tensor.device.type == "meta":+ self.skipped_meta_tensors.append(name)+ returnt_size = tensor.nbytesself.total_param_elems += tensor.numel()self.total_param_size_bytes += t_size@@ -162,6 +166,7 @@ class ShardWriter:tie_word_embeddings = getattr(getattr(self.model, "config", None), "tie_word_embeddings", True)all_saved_names = {p for meta in self.shard_meta for p in meta["params"]}+ finalize_skipped_meta_tensors = []for pname, tensor in full_sd.items():if pname in all_saved_names:continue@@ -176,6 +181,10 @@ class ShardWriter:self._flush_shard()+ total_skipped = len(self.skipped_meta_tensors) + len(finalize_skipped_meta_tensors)+ if total_skipped > 0:+ examples = (self.skipped_meta_tensors + finalize_skipped_meta_tensors)[:5]+# 2. Rename temp files to HF standard and map weightsif self.shard_counter == 0:logger.warning("No tensors saved.")diff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex 78f595745..d605268a2 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -11,16 +11,19 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+import ctypesimport functoolsimport gcimport osimport re+import shutilimport sys+import tempfilefrom contextlib import ContextDecorator, contextmanagerfrom functools import lru_cachefrom itertools import combinationsfrom threading import Lock-from typing import Callable, Union+from typing import Any, Callable, Optional, Unionimport cpuinfoimport psutil@@ -513,6 +516,7 @@ def _clear_memory_for_cpu_and_cuda(tensor[i] = Nonetensor = Nonegc.collect()+ _maybe_trim_malloc()# ------------------------# Normalize device_list@@ -556,6 +560,40 @@ def _clear_memory_for_cpu_and_cuda(torch.xpu.empty_cache()+_malloc_trim_counter = 0+++def _maybe_trim_malloc() -> None:+ """Optionally release glibc heap pages back to OS on Linux.++ Controlled by environment variables:+ - AR_ENABLE_MALLOC_TRIM: default "1" (enabled)+ - AR_MALLOC_TRIM_EVERY: default "10" (trigger every N clear_memory calls)+ """+ global _malloc_trim_counter++ if os.name != "posix":+ return+ if os.environ.get("AR_ENABLE_MALLOC_TRIM", "1") != "1":+ return++ try:+ every = int(os.environ.get("AR_MALLOC_TRIM_EVERY", "10"))+ except ValueError:+ every = 10+ every = max(1, every)++ _malloc_trim_counter += 1+ if _malloc_trim_counter % every != 0:+ return++ try:+ libc = ctypes.CDLL("libc.so.6")+ libc.malloc_trim(0)+ except Exception:+ pass++class ClearMemory:def __init__(self, device_list: list | tuple | None = None):self.device_list = device_listdiff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 0e8bb5938..91f2b7e89 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -1164,13 +1164,27 @@ def mv_module_from_gpu(module):The module on the specified device."""if hasattr(module, "device"):- target_device = "cpu"- if module.device.type == target_device:+ if module.device.type in ("cpu", "meta"):return module- else:- return module.to(target_device)- else:- return module.to("cpu")++ has_meta = any(p.device.type == "meta" for p in module.parameters())+ if not has_meta:+ has_meta = any(b.device.type == "meta" for b in module.buffers())++ if has_meta:+ for _, child in module.named_children():+ mv_module_from_gpu(child)+ for attr_name in list(module._parameters.keys()):+ p = module._parameters[attr_name]+ if p is not None and p.device.type != "meta" and p.device.type != "cpu":+ module._parameters[attr_name] = p.to("cpu")+ for attr_name in list(module._buffers.keys()):+ b = module._buffers[attr_name]+ if b is not None and b.device.type != "meta" and b.device.type != "cpu":+ module._buffers[attr_name] = b.to("cpu")+ return module++ return module.to("cpu")def safe_device_move_with_meta_handling(diff --git a/auto_round/utils/offload.py b/auto_round/utils/offload.pynew file mode 100644index 000000000..b39d2fbc9--- /dev/null+++ b/auto_round/utils/offload.py@@ -0,0 +1,692 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Offloading module for CPU RAM optimization.++This module provides :class:`OffloadManager`, a single class that can+**offload** any named module (block, layer, or sub-module) to free CPU RAM+and **reload** it on demand. Two operating modes are supported:++* ``"offload"`` -- saves the module's ``state_dict`` to a temporary directory+ before clearing. Best when the original weights may have been modified+ (e.g. after quantization) and need to be restored later.+* ``"clean"`` -- simply clears weights without saving; reloads from the+ original HuggingFace model checkpoint files. Zero extra disk writes.++Usage patterns+--------------++**Compressor (offload mode)**::++ offloader = OffloadManager(mode="offload")+ offloader.offload(model, "model.layers.0") # save + clear one module+ offloader.offload(model, block_names, clear_memory=True) # save + clear many + gc+ offloader.reload(model, "model.layers.0") # load back + auto-clean temp file++**AutoScheme (clean mode with hooks)**::++ offloader = OffloadManager(mode="clean",+ model_dir="/path/to/model")+ offloader.add_offload_hooks(model, block_names) # clear all + register hooks+ # ... forward passes transparently reload on demand ...+ offloader.remove_offload_hooks(model, block_names) # remove hooks + restore all+"""++import gc+import json+import os+import shutil+import tempfile+from collections import defaultdict+from functools import partial+from typing import Any, Optional, Union++import torch++from auto_round.logger import logger+from auto_round.utils.model import get_module++__all__ = ["OffloadManager"]++# =====================================================================+# Low-level helpers+# =====================================================================+++def _load_state_dict_into_module(state_dict: dict, module: torch.nn.Module) -> None:+ """Assign every key in *state_dict* to the corresponding sub-module.++ Handles cleared parameters (empty tensors) and wrapper objects that store+ the original layer in an ``orig_layer`` attribute.+ """+ for name, param in state_dict.items():+ parts = name.split(".")+ target = module+ try:+ for part in parts[:-1]:+ target = getattr(target, part)+ except AttributeError:+ continue+ if hasattr(target, "orig_layer"):+ target = target.orig_layer+ param_name = parts[-1]+ if hasattr(target, param_name):+ old_param = getattr(target, param_name)+ if isinstance(old_param, torch.nn.Parameter):+ param = param.to(dtype=old_param.dtype, device=old_param.device)+ setattr(target, param_name, torch.nn.Parameter(param, requires_grad=old_param.requires_grad))+ else:+ setattr(target, param_name, param)+ del state_dict+++def _clear_module_weights(module: torch.nn.Module, cache_numel: bool = False) -> None:+ """Replace a single module's weight/bias with empty tensors.++ Args:+ module: The leaf module to clear.+ cache_numel: If *True*, store ``_cached_weight_numel`` and+ ``_cached_weight_shape`` before clearing.+ """+ if module is None:+ return+ if hasattr(module, "orig_layer"):+ return++ with torch.no_grad():+ for name, param in list(module.named_parameters(recurse=False)):+ if param is None or param.numel() == 0:+ continue+ if cache_numel and name == "weight":+ module._cached_weight_numel = param.numel()+ module._cached_weight_shape = tuple(param.shape)+ setattr(+ module,+ name,+ torch.nn.Parameter(torch.empty(0, dtype=param.dtype, device="cpu"), requires_grad=param.requires_grad),+ )+ for name, buf in list(module.named_buffers(recurse=False)):+ if buf is None or buf.numel() == 0:+ continue+ module.register_buffer(name, torch.empty(0, dtype=buf.dtype, device="cpu"))+++# =====================================================================+# Checkpoint file helpers+# =====================================================================+++def _resolve_model_dir(model_dir: str) -> str:+ """Resolve a model name/path to a local directory containing weight files."""+ if os.path.isdir(model_dir):+ return model_dir+ try:+ from huggingface_hub import snapshot_download++ return snapshot_download(model_dir, local_files_only=True)+ except Exception:+ return model_dir+++def _build_weight_map(model_dir: str) -> dict[str, str]:+ """Build ``{tensor_name: shard_filename}`` from the model directory."""+ index_path = os.path.join(model_dir, "model.safetensors.index.json")+ if os.path.exists(index_path):+ with open(index_path) as f:+ return json.load(f)["weight_map"]++ single_path = os.path.join(model_dir, "model.safetensors")+ if os.path.exists(single_path):+ from safetensors import safe_open++ with safe_open(single_path, framework="pt") as f:+ return {k: "model.safetensors" for k in f.keys()}++ bin_index_path = os.path.join(model_dir, "pytorch_model.bin.index.json")+ if os.path.exists(bin_index_path):+ with open(bin_index_path) as f:+ return json.load(f)["weight_map"]++ single_bin = os.path.join(model_dir, "pytorch_model.bin")+ if os.path.exists(single_bin):+ state_dict = torch.load(single_bin, map_location="cpu")+ return {k: "pytorch_model.bin" for k in state_dict.keys()}++ raise FileNotFoundError(+ f"Could not find model weight files in {model_dir}. "+ "Expected model.safetensors or pytorch_model.bin (with optional index.json)."+ )+++def load_block_from_model_files(model_dir: str, block_name: str, block: torch.nn.Module) -> None:+ """Reload a module's weights directly from the original model checkpoint.++ Selectively loads only tensors belonging to *block_name* without loading+ the entire model into memory.++ Args:+ model_dir: Path to the model directory (local or HuggingFace hub name).+ block_name: The module name prefix (e.g. ``"model.layers.0"``).+ block: The ``nn.Module`` to load weights into.+ """+ model_dir = _resolve_model_dir(model_dir)+ weight_map = _build_weight_map(model_dir)++ prefix = block_name + "."+ matching = {k: v for k, v in weight_map.items() if k.startswith(prefix)}+ if not matching:+ logger.warning(f"No tensors found for {block_name} in {model_dir}")+ return++ shard_to_tensors: dict[str, list[str]] = defaultdict(list)+ for tensor_name, shard_file in matching.items():+ shard_to_tensors[shard_file].append(tensor_name)++ state_dict = {}+ for shard_file, tensor_names in shard_to_tensors.items():+ shard_path = os.path.join(model_dir, shard_file)+ if shard_file.endswith(".safetensors"):+ from safetensors import safe_open++ with safe_open(shard_path, framework="pt", device="cpu") as f:+ for name in tensor_names:+ state_dict[name[len(prefix) :]] = f.get_tensor(name)+ else:+ full_state = torch.load(shard_path, map_location="cpu")+ for name in tensor_names:+ if name in full_state:+ state_dict[name[len(prefix) :]] = full_state[name]+ del full_state++ _load_state_dict_into_module(state_dict, block)+++# =====================================================================+# OffloadManager -- the offload manager class+# =====================================================================+++class OffloadManager:+ """Module offloader for CPU RAM optimization.++ Provides two core operations:++ * `offload` -- save/clear a named module's weights to free CPU RAM.+ * `reload` -- restore the weights back into the module.++ Works at any granularity (block, layer, individual module) and supports+ two operating modes:++ * ``"offload"`` -- saves ``state_dict`` to a temp directory on disk+ before clearing. Supports ``overwrite=True`` for re-saving modified+ (e.g. quantized) weights.+ * ``"clean"`` -- simply clears weights; reloads from the original+ HuggingFace model checkpoint files. Zero extra disk writes but+ cannot persist modifications.++ The manager tracks its own state and supports automatic cleanup via+ context manager (``with`` statement) or explicit `cleanup`.++ Parameters+ ----------+ enabled : bool+ When *False*, every public method is a no-op.+ mode : str+ ``"offload"`` or ``"clean"``.+ model_dir : str, optional+ Path to the model checkpoint directory. Required for ``"clean"``+ mode; optional for ``"offload"``.+ offload_dir_prefix : str+ Prefix for the temp directory name (``"offload"`` mode only).+ cache_numel : bool+ If *True*, cache original ``weight.numel()`` before clearing so+ downstream code can still determine the original size.+ """++ def __init__(+ self,+ enabled: bool = True,+ mode: str = "offload",+ model_dir: Optional[str] = None,+ offload_dir_prefix: str = "ar_offload",+ cache_numel: bool = False,+ ):+ self.enabled = enabled+ self.mode = mode+ self.model_dir = model_dir+ self.cache_numel = cache_numel+ self._prefix = offload_dir_prefix++ # Disk state (offload mode)+ self._tempdir: Optional[str] = None+ self._saved: dict[str, dict] = {} # name -> {"save_path": str}++ # Hook state (for add_offload_hooks/remove_offload_hooks transparent offloading)+ self._hook_handles: list = []+ self._model_ref: Optional[torch.nn.Module] = None+ self._module_names: list[str] = []+ self._last_loaded: Optional[str] = None++ # Ensure-style state (for wrapping loops)+ self._current_loaded: Optional[str] = None++ # ------------------------------------------------------------------+ # Context manager+ # ------------------------------------------------------------------++ def __enter__(self):+ return self++ def __exit__(self, exc_type, exc_val, exc_tb):+ self.cleanup()+ return False++ def __del__(self):+ try:+ self.cleanup(_skip_reload=True)+ except Exception:+ pass++ # ------------------------------------------------------------------+ # Core API+ # ------------------------------------------------------------------++ def offload(+ self,+ model: torch.nn.Module,+ names: Union[str, list[str], list[list[str]]],+ *,+ skip_if_saved: bool = False,+ overwrite: bool = False,+ clear_memory: bool = False,+ device_list=None,+ ) -> float:+ """Offload one or more named modules' weights to free CPU RAM.++ Accepts a single module name, a flat list, or a nested list of names.+ For ``"offload"`` mode: saves state_dict to disk, then clears weights.+ For ``"clean"`` mode: simply clears weights (will reload from+ model files later).++ Args:+ model: The root model containing the module(s).+ names: Module name(s) -- a single string, a list of strings,+ or a nested list (e.g. ``all_blocks``).+ skip_if_saved: If *True* and a name was already saved, skip it.+ overwrite: If *True*, discard any previous save and re-save the+ current (possibly modified) weights before clearing. Useful+ after quantization when updated weights must be persisted.+ Only meaningful for ``"offload"`` mode.+ clear_memory: If *True*, call ``clear_memory()`` after offloading+ all modules and log the total freed size.+ device_list: Device list passed to ``clear_memory`` (only used+ when *clear_memory* is *True*).++ Returns:+ Total offloaded size in GB (non-zero only when *clear_memory* is True).+ """+ if not self.enabled:+ return 0.0+ if isinstance(names, str):+ self._offload(model, names, skip_if_saved=skip_if_saved, overwrite=overwrite)+ return 0.0+ flat_names = self._flatten_names(names)+ if clear_memory:+ logger.info("offloading module weights...")+ total_gb = 0.0+ for name in flat_names:+ if skip_if_saved and self.has(name):+ continue+ if clear_memory:+ module = get_module(model, name)+ if module is not None:+ total_gb += self.estimate_module_size_gb(module)+ self._offload(model, name, skip_if_saved=skip_if_saved, overwrite=overwrite)+ if clear_memory:+ from auto_round.utils import clear_memory as _clear_memory++ _clear_memory(device_list=device_list)+ logger.info(f"offload done, freed {total_gb:.2f} GB")+ return total_gb++ def _offload(+ self, model: torch.nn.Module, name: str, *, skip_if_saved: bool = False, overwrite: bool = False+ ) -> None:+ """Offload a single named module (internal helper)."""+ self._model_ref = model+ module = get_module(model, name)+ if module is None:+ return+ if self.mode == "offload":+ if overwrite:+ old_meta = self._saved.pop(name, None)+ if old_meta:+ old_path = old_meta.get("save_path")+ if old_path and os.path.exists(old_path):+ try:+ os.remove(old_path)+ except Exception as e:+ logger.warning(f"OffloadManager: could not remove {old_path}: {e}")+ elif skip_if_saved and name in self._saved:+ return+ self._save_to_disk(name, module)+ self._clear(module)++ def reload(self, model: torch.nn.Module, names: Union[str, list[str], None] = None) -> None:+ """Reload previously offloaded module(s).++ For ``"offload"`` mode: loads from the temp directory, then+ removes the temp file. When all offloaded modules have been+ reloaded the temp directory is automatically deleted.+ For ``"clean"`` mode: loads from original model files.++ Args:+ model: The root model containing the module(s).+ names: A single module name, a list of names, or *None*+ to reload all tracked modules.+ """+ if not self.enabled:+ return+ if names is None:+ if self.mode == "offload":+ names = list(self._saved.keys())+ else:+ names = list(self._module_names)+ if isinstance(names, str):+ self._reload(model, names)+ return+ for name in names:+ self._reload(model, name)++ def _reload(self, model: torch.nn.Module, name: str) -> None:+ """Reload a single named module (internal helper)."""+ module = get_module(model, name)+ if module is None:+ return+ if self.mode == "offload":+ self._load_from_disk(name, module)+ self._remove_saved_entry(name)+ else:+ if self.model_dir is None:+ logger.warning("OffloadManager: model_dir is required for clean mode")+ return+ load_block_from_model_files(self.model_dir, name, module)++ # ------------------------------------------------------------------+ # Hook-based transparent offloading+ # ------------------------------------------------------------------++ def add_offload_hooks(self, model: torch.nn.Module, names: list[str]) -> None:+ """Clear all named modules and register pre-forward hooks for+ transparent reload-on-demand.++ Args:+ model: The root model.+ names: List of module names to manage.+ """+ if not self.enabled:+ return+ self._model_ref = model+ self._module_names = list(names)+ # Register hooks+ for name in names:+ module = get_module(model, name)+ if module is None:+ continue+ handle = module.register_forward_pre_hook(partial(self._pre_forward_hook, name=name))+ self._hook_handles.append(handle)+ # Clear all+ logger.info("clearing module weights to free RAM...")+ for name in names:+ self._offload(model, name, skip_if_saved=True)+ gc.collect()+ from auto_round.utils import clear_memory++ clear_memory()+ logger.info("module weights cleared")++ def remove_offload_hooks(self, model: torch.nn.Module, names: Optional[list[str]] = None) -> None:+ """Remove hooks and reload all managed modules.++ Args:+ model: The root model.+ names: Module names to reload. If None, uses the names from+ `add_offload_hooks`.+ """+ self._remove_hooks()+ if names is None:+ names = self._module_names+ self.reload(model, names)+ self._model_ref = None++ def _pre_forward_hook(self, module: torch.nn.Module, args, *, name: str) -> None:+ """Pre-forward hook: reload this module if cleared, clear previous."""+ if not self._needs_loading(module):+ return+ # Clear previous to keep only one module loaded at a time+ if self._last_loaded is not None and self._last_loaded != name:+ prev = get_module(self._model_ref, self._last_loaded)+ if prev is not None:+ self._clear(prev)+ self.reload(self._model_ref, name)+ self._last_loaded = name++ def _remove_hooks(self) -> None:+ for handle in self._hook_handles:+ handle.remove()+ self._hook_handles.clear()+ self._last_loaded = None++ # ------------------------------------------------------------------+ # Ensure-style API (for wrapping loops)+ # ------------------------------------------------------------------++ def ensure_loaded(self, model: torch.nn.Module, layer_name: str) -> None:+ """Ensure the module containing *layer_name* is loaded.++ When the loop moves to a layer in a different module, the previous+ module is automatically cleared. Call `flush_loaded` after+ the loop to release the last module.++ Args:+ model: The root model.+ layer_name: Full layer name (e.g. ``"model.layers.0.attn.q"``).+ """+ if not self.enabled or not self._module_names:+ return+ target = None+ for mn in self._module_names:+ if layer_name.startswith(mn + "."):+ target = mn+ break+ if target is None:+ return+ if target == self._current_loaded:+ return+ # Clear previous+ if self._current_loaded is not None:+ module = get_module(model, self._current_loaded)+ if module is not None:+ self._clear(module)+ # Load new+ self.reload(model, target)+ self._current_loaded = target++ def flush_loaded(self, model: torch.nn.Module) -> None:+ """Clear the last module loaded by `ensure_loaded`."""+ if self._current_loaded is not None:+ module = get_module(model, self._current_loaded)+ if module is not None:+ self._clear(module)+ self._current_loaded = None++ # ------------------------------------------------------------------+ # State queries+ # ------------------------------------------------------------------++ def has(self, name: str) -> bool:+ """Return *True* if *name* has been offloaded."""+ if self.mode == "offload":+ return name in self._saved+ return False++ def reset(self) -> None:+ """Clear tracking state (temp directory is kept)."""+ self._saved = {}+ self._current_loaded = None+ self._last_loaded = None++ def cleanup(self, _skip_reload: bool = False) -> None:+ """Reload remaining modules and remove temp files.++ Normally this is called automatically -- ``reload`` deletes each+ temp file as it goes and removes the temp directory when done.+ Explicit calls are usually unnecessary; the context-manager exit+ and ``__del__`` use this as a fallback.+ """+ self._remove_hooks()+ # Auto-reload offloaded modules if we still have a model reference+ if not _skip_reload and self._model_ref is not None and self._saved:+ try:+ self.reload(self._model_ref)+ except Exception as e:+ logger.warning(f"OffloadManager: auto-reload during cleanup failed: {e}")+ self._cleanup_tempdir()+ self._saved = {}+ self._current_loaded = None+ self._module_names = []+ self._model_ref = None++ # ------------------------------------------------------------------+ # Static estimation helpers+ # ------------------------------------------------------------------++ @staticmethod+ def estimate_tensor_size_gb(tensor: Any) -> float:+ """Estimate size of a tensor (or nested container) in GB."""+ if tensor is None:+ return 0.0+ if isinstance(tensor, torch.Tensor):+ return tensor.numel() * tensor.element_size() / (1024**3)+ if isinstance(tensor, list):+ return sum(OffloadManager.estimate_tensor_size_gb(t) for t in tensor)+ if isinstance(tensor, dict):+ return sum(OffloadManager.estimate_tensor_size_gb(v) for v in tensor.values())+ return 0.0++ @staticmethod+ def estimate_inputs_size_gb(all_inputs: dict) -> float:+ """Estimate total size of calibration inputs in GB."""+ return sum(OffloadManager.estimate_tensor_size_gb(v) for v in all_inputs.values())++ @staticmethod+ def estimate_model_size_gb(model: torch.nn.Module) -> float:+ """Estimate model weights size in GB."""+ return sum(p.numel() * p.element_size() / (1024**3) for p in model.parameters() if p.numel() > 0)++ @staticmethod+ def estimate_module_size_gb(module: torch.nn.Module) -> float:+ """Estimate a module's parameter size in GB."""+ return sum(p.numel() * p.element_size() / (1024**3) for p in module.parameters() if p.numel() > 0)++ # ------------------------------------------------------------------+ # Internal: disk operations (offload mode)+ # ------------------------------------------------------------------++ def _ensure_dir(self) -> str:+ if self._tempdir is None:+ self._tempdir = tempfile.mkdtemp(prefix=f"{self._prefix}_")+ logger.info(f"OffloadManager ({self._prefix}): tempdir = {self._tempdir}")+ return self._tempdir++ def _save_to_disk(self, name: str, module: torch.nn.Module) -> None:+ tmpdir = self._ensure_dir()+ safe_name = name.replace(".", "_")+ save_path = os.path.join(tmpdir, f"{safe_name}.pt")+ try:+ # Skip meta tensors: they contain no real data (e.g. quantized weights+ # already flushed to disk by an immediate-saving shard writer).+ state_dict = {k: v.cpu().contiguous() for k, v in module.state_dict().items() if v.device.type != "meta"}+ torch.save(state_dict, save_path)+ self._saved[name] = {"save_path": save_path}+ del state_dict+ except Exception as e:+ logger.warning(f"OffloadManager: failed to save {name}: {e}")++ def _load_from_disk(self, name: str, module: torch.nn.Module) -> None:+ metadata = self._saved.get(name)+ if not metadata:+ return+ save_path = metadata["save_path"]+ if not os.path.exists(save_path):+ logger.warning(f"OffloadManager: file not found {save_path}")+ return+ try:+ state_dict = torch.load(save_path, map_location="cpu")+ _load_state_dict_into_module(state_dict, module)+ except Exception as e:+ logger.warning(f"OffloadManager: failed to load {name}: {e}")++ def _remove_saved_entry(self, name: str) -> None:+ """Remove a single saved entry and its temp file; clean tempdir if empty."""+ meta = self._saved.pop(name, None)+ if meta:+ path = meta.get("save_path")+ if path and os.path.exists(path):+ try:+ os.remove(path)+ except Exception as e:+ logger.warning(f"OffloadManager: could not remove {path}: {e}")+ if not self._saved:+ self._cleanup_tempdir()++ def _cleanup_tempdir(self) -> None:+ """Remove the temp directory if it exists."""+ if self._tempdir and os.path.isdir(self._tempdir):+ try:+ shutil.rmtree(self._tempdir)+ except Exception as e:+ logger.warning(f"OffloadManager: cleanup failed for {self._tempdir}: {e}")+ self._tempdir = None++ # ------------------------------------------------------------------+ # Internal: clearing+ # ------------------------------------------------------------------++ def _clear(self, module: torch.nn.Module) -> None:+ """Clear all weight/bias tensors in *module* and its sub-modules."""+ for submodule in module.modules():+ _clear_module_weights(submodule, cache_numel=self.cache_numel)++ @staticmethod+ def _needs_loading(module: torch.nn.Module) -> bool:+ """Return *True* if any parameter in *module* has been cleared."""+ for submodule in module.modules():+ if hasattr(submodule, "orig_layer"):+ submodule = submodule.orig_layer+ for param in submodule.parameters(recurse=False):+ if param is not None and param.numel() == 0:+ return True+ return False++ @staticmethod+ def _flatten_names(names: Union[list[str], list[list[str]]]) -> list[str]:+ """Flatten a potentially nested list of names."""+ flat = []+ for item in names:+ if isinstance(item, list):+ flat.extend(item)+ else:+ flat.append(item)+ return flatdiff --git a/test/test_cpu/core/test_low_cpu_mem_options.py b/test/test_cpu/core/test_low_cpu_mem_options.pynew file mode 100644index 000000000..543a9a515--- /dev/null+++ b/test/test_cpu/core/test_low_cpu_mem_options.py@@ -0,0 +1,95 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""+Unit tests for low_cpu_mem_usage option:+When enabled, block weights are offloaded to disk upfront and loaded on demand+during quantization, significantly reducing peak CPU RAM usage.+"""++import torch++from auto_round import AutoRound+from auto_round.utils import offload as offload_module+++class TestLowCpuMemUsage:+ """Tests for low_cpu_mem_usage block offloading."""++ def test_option_stored_correctly(self, tiny_opt_model_path):+ """Test that low_cpu_mem_usage option is stored correctly."""+ autoround = AutoRound(+ tiny_opt_model_path,+ bits=4,+ low_cpu_mem_usage=True,+ iters=0,+ disable_opt_rtn=True,+ nsamples=1,+ seqlen=32,+ )+ assert autoround.low_cpu_mem_usage is True++ autoround2 = AutoRound(+ tiny_opt_model_path,+ bits=4,+ low_cpu_mem_usage=False,+ iters=0,+ disable_opt_rtn=True,+ nsamples=1,+ seqlen=32,+ )+ assert autoround2.low_cpu_mem_usage is False++ def test_offloader_disabled_when_low_cpu_mem_false(self, tiny_opt_model_path):+ """Test that the offloader is disabled when low_cpu_mem_usage=False."""+ autoround = AutoRound(+ tiny_opt_model_path,+ bits=4,+ low_cpu_mem_usage=False,+ iters=0,+ disable_opt_rtn=True,+ nsamples=1,+ seqlen=32,+ )+ assert autoround.low_cpu_mem_usage is False+ # offload with list should be a no-op when offloader is disabled+ autoround._offloader.offload(+ autoround.model, [["model.layers.0"]], clear_memory=True, device_list=autoround.device_list+ )+ assert autoround._offloader._saved == {}++ def test_offload_all_records_blocks(self, tiny_opt_model_path, tmp_path, monkeypatch):+ """Test that offload(list) records offloaded blocks when enabled."""+ autoround = AutoRound(+ tiny_opt_model_path,+ bits=4,+ low_cpu_mem_usage=True,+ iters=0,+ disable_opt_rtn=True,+ nsamples=1,+ seqlen=32,+ )++ dummy_block = torch.nn.Linear(4, 4)+ # Monkeypatch get_module used by offload in offload.py++ monkeypatch.setattr(offload_module, "get_module", lambda _model, _name: dummy_block)+ monkeypatch.setattr(torch, "save", lambda *args, **kwargs: None)++ # Force the offloader to think it has a tempdir already+ autoround._offloader._tempdir = str(tmp_path)+ autoround._offloader.offload(+ autoround.model, [["model.layers.0"]], clear_memory=True, device_list=autoround.device_list+ )+ assert autoround._offloader.has("model.layers.0")diff --git a/test/test_cpu/schemes/test_auto_scheme_low_cpu_mem.py b/test/test_cpu/schemes/test_auto_scheme_low_cpu_mem.pynew file mode 100644index 000000000..4776db7c0--- /dev/null+++ b/test/test_cpu/schemes/test_auto_scheme_low_cpu_mem.py@@ -0,0 +1,361 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""+Unit tests for AutoScheme CPU RAM optimization (low_cpu_mem_usage option).++This tests the disk offload mechanism for AutoScheme that reduces CPU RAM usage+by offloading block weights to disk during gradient computation.+"""++import shutil++import pytest+import torch++from auto_round import AutoRound, AutoScheme+from auto_round.auto_scheme.utils import compute_layer_bits+from auto_round.utils import get_block_names, get_module+from auto_round.utils.offload import OffloadManager, _clear_module_weights+++class TestOffloadManager:+ """Tests for OffloadManager class."""++ def test_context_init_disabled(self):+ """Test that manager is properly initialized when disabled."""+ mgr = OffloadManager(enabled=False)+ assert mgr.enabled is False++ def test_context_init_enabled(self):+ """Test that manager is properly initialized when enabled."""+ mgr = OffloadManager(enabled=True, mode="clean", model_dir="/tmp/fake")+ assert mgr.enabled is True+ assert mgr.model_dir == "/tmp/fake"+ assert mgr.mode == "clean"++ def test_clear_clears_weights(self):+ """Test that _clear clears weights from a module."""+ mgr = OffloadManager(enabled=True, mode="clean")+ module = torch.nn.Linear(4, 4)+ mgr._clear(module)++ # Verify weight was cleared+ assert module.weight.numel() == 0++ def test_add_hooks_noop_when_disabled(self):+ """Test that add_offload_hooks does nothing when disabled."""+ mgr = OffloadManager(enabled=False)+ model = torch.nn.Sequential(torch.nn.Linear(4, 4))+ original_numel = model[0].weight.numel()+ mgr.add_offload_hooks(model, ["0"])+ assert model[0].weight.numel() == original_numel # unchanged++ def test_ensure_loaded_noop_when_disabled(self):+ """Test that ensure_loaded does nothing when disabled."""+ mgr = OffloadManager(enabled=False)+ model = torch.nn.Sequential(torch.nn.Linear(4, 4))+ mgr.ensure_loaded(model, "0.weight") # should not raise++ def test_cleanup_resets_state(self):+ """Test that cleanup resets internal state."""+ mgr = OffloadManager(enabled=True, mode="clean")+ module = torch.nn.Linear(4, 4)+ mgr._clear(module)+ mgr._module_names = ["test"]+ assert len(mgr._module_names) == 1++ mgr.cleanup()+ assert len(mgr._module_names) == 0++ def test_reset_clears_tracking(self):+ """Test that reset clears tracking state."""+ mgr = OffloadManager(enabled=True, mode="offload")+ mgr._saved["b"] = {"save_path": "/tmp/x"}+ assert len(mgr._saved) == 1++ mgr.reset()+ assert len(mgr._saved) == 0++ def test_model_dir_property(self):+ """Test model_dir getter/setter."""+ mgr = OffloadManager(enabled=True, mode="clean", model_dir="/path/a")+ assert mgr.model_dir == "/path/a"+ mgr.model_dir = "/path/b"+ assert mgr.model_dir == "/path/b"++ def test_context_manager(self):+ """Test OffloadManager as context manager."""+ with OffloadManager(enabled=True, mode="offload") as mgr:+ assert mgr.enabled is True+ # After exit, state should be cleaned up+ assert mgr._saved == {}++ def test_reload_auto_cleans_temp_file(self):+ """Test that reload() removes the temp file and auto-cleans tempdir."""+ model = torch.nn.Sequential(torch.nn.Linear(4, 4))+ original_weight = model[0].weight.data.clone()+ mgr = OffloadManager(enabled=True, mode="offload")+ mgr.offload(model, "0")+ assert model[0].weight.numel() == 0+ assert mgr._tempdir is not None+ tempdir = mgr._tempdir++ # reload restores weights and auto-cleans temp file + tempdir+ mgr.reload(model, "0")+ assert model[0].weight.numel() > 0+ assert torch.allclose(model[0].weight.data, original_weight)+ assert len(mgr._saved) == 0+ assert mgr._tempdir is None # auto-cleaned++ def test_reload_all_auto_cleans(self):+ """Test that reload() with no names auto-cleans all temp files."""+ model = torch.nn.Sequential(+ torch.nn.Linear(4, 4),+ torch.nn.Linear(4, 4),+ )+ mgr = OffloadManager(enabled=True, mode="offload")+ mgr.offload(model, "0")+ mgr.offload(model, "1")+ assert len(mgr._saved) == 2++ mgr.reload(model)+ assert len(mgr._saved) == 0+ assert mgr._tempdir is None # auto-cleaned++ def test_context_manager_auto_reloads(self):+ """Test that context manager exit auto-reloads offloaded modules."""+ model = torch.nn.Sequential(torch.nn.Linear(4, 4))+ original_weight = model[0].weight.data.clone()+ with OffloadManager(enabled=True, mode="offload") as mgr:+ mgr.offload(model, "0")+ assert model[0].weight.numel() == 0+ # After exiting context, weights should be restored+ assert model[0].weight.numel() > 0+ assert torch.allclose(model[0].weight.data, original_weight)+++class TestClearModuleWeights:+ """Tests for _clear_module_weights helper function."""++ def test_clear_linear_weights(self):+ """Test clearing weights from a Linear module."""+ module = torch.nn.Linear(16, 8)+ _clear_module_weights(module)++ assert module.weight.numel() == 0+ assert module.bias.numel() == 0++ def test_clear_caches_numel(self):+ """Test that clearing weights caches the original numel."""+ module = torch.nn.Linear(32, 16)+ expected = 32 * 16+ _clear_module_weights(module, cache_numel=True)+ assert hasattr(module, "_cached_weight_numel")+ assert module._cached_weight_numel == expected++ def test_compute_layer_bits_with_empty_weight(self):+ """Test compute_layer_bits works after weight is cleared."""+ layer = torch.nn.Linear(64, 128)+ layer.bits = 4+ layer.group_size = 32+ layer.data_type = "int"+ layer.sym = True+ layer.super_group_size = None+ layer.super_bits = None++ bits_before, _ = compute_layer_bits(layer, False)+ _clear_module_weights(layer, cache_numel=True)+ bits_after, _ = compute_layer_bits(layer, False)+ assert bits_before == bits_after++ def test_clear_module_none(self):+ """Test clearing weights with None module doesn't crash."""+ _clear_module_weights(None) # Should not raise++ def test_clear_module_no_weights(self):+ """Test clearing module without weights doesn't crash."""+ module = torch.nn.ReLU()+ _clear_module_weights(module) # Should not raise+++class TestAutoSchemeDataclassLowCpuMem:+ """Tests for low_cpu_mem_usage parameter in AutoScheme dataclass."""++ def test_auto_scheme_default_low_cpu_mem_usage(self):+ """Test that low_cpu_mem_usage defaults to True."""+ scheme = AutoScheme(avg_bits=4, options="W4A16")+ assert scheme.low_cpu_mem_usage is True++ def test_auto_scheme_low_cpu_mem_usage_enabled(self):+ """Test that low_cpu_mem_usage can be enabled."""+ scheme = AutoScheme(avg_bits=4, options="W4A16", low_cpu_mem_usage=True)+ assert scheme.low_cpu_mem_usage is True++ def test_auto_scheme_low_cpu_mem_usage_with_low_gpu_mem_usage(self):+ """Test that both low_cpu_mem_usage and low_gpu_mem_usage can be set."""+ scheme = AutoScheme(+ avg_bits=4,+ options="W4A16",+ low_cpu_mem_usage=True,+ low_gpu_mem_usage=True,+ )+ assert scheme.low_cpu_mem_usage is True+ assert scheme.low_gpu_mem_usage is True+++class TestAutoSchemeIntegration:+ """Integration tests for AutoScheme with low_cpu_mem_usage."""++ save_dir = "./saved"++ @pytest.fixture(autouse=True, scope="class")+ def setup_and_teardown_class(self):+ yield+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_auto_scheme_with_low_cpu_mem_disabled(self, tiny_opt_model_path):+ """Test AutoScheme works normally with low_cpu_mem_usage disabled."""+ model_name = tiny_opt_model_path+ scheme = AutoScheme(+ avg_bits=4,+ options="W2A16,W4A16",+ nsamples=1,+ ignore_scale_zp_bits=True,+ low_cpu_mem_usage=False,+ low_gpu_mem_usage=True,+ )+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)+ _, layer_config = ar.quantize()+ assert layer_config is not None+ assert len(layer_config) > 0++ def test_auto_scheme_with_low_cpu_mem_enabled(self, tiny_opt_model_path):+ """Test AutoScheme works with low_cpu_mem_usage enabled."""+ model_name = tiny_opt_model_path+ scheme = AutoScheme(+ avg_bits=4,+ options="W2A16,W4A16",+ nsamples=1,+ ignore_scale_zp_bits=True,+ low_cpu_mem_usage=True,+ low_gpu_mem_usage=True,+ )+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)+ _, layer_config = ar.quantize()+ assert layer_config is not None+ assert len(layer_config) > 0++ def test_auto_scheme_low_cpu_mem_results_consistent(self, tiny_opt_model_path):+ """Test that results are consistent with and without low_cpu_mem_usage."""+ model_name = tiny_opt_model_path++ # Without low_cpu_mem_usage+ scheme1 = AutoScheme(+ avg_bits=4,+ options="W4A16",+ nsamples=1,+ ignore_scale_zp_bits=True,+ low_cpu_mem_usage=False,+ low_gpu_mem_usage=True,+ )+ ar1 = AutoRound(model=model_name, scheme=scheme1, iters=0, nsamples=1, seed=42)+ _, layer_config1 = ar1.quantize()++ # With low_cpu_mem_usage+ scheme2 = AutoScheme(+ avg_bits=4,+ options="W4A16",+ nsamples=1,+ ignore_scale_zp_bits=True,+ low_cpu_mem_usage=True,+ low_gpu_mem_usage=True,+ )+ ar2 = AutoRound(model=model_name, scheme=scheme2, iters=0, nsamples=1, seed=42)+ _, layer_config2 = ar2.quantize()++ # Layer configs should have same keys+ assert set(layer_config1.keys()) == set(layer_config2.keys())+++class TestOffloadManagerWithModel:+ """Tests for OffloadManager with actual model blocks."""++ def test_clear_and_load_model_block(self, tiny_opt_model_path):+ """Test clearing and reloading an actual model block from checkpoint files."""+ from transformers import AutoModelForCausalLM++ from auto_round.utils.offload import load_block_from_model_files++ model = AutoModelForCausalLM.from_pretrained(tiny_opt_model_path, torch_dtype=torch.float32)+ block_names = get_block_names(model)[0]++ if len(block_names) == 0:+ pytest.skip("Model has no blocks")++ block_name = block_names[0]+ block = get_module(model, block_name)++ mgr = OffloadManager(enabled=True, mode="clean", model_dir=tiny_opt_model_path, cache_numel=True)++ # Get original weights+ original_params = sum(p.numel() for p in block.parameters())+ original_weight = next(block.parameters()).data.clone()++ # Clear+ mgr._clear(block)+ current_params = sum(p.numel() for p in block.parameters())+ assert current_params < original_params++ # Load back from model files+ load_block_from_model_files(tiny_opt_model_path, block_name, block)+ restored_params = sum(p.numel() for p in block.parameters())+ assert restored_params == original_params++ # Verify values match+ restored_weight = next(block.parameters()).data+ assert torch.allclose(restored_weight, original_weight)++ def test_attach_and_detach(self, tiny_opt_model_path):+ """Test clearing all model blocks via attach and loading them back via detach."""+ from transformers import AutoModelForCausalLM++ model = AutoModelForCausalLM.from_pretrained(tiny_opt_model_path, torch_dtype=torch.float32)+ block_names = get_block_names(model)[0]++ if len(block_names) == 0:+ pytest.skip("Model has no blocks")++ mgr = OffloadManager(enabled=True, mode="clean", model_dir=tiny_opt_model_path, cache_numel=True)++ # Save original param counts+ original_counts = {}+ for bn in block_names:+ blk = get_module(model, bn)+ original_counts[bn] = sum(p.numel() for p in blk.parameters())++ # add_offload_hooks clears all blocks and registers hooks+ mgr.add_offload_hooks(model, block_names)+ for bn in block_names:+ blk = get_module(model, bn)+ block_params = sum(p.numel() for p in blk.parameters())+ assert block_params == 0++ # remove_offload_hooks removes hooks and reloads all blocks+ mgr.remove_offload_hooks(model, block_names)+ for bn in block_names:+ blk = get_module(model, bn)+ block_params = sum(p.numel() for p in blk.parameters())+ assert block_params == original_counts[bn]