PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Offloading module for CPU RAM optimization.
This module provides :class:`OffloadManager`, a single class that can
**offload** any named module (block, layer, or sub-module) to free CPU RAM
and **reload** it on demand. Two operating modes are supported:
* ``"offload"`` -- saves the module's ``state_dict`` to a temporary directory
before clearing. Best when the original weights may have been modified
(e.g. after quantization) and need to be restored later.
* ``"clean"`` -- simply clears weights without saving; reloads from the
original HuggingFace model checkpoint files. Zero extra disk writes.
Usage patterns
--------------
**Compressor (offload mode)**::
offloader = OffloadManager(mode="offload")
offloader(model, "model.layers.0") # save + clear one module
offloader(model, block_names, clear_memory=True) # save + clear many + gc
offloader.reload(model, "model.layers.0") # load back + auto-clean temp file
**AutoScheme (clean mode with hooks)**::
offloader = OffloadManager(mode="clean",
model_dir="/path/to/model")
offloader.add_offload_hooks(model, block_names) # clear all + register hooks
# ... forward passes transparently reload on demand ...
offloader.remove_offload_hooks(model, block_names) # remove hooks + restore all
"""
import gc
import json
import os
import shutil
import sys
import tempfile
from collections import defaultdict
from functools import partial
from typing import Any
import torch
from auto_round.logger import logger
from auto_round.utils.model import get_module
from auto_round.utils.path_safety import resolve_within_directory, validate_weight_map
__all__ = ["OffloadManager"]
# =====================================================================
# Low-level helpers
# =====================================================================
def _maybe_split_fused_expert_keys(state_dict: dict, module: torch.nn.Module) -> dict:
"""A checkpoint whose on-disk MoE layout is
the fused 3D one (``...experts.gate_up_proj [N, 2*inter, hidden]`` /
``...experts.down_proj``, e.g. the real qwen3.5-397b-base) cannot be
assigned onto a module tree whose experts were already replaced by the
unfused ``SequentialQwen3_5MoeExperts`` (per-expert ``nn.Linear``s) --
the fused key silently resolves to nothing and the experts stay meta,
crashing far downstream at ``block.to(device)``. Detect that exact case
(fused key present, target tree lacks the fused attribute) and split the
fused tensor into per-expert keys via the same
``split_fused_expert_tensors`` helper the missing-tensors export pass
uses. Trees that still hold the original fused module are left alone.
Never triggered before because every prior fixture's on-disk layout was
already unfused (transformers >=5.10 unfuses on save); the real
checkpoint is fused."""
fused_keys = [
k
for k, v in state_dict.items()
if k.endswith((".experts.gate_up_proj", ".experts.down_proj")) and torch.is_tensor(v) and v.dim() == 3
]
if not fused_keys:
return state_dict
to_split = {}
for key in fused_keys:
parts = key.split(".")
target = module
resolved = True
for part in parts[:-1]:
if not hasattr(target, part):
resolved = False
break
target = getattr(target, part)
if resolved and hasattr(target, parts[-1]):
continue # original fused module still in the tree; assign as-is
to_split[key] = state_dict.pop(key)
if to_split:
from auto_round.utils.model_free_utils import split_fused_expert_tensors
state_dict.update(split_fused_expert_tensors(to_split))
return state_dict
def _load_state_dict_into_module(state_dict: dict, module: torch.nn.Module) -> None:
"""Assign every key in *state_dict* to the corresponding sub-module.
Handles cleared parameters (empty tensors) and wrapper objects that store
the original layer in an ``orig_layer`` attribute.
"""
state_dict = _maybe_split_fused_expert_keys(state_dict, module)
for name, param in state_dict.items():
parts = name.split(".")
target = module
try:
for part in parts[:-1]:
target = getattr(target, part)
except AttributeError:
continue
if hasattr(target, "orig_layer"):
target = target.orig_layer
param_name = parts[-1]
if hasattr(target, param_name):
old_param = getattr(target, param_name)
if isinstance(old_param, torch.nn.Parameter):
# `old_param` is on meta when the
# module started as a meta skeleton (AR_DISK_STREAM_MODEL=1) and
# is being materialized for the first time, rather than reloaded
# onto a previously-cleared real (cpu/cuda) tensor. Target "cpu"
# in that case instead of literally copying to meta (which would
# silently discard the just-read real data).
target_device = "cpu" if old_param.device.type == "meta" else old_param.device
param = param.to(dtype=old_param.dtype, device=target_device)
setattr(target, param_name, torch.nn.Parameter(param, requires_grad=old_param.requires_grad))
else:
setattr(target, param_name, param)
del state_dict
def _clear_module_weights(
module: torch.nn.Module, cache_numel: bool = False, restorable_params: set | None = None
) -> None:
"""Replace a single module's weight/bias with empty tensors.
Args:
module: The leaf module to clear.
cache_numel: If *True*, store ``_cached_weight_numel`` and
``_cached_weight_shape`` before clearing.
restorable_params: If provided, only clear state-dict entries
(parameters **and** buffers) whose names are in this set.
Entries not in the set are kept intact because they cannot be
restored from the checkpoint (e.g. dynamically registered
calibration parameters).
"""
if module is None:
return
if hasattr(module, "orig_layer"):
return
with torch.no_grad():
for name, param in list(module.named_parameters(recurse=False)):
if param is None or param.numel() == 0:
continue
if restorable_params is not None and name not in restorable_params:
continue
if cache_numel and name == "weight":
module._cached_weight_numel = param.numel()
module._cached_weight_shape = tuple(param.shape)
setattr(
module,
name,
torch.nn.Parameter(torch.empty(0, dtype=param.dtype, device="cpu"), requires_grad=param.requires_grad),
)
for name, buf in list(module.named_buffers(recurse=False)):
if buf is None or buf.numel() == 0:
continue
if restorable_params is not None and name not in restorable_params:
continue
module.register_buffer(name, torch.empty(0, dtype=buf.dtype, device="cpu"))
# =====================================================================
# Checkpoint file helpers
# =====================================================================
def _resolve_model_dir(model_dir: str, revision: str | None = None) -> str:
"""Resolve a model name/path to a local directory containing weight files."""
if os.path.isdir(model_dir):
return model_dir
try:
from huggingface_hub import snapshot_download, try_to_load_from_cache
for filename in (
"model.safetensors.index.json",
"model.safetensors",
"pytorch_model.bin.index.json",
"pytorch_model.bin",
):
cached_file = try_to_load_from_cache(model_dir, filename, revision=revision)
if isinstance(cached_file, str):
return os.path.dirname(cached_file)
if revision is not None:
return snapshot_download(model_dir, revision=revision, local_files_only=True)
return snapshot_download(model_dir, local_files_only=True)
except Exception:
if revision is not None:
raise
return model_dir
def _build_weight_map(model_dir: str) -> dict[str, str]:
"""Build ``{tensor_name: shard_filename}`` from the model directory."""
index_path = os.path.join(model_dir, "model.safetensors.index.json")
if not os.path.exists(index_path) and os.path.isdir(model_dir):
custom_indexes = sorted(
filename for filename in os.listdir(model_dir) if filename.endswith(".safetensors.index.json")
)
if custom_indexes:
index_path = os.path.join(model_dir, custom_indexes[0])
if os.path.exists(index_path):
with open(index_path) as f:
# Shard names are declared by the checkpoint itself; validate them
# before any caller joins one onto model_dir.
return validate_weight_map(json.load(f)["weight_map"], model_dir, index_path=index_path)
single_path = os.path.join(model_dir, "model.safetensors")
if os.path.exists(single_path):
from safetensors import safe_open
with safe_open(single_path, framework="pt") as f:
return {k: "model.safetensors" for k in f.keys()}
bin_index_path = os.path.join(model_dir, "pytorch_model.bin.index.json")
if not os.path.exists(bin_index_path) and os.path.isdir(model_dir):
custom_indexes = sorted(filename for filename in os.listdir(model_dir) if filename.endswith(".bin.index.json"))
if custom_indexes:
bin_index_path = os.path.join(model_dir, custom_indexes[0])
if os.path.exists(bin_index_path):
with open(bin_index_path) as f:
return validate_weight_map(json.load(f)["weight_map"], model_dir, index_path=bin_index_path)
single_bin = os.path.join(model_dir, "pytorch_model.bin")
if os.path.exists(single_bin):
# No unrestricted fallback: this pickle comes from an untrusted artifact.
state_dict = torch.load(single_bin, map_location="cpu", weights_only=True)
return {k: "pytorch_model.bin" for k in state_dict.keys()}
raise FileNotFoundError(
f"Could not find model weight files in {model_dir}. "
"Expected a safetensors or PyTorch .bin checkpoint (with optional index.json)."
)
def load_block_from_model_files(model_dir: str, block_name: str, block: torch.nn.Module) -> None:
"""Reload a module's weights directly from the original model checkpoint.
Selectively loads only tensors belonging to *block_name* without loading
the entire model into memory.
Args:
model_dir: Path to the model directory (local or HuggingFace hub name).
block_name: The module name prefix (e.g. ``"model.layers.0"``).
block: The ``nn.Module`` to load weights into.
"""
model_dir = _resolve_model_dir(model_dir)
def _has_meta(module: torch.nn.Module) -> bool:
return any(tensor.device.type == "meta" for tensor in list(module.parameters()) + list(module.buffers()))
if _has_meta(block):
# A meta-skeleton block (disk-streamed model): resolve each parameter's
# checkpoint-side name (checkpoint-conversion renames, fused expert
# layouts) instead of raw prefix matching, which silently skips tensors
# whose checkpoint spelling differs (e.g. the router stored as
# ``mlp.router.gate`` behind a ``mlp.gate`` module) and leaves them on
# meta until a downstream ``.to(device)`` crashes.
try:
from auto_round.utils.disk_stream_util import get_safetensors_index, materialize_module
materialize_module(block, block_name, get_safetensors_index(model_dir), device="cpu")
if not _has_meta(block):
return
except FileNotFoundError:
pass # non-safetensors checkpoint: fall through to prefix loading
weight_map = _build_weight_map(model_dir)
prefix = block_name + "."
matching = {k: v for k, v in weight_map.items() if k.startswith(prefix)}
if not matching:
logger.warning(f"No tensors found for {block_name} in {model_dir}")
return
shard_to_tensors: dict[str, list[str]] = defaultdict(list)
for tensor_name, shard_file in matching.items():
shard_to_tensors[shard_file].append(tensor_name)
state_dict = {}
for shard_file, tensor_names in shard_to_tensors.items():
shard_path = str(resolve_within_directory(model_dir, shard_file))
if shard_file.endswith(".safetensors"):
from safetensors import safe_open
with safe_open(shard_path, framework="pt", device="cpu") as f:
for name in tensor_names:
state_dict[name[len(prefix) :]] = f.get_tensor(name)
else:
full_state = torch.load(shard_path, map_location="cpu", weights_only=True)
for name in tensor_names:
if name in full_state:
state_dict[name[len(prefix) :]] = full_state[name]
del full_state
_load_state_dict_into_module(state_dict, block)
# =====================================================================
# OffloadManager -- the offload manager class
# =====================================================================
class OffloadManager:
"""Module offloader for CPU RAM optimization.
Provides two core operations:
* `offload` -- save/clear a named module's weights to free CPU RAM.
* `reload` -- restore the weights back into the module.
Works at any granularity (block, layer, individual module) and supports
two operating modes:
* ``"offload"`` -- saves ``state_dict`` to a temp directory on disk
before clearing. Supports ``overwrite=True`` for re-saving modified
(e.g. quantized) weights.
* ``"clean"`` -- simply clears weights; reloads from the original
HuggingFace model checkpoint files. Zero extra disk writes but
cannot persist modifications.
The manager tracks its own state and supports automatic cleanup via
context manager (``with`` statement) or explicit `cleanup`.
Parameters
----------
enabled : bool
When *False*, every public method is a no-op.
mode : str
``"offload"`` or ``"clean"``.
model_dir : str, optional
Path to the model checkpoint directory. Required for ``"clean"``
mode; optional for ``"offload"``.
model_revision : str, optional
Commit hash used to resolve a Hugging Face model ID without relying
on a mutable branch reference.
offload_dir_prefix : str
Prefix for the temp directory name (``"offload"`` mode only).
cache_numel : bool
If *True*, cache original ``weight.numel()`` before clearing so
downstream code can still determine the original size.
"""
def __init__(
self,
enabled: bool = True,
mode: str = "offload",
model_dir: str | None = None,
offload_dir_prefix: str = "ar_offload",
cache_numel: bool = False,
retain_saved_entries: bool = False,
model_revision: str | None = None,
):
from auto_round import envs
self.enabled = enabled and not envs.AR_DISABLE_OFFLOAD
self.mode = mode
self.model_dir = (
_resolve_model_dir(model_dir, revision=model_revision)
if mode == "clean" and model_dir is not None and model_revision is not None
else model_dir
)
self.cache_numel = cache_numel
self._prefix = offload_dir_prefix
self.retain_saved_entries = retain_saved_entries
# Disk state (offload mode)
self._tempdir: str | None = None
self._saved: dict[str, dict] = {} # name -> {"save_path": str}
# Cached weight map for clean mode (avoids repeated disk I/O)
self._weight_map: dict[str, str] | None = None
# Hook state (for add_offload_hooks/remove_offload_hooks transparent offloading)
self._hook_handles: list = []
self._model_ref: torch.nn.Module | None = None
self._module_names: list[str] = []
self._last_loaded: str | None = None
# Ensure-style state (for wrapping loops)
self._current_loaded: str | None = None
# ------------------------------------------------------------------
# Context manager
# ------------------------------------------------------------------
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.cleanup()
return False
def __del__(self):
try:
self.cleanup(_skip_reload=True)
except Exception:
pass
def __call__(
self,
model: torch.nn.Module,
names: str | list[str] | list[list[str]],
*,
skip_if_saved: bool = False,
overwrite: bool = False,
clear_memory: bool = False,
device_list=None,
) -> float:
"""Alias for offload() so the manager can be invoked directly."""
return self.offload(
model,
names,
skip_if_saved=skip_if_saved,
overwrite=overwrite,
clear_memory=clear_memory,
device_list=device_list,
)
# ------------------------------------------------------------------
# Core API
# ------------------------------------------------------------------
def offload(
self,
model: torch.nn.Module,
names: str | list[str] | list[list[str]],
*,
skip_if_saved: bool = False,
overwrite: bool = False,
clear_memory: bool = False,
device_list=None,
) -> float:
"""Offload one or more named modules' weights to free CPU RAM.
Accepts a single module name, a flat list, or a nested list of names.
For ``"offload"`` mode: saves state_dict to disk, then clears weights.
For ``"clean"`` mode: simply clears weights (will reload from
model files later).
Args:
model: The root model containing the module(s).
names: Module name(s) -- a single string, a list of strings,
or a nested list (e.g. ``all_blocks``).
skip_if_saved: If *True* and a name was already saved, skip it.
overwrite: If *True*, discard any previous save and re-save the
current (possibly modified) weights before clearing. Useful
after quantization when updated weights must be persisted.
Only meaningful for ``"offload"`` mode.
clear_memory: If *True*, call ``clear_memory()`` after offloading
all modules and log the total freed size.
device_list: Device list passed to ``clear_memory`` (only used
when *clear_memory* is *True*).
Returns:
Total offloaded size in GB (non-zero only when *clear_memory* is True).
"""
if not self.enabled:
return 0.0
if self.mode == "offload" and not self._check_disk_space(model, names):
self.enabled = False
return 0.0
if isinstance(names, str):
self._offload(model, names, skip_if_saved=skip_if_saved, overwrite=overwrite)
return 0.0
flat_names = self._flatten_names(names)
if clear_memory:
logger.info("offloading module weights...")
total_gb = 0.0
for name in flat_names:
if skip_if_saved and self.has(name):
continue
if clear_memory:
module = get_module(model, name)
if module is not None:
total_gb += self.estimate_module_size_gb(module)
self._offload(model, name, skip_if_saved=skip_if_saved, overwrite=overwrite)
if clear_memory:
from auto_round.utils import clear_memory as _clear_memory
_clear_memory(device_list=device_list)
logger.info(f"offload done, freed {total_gb:.2f} GB")
return total_gb
def _check_disk_space(self, model: torch.nn.Module, names: str | list[str] | list[list[str]]) -> bool:
"""Check whether there is enough disk space to offload the given modules.
Args:
model: The root model containing the module(s).
names: Module name(s) to check.
Returns:
True if sufficient disk space is available, False otherwise.
"""
if isinstance(names, str):
flat_names = [names]
else:
flat_names = self._flatten_names(names)
total_bytes = 0
for name in flat_names:
module = get_module(model, name)
if module is None:
continue
# Estimate size based on state_dict (parameters + buffers), excluding meta tensors,
# to match what is actually written by the offload logic.
state_dict = module.state_dict()
for tensor in state_dict.values():
if not isinstance(tensor, torch.Tensor):
continue
if tensor.is_meta or tensor.numel() == 0:
continue
total_bytes += tensor.numel() * tensor.element_size()
# torch.save adds serialization overhead; use 1.2x safety margin
required_bytes = int(total_bytes * 1.2)
from auto_round import envs
target_dir = os.path.join(envs.AR_WORK_SPACE, "offload")
os.makedirs(target_dir, exist_ok=True)
free_bytes = shutil.disk_usage(target_dir).free
if free_bytes < required_bytes:
required_gb = required_bytes / (1024**3)
free_gb = free_bytes / (1024**3)
logger.warning(
f"Insufficient disk space for offloading: need ~{required_gb:.2f} GB "
f"(including safety margin) but only {free_gb:.2f} GB available at {target_dir}. Skipping offload."
)
return False
return True
def _offload(
self, model: torch.nn.Module, name: str, *, skip_if_saved: bool = False, overwrite: bool = False
) -> None:
"""Offload a single named module (internal helper)."""
self._model_ref = model
module = get_module(model, name)
if module is None:
return
if self.mode == "offload":
if overwrite:
old_meta = self._saved.pop(name, None)
if old_meta:
old_path = old_meta.get("save_path")
if old_path and os.path.exists(old_path):
try:
os.remove(old_path)
except Exception as e:
logger.warning(f"OffloadManager: could not remove {old_path}: {e}")
elif skip_if_saved and name in self._saved:
return
self._save_to_disk(name, module)
self._clear(module, block_name=name)
def reload(self, model: torch.nn.Module, names: str | list[str] | None = None) -> None:
"""Reload previously offloaded module(s).
For ``"offload"`` mode: loads from the temp directory, then
removes the temp file. When all offloaded modules have been
reloaded the temp directory is automatically deleted.
For ``"clean"`` mode: loads from original model files.
Args:
model: The root model containing the module(s).
names: A single module name, a list of names, or *None*
to reload all tracked modules.
"""
if not self.enabled:
return
if names is None:
if self.mode == "offload":
names = list(self._saved.keys())
else:
names = list(self._module_names)
if isinstance(names, str):
self._reload(model, names)
return
for name in names:
self._reload(model, name)
def _reload(self, model: torch.nn.Module, name: str) -> None:
"""Reload a single named module (internal helper)."""
module = get_module(model, name)
if module is None:
return
model_dir = self.model_dir
component_subfolder = getattr(model, "_autoround_checkpoint_subfolder", None)
if model_dir is not None and component_subfolder:
resolved_dir = _resolve_model_dir(model_dir)
component_dir = os.path.join(resolved_dir, component_subfolder)
if os.path.isdir(component_dir):
model_dir = component_dir
if self.mode == "offload":
if name not in self._saved:
# Before falling back to the
# original checkpoint, check whether a *prior process* already
# offloaded this block to the deterministic resume directory
# (see _ensure_dir()) -- this is what makes bare in-memory
# .quantize() resumability actually work: a resumed process's
# self._saved starts empty (it's an in-memory dict, not
# persisted), so without this check it would always look like
# nothing was ever offloaded, even when a crashed prior
# process's real work is sitting right there on disk.
from auto_round import envs
if envs.AR_RESUME_DIR:
safe_name = name.replace(".", "_")
candidate_path = os.path.join(self._ensure_dir(), f"{safe_name}.safetensors")
if os.path.exists(candidate_path):
self._saved[name] = {"save_path": candidate_path}
self._load_from_disk(name, module)
if not self.retain_saved_entries:
self._remove_saved_entry(name)
return
# This block was never actually offloaded with real data
# (either it's still on meta, or `_save_to_disk` found nothing
# real to save), which happens when the model started as a
# meta skeleton (AR_DISK_STREAM_MODEL=1) instead of a full CPU
# load. There is nothing on the temp dir to load from -- read
# straight from the original checkpoint instead.
if model_dir is not None:
load_block_from_model_files(model_dir, name, module)
return
self._load_from_disk(name, module)
if not self.retain_saved_entries:
self._remove_saved_entry(name)
else:
if self.model_dir is None:
logger.warning("OffloadManager: model_dir is required for clean mode")
return
load_block_from_model_files(model_dir, name, module)
# ------------------------------------------------------------------
# Hook-based transparent offloading
# ------------------------------------------------------------------
def add_offload_hooks(self, model: torch.nn.Module, names: list[str]) -> None:
"""Clear all named modules and register pre-forward hooks for
transparent reload-on-demand.
In ``"clean"`` mode, only parameters that exist in the original
checkpoint are cleared. Dynamically registered parameters (e.g.
calibration scales from attention/KV-cache quantization) are
preserved so they remain valid across reload cycles.
Args:
model: The root model.
names: List of module names to manage.
"""
if not self.enabled:
return
self._model_ref = model
self._module_names = list(names)
# Register hooks
for name in names:
module = get_module(model, name)
if module is None:
continue
handle = module.register_forward_pre_hook(partial(self._pre_forward_hook, name=name))
self._hook_handles.append(handle)
# Clear all
logger.info("clearing module weights to free RAM...")
for name in names:
self._offload(model, name, skip_if_saved=True)
gc.collect()
from auto_round.utils import clear_memory
clear_memory()
logger.info("module weights cleared")
def remove_offload_hooks(self, model: torch.nn.Module, names: list[str] | None = None) -> None:
"""Remove hooks and reload all managed modules.
Args:
model: The root model.
names: Module names to reload. If None, uses the names from
`add_offload_hooks`.
"""
self._remove_hooks()
if names is None:
names = self._module_names
self.reload(model, names)
self._model_ref = None
def _pre_forward_hook(self, module: torch.nn.Module, args, *, name: str) -> None:
"""Pre-forward hook: reload this module if cleared, clear previous."""
if not self._needs_loading(module):
return
# Clear previous to keep only one module loaded at a time
if self._last_loaded is not None and self._last_loaded != name:
prev = get_module(self._model_ref, self._last_loaded)
if prev is not None:
self._clear(prev, block_name=self._last_loaded)
self.reload(self._model_ref, name)
self._last_loaded = name
def _remove_hooks(self) -> None:
for handle in self._hook_handles:
handle.remove()
self._hook_handles.clear()
self._last_loaded = None
# ------------------------------------------------------------------
# Ensure-style API (for wrapping loops)
# ------------------------------------------------------------------
def ensure_loaded(self, model: torch.nn.Module, layer_name: str) -> None:
"""Ensure the module containing *layer_name* is loaded.
When the loop moves to a layer in a different module, the previous
module is automatically cleared. Call `flush_loaded` after
the loop to release the last module.
Args:
model: The root model.
layer_name: Full layer name (e.g. ``"model.layers.0.attn.q"``).
"""
if not self.enabled or not self._module_names:
return
target = None
for mn in self._module_names:
if layer_name.startswith(mn + "."):
target = mn
break
if target is None:
return
if target == self._current_loaded:
return
# Clear previous
if self._current_loaded is not None:
module = get_module(model, self._current_loaded)
if module is not None:
self._clear(module, block_name=self._current_loaded)
# Load new
self.reload(model, target)
self._current_loaded = target
def flush_loaded(self, model: torch.nn.Module) -> None:
"""Clear the last module loaded by `ensure_loaded`."""
if self._current_loaded is not None:
module = get_module(model, self._current_loaded)
if module is not None:
self._clear(module, block_name=self._current_loaded)
self._current_loaded = None
# ------------------------------------------------------------------
# State queries
# ------------------------------------------------------------------
def has(self, name: str) -> bool:
"""Return *True* if *name* has been offloaded."""
if self.mode == "offload":
return name in self._saved
return False
def reset(self) -> None:
"""Clear tracking state (temp directory is kept)."""
self._saved = {}
self._current_loaded = None
self._last_loaded = None
def cleanup(self, _skip_reload: bool = False) -> None:
"""Reload remaining modules and remove temp files.
Normally this is called automatically -- ``reload`` deletes each
temp file as it goes and removes the temp directory when done.
Explicit calls are usually unnecessary; the context-manager exit
and ``__del__`` use this as a fallback.
"""
self._remove_hooks()
# Auto-reload offloaded modules if we still have a model reference
if not _skip_reload and self._model_ref is not None and self._saved:
try:
self.reload(self._model_ref)
except Exception as e:
logger.warning(f"OffloadManager: auto-reload during cleanup failed: {e}")
self._cleanup_tempdir()
self._saved = {}
self._current_loaded = None
self._module_names = []
self._model_ref = None
# ------------------------------------------------------------------
# Static estimation helpers
# ------------------------------------------------------------------
@staticmethod
def estimate_tensor_size_gb(tensor: Any) -> float:
"""Estimate size of a tensor (or nested container) in GB."""
if tensor is None:
return 0.0
if isinstance(tensor, torch.Tensor):
return tensor.numel() * tensor.element_size() / (1024**3)
if isinstance(tensor, list):
return sum(OffloadManager.estimate_tensor_size_gb(t) for t in tensor)
if isinstance(tensor, dict):
return sum(OffloadManager.estimate_tensor_size_gb(v) for v in tensor.values())
return 0.0
@staticmethod
def estimate_inputs_size_gb(all_inputs: dict) -> float:
"""Estimate total size of calibration inputs in GB."""
return sum(OffloadManager.estimate_tensor_size_gb(v) for v in all_inputs.values())
@staticmethod
def estimate_model_size_gb(model: torch.nn.Module) -> float:
"""Estimate model weights size in GB."""
return sum(p.numel() * p.element_size() / (1024**3) for p in model.parameters() if p.numel() > 0)
@staticmethod
def estimate_module_size_gb(module: torch.nn.Module) -> float:
"""Estimate a module's parameter size in GB."""
return sum(p.numel() * p.element_size() / (1024**3) for p in module.parameters() if p.numel() > 0)
# ------------------------------------------------------------------
# Internal: disk operations (offload mode)
# ------------------------------------------------------------------
def _ensure_dir(self) -> str:
if self._tempdir is None:
from auto_round import envs
base_dir = os.path.join(envs.AR_WORK_SPACE, "offload")
os.makedirs(base_dir, exist_ok=True)
if envs.AR_RESUME_DIR:
# A fresh tempfile.mkdtemp()
# directory is unique to this process and can never be found
# again by a resumed run in a new process -- that's the whole
# reason bare in-memory .quantize() (no format=) resumability
# didn't actually work: ResumeState correctly skipped
# re-tuning already-done blocks, but their quantized weights,
# offloaded here, were unreachable from the resumed process,
# leaving those blocks on meta in the returned model. Use a
# stable, deterministic path instead whenever AR_RESUME_DIR is
# set, so a resumed process's OffloadManager can find (see
# _reload()'s discovery check below) and reuse what a prior
# crashed process already saved here.
self._tempdir = os.path.join(base_dir, f"{self._prefix}_resume")
os.makedirs(self._tempdir, exist_ok=True)
else:
self._tempdir = tempfile.mkdtemp(prefix=f"{self._prefix}_", dir=base_dir)
logger.info(f"OffloadManager ({self._prefix}): tempdir = {self._tempdir}")
return self._tempdir
def _save_to_disk(self, name: str, module: torch.nn.Module) -> None:
tmpdir = self._ensure_dir()
safe_name = name.replace(".", "_")
save_path = os.path.join(tmpdir, f"{safe_name}.safetensors")
try:
from safetensors.torch import save_file as safe_save_file
# Skip meta tensors: they contain no real data (e.g. quantized weights
# already flushed to disk by an immediate-saving shard writer).
state_dict = {
k: v.cpu().contiguous()
for k, v in module.state_dict().items()
if isinstance(v, torch.Tensor) and v.device.type != "meta"
}
if not state_dict:
# Nothing real to save -- the
# module was still on meta (e.g. the model started as a meta
# skeleton under AR_DISK_STREAM_MODEL=1 and this block hasn't
# been touched yet). Do NOT record it in self._saved: a later
# reload() must fall through to materializing straight from
# the original checkpoint (see _reload), not silently load an
# empty file and leave the block on meta.
return
safe_save_file(state_dict, save_path)
self._saved[name] = {"save_path": save_path}
del state_dict
except Exception as e:
# Keeping the module resident in memory after a failed save (e.g. disk
# quota exceeded) silently increases memory pressure and can lead to a
# confusing OOM later on, so fail fast instead.
logger.error(f"OffloadManager: failed to save {name}: {e}. Exiting to avoid silent memory growth.")
sys.exit(1)
def _load_from_disk(self, name: str, module: torch.nn.Module) -> None:
metadata = self._saved.get(name)
if not metadata:
return
save_path = metadata["save_path"]
if not os.path.exists(save_path):
logger.warning(f"OffloadManager: file not found {save_path}")
return
try:
from safetensors.torch import load_file as safe_load_file
state_dict = safe_load_file(save_path, device="cpu")
# Replace every tensor with a private copy so the mapping is
# released as soon as this method returns.
for key in list(state_dict.keys()):
tensor = state_dict[key]
state_dict[key] = tensor.clone()
del tensor
_load_state_dict_into_module(state_dict, module)
except Exception as e:
logger.warning(f"OffloadManager: failed to load {name}: {e}")
def _remove_saved_entry(self, name: str) -> None:
"""Remove a single saved entry and its temp file; clean tempdir if empty."""
meta = self._saved.pop(name, None)
if meta:
path = meta.get("save_path")
if path and os.path.exists(path):
try:
os.remove(path)
except Exception as e:
logger.warning(f"OffloadManager: could not remove {path}: {e}")
if not self._saved:
self._cleanup_tempdir()
def _cleanup_tempdir(self) -> None:
"""Remove the temp directory if it exists."""
if self._tempdir and os.path.isdir(self._tempdir):
parent = os.path.dirname(self._tempdir)
try:
shutil.rmtree(self._tempdir)
except Exception as e:
logger.warning(f"OffloadManager: cleanup failed for {self._tempdir}: {e}")
# Remove the parent offload/ directory if it is now empty
if parent and os.path.isdir(parent):
try:
os.rmdir(parent)
except OSError:
pass # not empty or already removed
self._tempdir = None
# ------------------------------------------------------------------
# Internal: clearing
# ------------------------------------------------------------------
def _get_restorable_params(self, block_name: str) -> set[str] | None:
"""Return the set of parameter names under *block_name* that exist in
the checkpoint, or *None* if the weight map is unavailable.
In ``"clean"`` mode the offload manager reloads weights from the
original checkpoint. Parameters dynamically registered at runtime
(e.g. calibration scales) do **not** appear in the checkpoint, so
clearing them would leave them as empty tensors with no way to
restore. This helper builds the set of parameters that *can* be
restored, allowing ``_clear`` to skip the rest.
"""
if self.mode != "clean" or not self.model_dir:
return None
try:
if self._weight_map is None:
model_dir = _resolve_model_dir(self.model_dir)
self._weight_map = _build_weight_map(model_dir)
weight_map = self._weight_map
except Exception as e:
logger.warning(
f"OffloadManager: failed to build weight map, skipping clear to preserve dynamic params: {e}"
)
return set()
prefix = block_name + "."
return {k[len(prefix) :] for k in weight_map if k.startswith(prefix)}
def _clear(self, module: torch.nn.Module, block_name: str | None = None) -> None:
"""Clear weight/bias tensors in module and its sub-modules.
In ``"clean"`` mode with a known block_name, only parameters that
exist in the original checkpoint are cleared. Dynamically registered
parameters (e.g. calibration scales) are preserved.
"""
restorable = self._get_restorable_params(block_name) if block_name else None
for name, submodule in module.named_modules():
if restorable is not None:
# Build the set of param basenames that are restorable for this submodule
sub_prefix = (name + ".") if name else ""
sub_restorable = {
k[len(sub_prefix) :]
for k in restorable
if k.startswith(sub_prefix) and "." not in k[len(sub_prefix) :]
}
else:
sub_restorable = None
_clear_module_weights(submodule, cache_numel=self.cache_numel, restorable_params=sub_restorable)
@staticmethod
def _needs_loading(module: torch.nn.Module) -> bool:
"""Return *True* if any parameter in *module* has been cleared."""
for submodule in module.modules():
if hasattr(submodule, "orig_layer"):
submodule = submodule.orig_layer
for param in submodule.parameters(recurse=False):
if param is not None and param.numel() == 0:
return True
return False
@staticmethod
def _flatten_names(names: list[str] | list[list[str]]) -> list[str]:
"""Flatten a potentially nested list of names."""
flat = []
for item in names:
if isinstance(item, list):
flat.extend(item)
else:
flat.append(item)
return flat