PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
# Lazy, mmap-backed reads of individual tensors by name straight from a
# checkpoint's safetensors shards, plus meta<->real materialize/free for a
# whole module. Used by auto_round/auto_scheme/delta_loss.py's streaming
# scoring path (get_score_for_scheme_streaming) to materialize one decoder
# block's real tensors right before scoring it and release them back to meta
# right after -- instead of loading the entire model onto CPU up front, which
# doesn't fit when the checkpoint is larger than available RAM+VRAM combined.
from __future__ import annotations
import json
import logging
import re
from functools import cache, lru_cache
from pathlib import Path
import torch
import torch.nn as nn
from accelerate.utils import set_module_tensor_to_device
from safetensors import SafetensorError, safe_open
from auto_round.utils.path_safety import resolve_within_directory, validate_weight_map
logger = logging.getLogger(__name__)
class SafetensorsIndex:
"""Lazy, mmap-backed access to a checkpoint's tensors by name.
Deliberately does NOT cache open safe_open() handles across calls: mmap'd pages
stay resident (counted in RSS) for as long as the mapping is open, even after
the torch tensor copied out of them is freed. For a checkpoint far larger than
available RAM, caching handles indefinitely would silently re-create the exact
problem this module exists to avoid (RSS creeping up by however much of the
file has been touched so far, instead of staying bounded to one block at a
time). Every read (or batch of reads via read_tensors) opens a shard, reads,
and closes/unmaps before returning.
"""
def __init__(self, checkpoint_dir: str):
self.checkpoint_dir = Path(checkpoint_dir)
index_path = self.checkpoint_dir / "model.safetensors.index.json"
if index_path.exists():
with open(index_path) as f:
# The shard names come from the checkpoint's own index, i.e. from the
# artifact being loaded -- validate them against the directory before
# anything downstream gets a chance to join and open them.
self.weight_map: dict[str, str] = validate_weight_map(
json.load(f)["weight_map"], self.checkpoint_dir, index_path=index_path
)
else:
# Small, unsharded checkpoint: one model.safetensors file.
single_file = self.checkpoint_dir / "model.safetensors"
with safe_open(str(single_file), framework="pt") as f:
self.weight_map = {name: single_file.name for name in f.keys()}
def has_tensor(self, name: str) -> bool:
return name in self.weight_map
def read_tensor(self, name: str, device: str = "cpu") -> torch.Tensor:
return self.read_tensors([name], device=device)[name]
def tensor_shape(self, name: str) -> tuple[int, ...]:
"""Return a tensor's shape from its safetensors header without reading its payload."""
shard_name = self.weight_map[name]
shard_path = resolve_within_directory(self.checkpoint_dir, shard_name)
with safe_open(str(shard_path), framework="pt") as f:
return tuple(f.get_slice(name).get_shape())
def read_tensors(self, names: list[str], device: str = "cpu") -> dict[str, torch.Tensor]:
"""Read several tensors, grouped by shard file so each shard is opened and
closed (unmapped) once regardless of how many tensors are pulled from it."""
by_shard: dict[str, list[str]] = {}
for name in names:
by_shard.setdefault(self.weight_map[name], []).append(name)
result: dict[str, torch.Tensor] = {}
for shard_name, shard_tensor_names in by_shard.items():
shard_path = resolve_within_directory(self.checkpoint_dir, shard_name)
with safe_open(str(shard_path), framework="pt") as f:
for name in shard_tensor_names:
tensor = f.get_tensor(name)
if device != "cpu":
tensor = tensor.to(device)
result[name] = tensor
return result
def tensor_names_with_prefix(self, prefix: str) -> list[str]:
dotted = prefix if prefix.endswith(".") else prefix + "."
return [n for n in self.weight_map if n == prefix or n.startswith(dotted)]
@lru_cache(maxsize=8)
def get_safetensors_index(checkpoint_dir: str) -> SafetensorsIndex:
"""Shared ``SafetensorsIndex`` per checkpoint directory.
Only the (cheap) name->shard map is cached; no ``safe_open`` handle is kept, so this
does not reintroduce the mmap-residency problem ``SafetensorsIndex`` avoids. Callers
used to build a fresh index for every block, re-reading the index JSON 40+ times.
"""
return SafetensorsIndex(str(checkpoint_dir))
def checkpoint_has_native_fused_moe_experts(checkpoint_dir: str) -> bool:
"""Whether a checkpoint directly stores supported fused 3D expert tensors.
Unlike :func:`config_has_fused_moe_experts`, which detects experts fused by a
Transformers checkpoint converter, this inspects the checkpoint layout itself.
Only the ``gate_up_proj`` + ``down_proj`` layout understood by
:func:`materialize_module` is accepted. Tensor shapes come from safetensors
headers, so this check never maps or copies the weight payloads.
"""
try:
index = get_safetensors_index(str(checkpoint_dir))
except (OSError, ValueError, KeyError, SafetensorError):
return False
gate_up_suffix = ".experts.gate_up_proj"
for gate_up_name in index.weight_map:
if not gate_up_name.endswith(gate_up_suffix):
continue
prefix = gate_up_name[: -len(gate_up_suffix)]
down_name = f"{prefix}.experts.down_proj"
if not index.has_tensor(down_name):
continue
try:
gate_up_shape = index.tensor_shape(gate_up_name)
down_shape = index.tensor_shape(down_name)
except (OSError, ValueError, KeyError, SafetensorError):
continue
if len(gate_up_shape) == 3 and len(down_shape) == 3 and gate_up_shape[0] == down_shape[0] > 1:
logger.debug(
"Native fused-MoE checkpoint tensors detected: %s %s and %s %s",
gate_up_name,
gate_up_shape,
down_name,
down_shape,
)
return True
return False
# Model-side module names sometimes differ from the checkpoint-side tensor
# names. Transformers already maintains the authoritative per-family renames
# (``transformers/conversion_mapping.py``, checkpoint -> model direction);
# the disk-stream materializers know only model-side names, so those entries
# are inverted here. A rename is applied only when the model-side name is
# absent from the index AND the renamed candidate exists in it, so checkpoints
# whose names already match the model are untouched.
@cache
def _reverse_renamings_for(model_type):
"""Invert the checkpoint-conversion WeightRenaming entries for one family.
``transformers`` owns the checkpoint->model renames, so reversing them is best left to
``WeightTransform.reverse_transform()``: it handles capturing groups and the
``PrefixChange`` special cases correctly. Naively reusing a ``target_pattern`` as a
regex pattern does not -- the targets are *replacement* strings and a ``\\1``
backreference in one (e.g. ``model.language_model.\\1``) raises
``re.PatternError: invalid group reference``.
Returns reversed ``WeightTransform`` objects; call ``rename_source_key(model_side_name)``
on them to get the checkpoint-side name.
"""
if not model_type:
return ()
try:
from transformers.conversion_mapping import WeightRenaming, get_checkpoint_conversion_mapping
except ImportError: # pragma: no cover - transformers is a hard dep, guarded for safety
return ()
mapping = get_checkpoint_conversion_mapping(model_type)
if not mapping:
return ()
reversed_transforms = []
for entry in mapping:
if not isinstance(entry, WeightRenaming):
# Other converter types (expert fusions) are handled separately, by
# `_renamed_expert_candidates` and `materialize_module`'s fused lookup.
continue
try:
reversed_transforms.append(entry.reverse_transform())
except Exception as exc: # pragma: no cover - not every transform is reversible
logger.debug("skipping non-reversible rename %s: %s", entry, exc)
return tuple(reversed_transforms)
@cache
def _model_types_for_dir(checkpoint_dir: str):
"""Every model_type in the checkpoint's config, including nested sub-configs.
A VLM config nests the text model (``text_config.model_type = 'qwen3_5_moe_text'``)
and that is where the MoE expert conversion rules are registered, so the top-level
``model_type`` alone is not enough to resolve expert tensor names.
Keyed by directory rather than by ``SafetensorsIndex`` instance: callers build a fresh
index per block, so caching on the object would both miss every time and pin every
index (and its ``weight_map``) in memory for the whole run.
"""
try:
with open(Path(checkpoint_dir) / "config.json") as f:
config = json.load(f)
except OSError:
return ()
found: list[str] = []
def _walk(node):
if not isinstance(node, dict):
return
model_type = node.get("model_type")
if isinstance(model_type, str) and model_type not in found:
found.append(model_type)
for value in node.values():
if isinstance(value, dict):
_walk(value)
_walk(config)
return tuple(found)
def _index_model_type(index):
model_types = _model_types_for_dir(str(getattr(index, "checkpoint_dir", None)))
return model_types[0] if model_types else None
def _index_model_types(index):
# Some lightweight/test indices carry no ``checkpoint_dir``; degrade to no
# known model types rather than raising AttributeError.
return _model_types_for_dir(str(getattr(index, "checkpoint_dir", None)))
# Model-side per-expert projection -> (fused checkpoint projection, index within it).
# ``gate_proj``/``up_proj`` come from splitting a fused ``gate_up_proj`` in that order,
# which matches the order of the corresponding WeightConverter ``source_patterns``.
_MODEL_PROJ_TO_FUSED = {
"gate_proj": ("gate_up_proj", 0),
"up_proj": ("gate_up_proj", 1),
"down_proj": ("down_proj", 0),
}
def _renamed_expert_candidates(index, full_name: str):
"""Checkpoint-name candidates for an unfused per-expert parameter."""
match = _MODEL_SIDE_EXPERT_RE.match(full_name)
if match is None:
return
fused_position = _MODEL_PROJ_TO_FUSED.get(match["proj"])
if fused_position is None:
return
prefix, expert = match["prefix"], match["expert"]
for model_type in _index_model_types(index):
for key, suffix in _expert_projection_renames_for(model_type):
if key != fused_position:
continue
# "w1.weight" keeps the checkpoint's own attribute name; drop it for a bias.
suffix_attr = suffix.rsplit(".", 1)
if len(suffix_attr) == 2:
suffix = f"{suffix_attr[0]}.{match['attr']}"
yield f"{prefix}.{expert}.{suffix}"
# On-disk layout of fused expert tensors, keyed by checkpoint ``model_type``.
# Two axes vary between families:
# * ``checkpoint_transposed`` -- the per-expert 2-D slice is stored as
# ``[in_features, out_features]`` (matmul layout, ``x @ W``) instead of the
# ``nn.Linear`` ``[out_features, in_features]`` weight layout, so it must be
# transposed before being loaded into the unfused per-expert Linear.
# * ``gate_up_interleaved`` -- the fused ``gate_up_proj`` interleaves the gate
# and up columns (``gate = fused[..., 0::2]``, ``up = fused[..., 1::2]``)
# instead of storing them as two contiguous halves.
# The transpose axis is normally inferred from the *target* parameter shape and
# does not need this table; the table is only consulted (a) to pick the
# interleave axis, which is invisible to shapes, and (b) as a tiebreak for the
# rare square/ambiguous shapes where transposed and non-transposed slices share
# the same shape. Families not listed here default to the non-transposed,
# contiguous ``[N, 2*inter, hidden]`` layout (qwen3_moe / phimoe / mixtral-style).
_FUSED_EXPERT_LAYOUTS = {
# model_type: (checkpoint_transposed, gate_up_interleaved)
"gpt_oss": (True, True),
"llama4": (True, False),
}
def _fused_expert_layout(index) -> tuple[bool, bool]:
"""Return the ``(checkpoint_transposed, gate_up_interleaved)`` hint for ``index``.
The transpose flag is only a *hint* (shape inference takes precedence); the
interleave flag is authoritative since it cannot be recovered from shapes.
Unknown model types default to ``(False, False)``.
"""
for model_type in _index_model_types(index):
if model_type in _FUSED_EXPERT_LAYOUTS:
return _FUSED_EXPERT_LAYOUTS[model_type]
return (False, False)
def _slice_fused_expert(fused, proj, attr, is_gate_up, target_shape, transposed_hint, interleaved):
"""Slice one unfused per-expert tensor out of a fused per-expert tensor.
``fused`` is a single expert's slice of the fused checkpoint tensor (2-D for a
weight, 1-D for a bias). ``target_shape`` is the shape the unfused
``nn.Linear`` parameter expects. Whether the checkpoint stores the weight
transposed is inferred from ``target_shape`` first; ``transposed_hint`` only
breaks ambiguous (square) ties, and ``interleaved`` selects the gate/up split
order. Returns ``None`` when no slice can produce ``target_shape`` (the caller
then leaves the parameter on meta and reports an actionable error).
"""
if attr == "bias":
# 1-D per-expert bias; only gate/up need splitting.
if not is_gate_up:
return fused.contiguous()
if interleaved:
return (fused[0::2] if proj == "gate_proj" else fused[1::2]).contiguous()
half = fused.shape[0] // 2
return (fused[:half] if proj == "gate_proj" else fused[half:]).contiguous()
if fused.ndim != 2:
return None
if not is_gate_up: # down_proj weight, target (out, in)
reversed_shape = (target_shape[1], target_shape[0])
if target_shape == reversed_shape:
transposed = transposed_hint # square: shape can't decide
elif tuple(fused.shape) == target_shape:
transposed = False
elif tuple(fused.shape) == reversed_shape:
transposed = True
else:
return None
return fused.t().contiguous() if transposed else fused.contiguous()
# gate/up weight, target (out, in) = (inter, hidden)
out, in_features = target_shape
non_transposed = tuple(fused.shape) == (2 * out, in_features)
transposed = tuple(fused.shape) == (in_features, 2 * out)
if non_transposed and transposed:
transposed = transposed_hint # square-ish (2*out == in): shape can't decide
non_transposed = not transposed
if non_transposed:
return (fused[:out] if proj == "gate_proj" else fused[out:]).contiguous()
if transposed:
# fused is [in_features, 2*out]; split along the 2*out dim, then transpose
# back to the [out, in_features] nn.Linear weight layout.
if interleaved:
sliced = fused[:, 0::2] if proj == "gate_proj" else fused[:, 1::2]
else:
sliced = fused[:, :out] if proj == "gate_proj" else fused[:, out:]
return sliced.t().contiguous()
return None
_MODEL_SIDE_EXPERT_RE = re.compile(
r"^(?P<prefix>.*\.experts)\.(?P<expert>\d+)\.(?P<proj>[A-Za-z0-9_]+)\.(?P<attr>weight|bias)$"
)
@cache
def _expert_projection_renames_for(model_type):
"""Map a fused projection to the checkpoint-side per-expert projection names.
``transformers`` describes MoE experts with a ``WeightConverter`` whose
``source_patterns`` are the per-expert checkpoint keys and whose single
``target_pattern`` is the fused model-side parameter, e.g. for phimoe::
source_patterns = [".experts.*.w1.weight", ".experts.*.w3.weight"]
target_patterns = ".experts.gate_up_proj"
After AutoRound unfuses the experts the model-side names are
``experts.<i>.gate_proj.weight`` / ``experts.<i>.up_proj.weight``, so we need
``(gate_up_proj, 0) -> "w1.weight"`` and ``(gate_up_proj, 1) -> "w3.weight"`` to find
the tensors back on disk. Returns a tuple of ``((fused, index), suffix)`` pairs.
"""
if not model_type:
return ()
try:
from transformers.conversion_mapping import get_checkpoint_conversion_mapping
from transformers.core_model_loading import MergeModulelist, WeightConverter
except ImportError: # pragma: no cover - transformers < 5
return ()
mapping = get_checkpoint_conversion_mapping(model_type)
if not mapping:
return ()
renames: dict[tuple[str, int], str] = {}
for entry in mapping:
if not isinstance(entry, WeightConverter):
continue
# Only expert merges (`experts.*.<proj>` -> fused 3D) are relevant here.
if not any(isinstance(op, MergeModulelist) for op in entry.operations):
continue
if len(entry.target_patterns) != 1:
continue
fused = entry.target_patterns[0].rstrip("$").rsplit(".", 1)[-1]
for position, source_pattern in enumerate(entry.source_patterns):
# ".experts.*.w1.weight" -> "w1.weight"
_, star, suffix = source_pattern.partition("*.")
if not star:
continue
renames.setdefault((fused, position), suffix.rstrip("$"))
return tuple(renames.items())
@cache
def _concat_converters_for(model_type):
"""Model-side params assembled by concatenating several checkpoint tensors.
Some families describe a single model-side parameter as the concatenation of
several separately-stored checkpoint tensors, via a ``WeightConverter`` whose
``operations`` contain a ``Concatenate`` (and *no* ``MergeModulelist`` -- that is the
expert-fusion case, handled elsewhere). GLM-5.3-Flash (``glm5_next``) is the motivating
example: ``self_attn.conv1d.weight`` is ``cat([q_conv1d.weight, k_conv1d.weight,
v_conv1d.weight], dim=0)``. The meta skeleton only knows the fused model-side name, so
map it back to its checkpoint-side sources to materialize it.
Returns a tuple of ``(target_suffix, (source_suffixes...), concat_dim)`` triples, using
only literal (wildcard-free) patterns -- wildcard/expert converters are handled by the
expert-specific helpers.
"""
if not model_type:
return ()
try:
from transformers.conversion_mapping import get_checkpoint_conversion_mapping
from transformers.core_model_loading import Concatenate, MergeModulelist, WeightConverter
except ImportError: # pragma: no cover - transformers < 5
return ()
mapping = get_checkpoint_conversion_mapping(model_type)
if not mapping:
return ()
converters = []
for entry in mapping:
if not isinstance(entry, WeightConverter):
continue
# Expert fusions (MergeModulelist) are handled by `_renamed_expert_candidates`
# and `materialize_module`'s fused lookup.
if any(isinstance(op, MergeModulelist) for op in entry.operations):
continue
concat = next((op for op in entry.operations if isinstance(op, Concatenate)), None)
if concat is None or len(entry.target_patterns) != 1:
continue
target = entry.target_patterns[0].rstrip("$")
sources = tuple(src.rstrip("$") for src in entry.source_patterns)
# Only literal patterns here; wildcard (`*`) converters are per-expert.
if "*" in target or any("*" in src for src in sources):
continue
converters.append((target, sources, getattr(concat, "dim", 0)))
return tuple(converters)
def _resolve_checkpoint_name(index, full_name: str):
"""Map a model-side parameter name to its checkpoint-side tensor name."""
if index.has_tensor(full_name):
return full_name
# Unfused MoE experts whose checkpoint spells the projections differently
# (e.g. phimoe's w1/w2/w3, mixtral-style checkpoints).
for candidate in _renamed_expert_candidates(index, full_name):
if index.has_tensor(candidate):
return candidate
for model_type in _index_model_types(index):
transforms = _reverse_renamings_for(model_type)
for transform in transforms:
try:
candidate, _ = transform.rename_source_key(full_name)
except Exception as exc: # pragma: no cover - defensive, never fail a load here
logger.debug("rename %s via %s failed: %s", full_name, transform, exc)
continue
if candidate != full_name and index.has_tensor(candidate):
return candidate
# Some names need several of a family's renames at once, e.g. RADIO's
# ``encoder.layer.N.attention.output.dense`` <- ``radio_model.model.blocks.N.attn.proj``.
candidate = full_name
for transform in transforms:
try:
candidate, _ = transform.rename_source_key(candidate)
except Exception as exc: # pragma: no cover - defensive, never fail a load here
logger.debug("chained rename %s via %s failed: %s", candidate, transform, exc)
continue
if candidate != full_name and index.has_tensor(candidate):
return candidate
return None
def _dot_natural_key(name: str):
"""Natural-sort key matching transformers' ``dot_natural_key``.
Falls back to a local reimplementation when the transformers symbol is
unavailable, so shards sort as ``shard_2`` < ``shard_11`` (numeric), exactly
like ``from_pretrained`` orders them before concatenating.
"""
try:
from transformers.core_model_loading import dot_natural_key
return dot_natural_key(name)
except Exception: # pragma: no cover - transformers < 5 / API drift
parts: list = []
for part in name.split("."):
if part.isdigit():
parts.append((0, int(part)))
else:
text = part.rstrip("0123456789")
trailing = part[len(text) :]
parts.append((1, text, int(trailing)) if trailing else (1, text))
return parts
@cache
def _wildcard_concat_converters_for(model_type):
"""Wildcard shard-concat converters registered for one family.
Mirrors :func:`_concat_converters_for` but keeps the *wildcard* converters it
intentionally skips: a single-source ``source_patterns`` containing ``*`` whose
only operation is a ``Concatenate``. Qwen3-Next "Flash" (``qwen4_exp_text``) is
the motivating case -- its per-layer PLE n-gram embedding is stored as
``...ngram_embedding.shard_<i>.weight`` and merged into a single
``...ngram_embedding.weight`` via
``Concatenate(dim=0, num_shards_attribute="split_ngram_parts")`` with
``force_cpu=True`` (the assembled embedding is far too large to concatenate on
an accelerator).
Returns a tuple of ``(target_suffix, source_suffix, concat_dim, force_cpu)``
tuples; ``source_suffix`` still contains the ``*`` shard wildcard.
"""
if not model_type:
return ()
try:
from transformers.conversion_mapping import get_checkpoint_conversion_mapping
from transformers.core_model_loading import Concatenate, WeightConverter
except ImportError: # pragma: no cover - transformers < 5
return ()
mapping = get_checkpoint_conversion_mapping(model_type)
if not mapping:
return ()
converters = []
for entry in mapping:
if not isinstance(entry, WeightConverter):
continue
if len(entry.source_patterns) != 1 or len(entry.target_patterns) != 1:
continue
# Only a single Concatenate op (no MergeModulelist -> that is expert fusion,
# no Chunk/Transpose/etc. -> not a plain shard merge).
if len(entry.operations) != 1 or not isinstance(entry.operations[0], Concatenate):
continue
source = entry.source_patterns[0].rstrip("$")
target = entry.target_patterns[0].rstrip("$")
if "*" not in source or "*" in target:
continue
converters.append((target, source, getattr(entry.operations[0], "dim", 0), getattr(entry, "force_cpu", False)))
return tuple(converters)
@cache
def _wildcard_split_converters_for(model_type):
"""Save-side inverse of :func:`_wildcard_concat_converters_for`.
Returns the same wildcard shard-concat converters, but also surfaces the
``Concatenate`` op's ``num_shards_attribute`` so the merged model-side
parameter can be split back into the exact number of checkpoint shards at
save time.
Returns a tuple of ``(target_suffix, source_suffix, concat_dim,
num_shards_attribute)`` tuples; ``source_suffix`` still contains the ``*``
shard wildcard and ``num_shards_attribute`` may be ``None`` when the op does
not declare one (the caller then infers the count another way).
"""
if not model_type:
return ()
try:
from transformers.conversion_mapping import get_checkpoint_conversion_mapping
from transformers.core_model_loading import Concatenate, WeightConverter
except ImportError: # pragma: no cover - transformers < 5
return ()
mapping = get_checkpoint_conversion_mapping(model_type)
if not mapping:
return ()
converters = []
for entry in mapping:
if not isinstance(entry, WeightConverter):
continue
if len(entry.source_patterns) != 1 or len(entry.target_patterns) != 1:
continue
if len(entry.operations) != 1 or not isinstance(entry.operations[0], Concatenate):
continue
source = entry.source_patterns[0].rstrip("$")
target = entry.target_patterns[0].rstrip("$")
if "*" not in source or "*" in target:
continue
op = entry.operations[0]
converters.append((target, source, getattr(op, "dim", 0), getattr(op, "num_shards_attribute", None)))
return tuple(converters)
def _config_model_types(config):
"""Collect ``model_type`` values from a config and its ``text_config`` (if any)."""
model_types = []
for candidate in (config, getattr(config, "text_config", None)):
model_type = getattr(candidate, "model_type", None)
if model_type and model_type not in model_types:
model_types.append(model_type)
return model_types
def _resolve_num_shards(config, num_shards_attribute):
"""Read the shard count declared by a ``Concatenate`` converter from the config."""
if not num_shards_attribute:
return None
for candidate in (config, getattr(config, "text_config", None)):
value = getattr(candidate, num_shards_attribute, None)
if isinstance(value, int) and value > 0:
return value
return None
def split_merged_concat_tensor(config, full_name: str, tensor: torch.Tensor):
"""Split a merged model-side concat parameter back into its checkpoint shards.
Inverse of the load-time :func:`_assemble_sharded_tensor`. Qwen3-Next "Flash"
(``qwen4_exp_text``) merges the per-layer PLE n-gram embedding shards
(``...ngram_embedding.shard_<i>.weight``) into a single
``...ngram_embedding.weight`` on load; at save time (immediate ShardWriter
path, which does not go through ``save_pretrained``) that merge must be
reversed so the checkpoint keeps its original numbered shards -- otherwise a
single, unusable ``[sum_rows, dim]`` blob is written and reload fails.
Args:
config: The model config (used for ``model_type`` and the shard count).
full_name: The model-side parameter name (e.g. ``....ngram_embedding.weight``).
tensor: The merged tensor.
Returns:
A list of ``(shard_name, shard_tensor)`` pairs, or ``None`` when
*full_name* is not a wildcard-concat target (nothing to split).
"""
if config is None:
return None
for model_type in _config_model_types(config):
for target_suffix, source_suffix, dim, num_shards_attribute in _wildcard_split_converters_for(model_type):
if not full_name.endswith(target_suffix):
continue
prefix = full_name[: -len(target_suffix)]
# Require a clean path boundary so a suffix does not match mid-token.
if prefix and not prefix.endswith("."):
continue
num_shards = _resolve_num_shards(config, num_shards_attribute)
if not num_shards:
# Fall back to an even split by the merged size if the row count is
# divisible by a plausible shard count declared elsewhere; without a
# count we cannot safely reconstruct shards, so leave the tensor as-is.
logger.warning(
"Cannot split merged concat tensor %r: shard count "
"(config.%s) is unavailable; leaving it merged.",
full_name,
num_shards_attribute,
)
return None
if tensor.shape[dim] % num_shards != 0:
logger.warning(
"Cannot evenly split merged concat tensor %r (dim %d size %d) "
"into %d shards; leaving it merged.",
full_name,
dim,
tensor.shape[dim],
num_shards,
)
return None
chunks = torch.chunk(tensor, num_shards, dim=dim)
source_prefix = prefix + source_suffix # still contains the '*' wildcard
return [(source_prefix.replace("*", str(i)), chunk) for i, chunk in enumerate(chunks)]
return None
def _assemble_sharded_tensor(index, full_name: str, device: str):
"""Reconstruct a fused model-side param stored on disk as numbered shards.
Follows the transformers checkpoint converter for the checkpoint's family: it
finds a wildcard ``Concatenate`` converter whose target suffix matches
``full_name``, expands the ``shard_*`` source glob against the checkpoint index,
orders the shards with transformers' natural-sort key, and concatenates them
along the converter's declared ``dim`` (honoring its ``force_cpu`` flag so a
huge embedding is never assembled on an accelerator).
Returns ``(tensor, dst_device)`` -- ``dst_device`` is ``"cpu"`` when the
converter sets ``force_cpu`` -- or ``None`` when no converter matches (the
caller then leaves the param on meta and surfaces an actionable error).
"""
for model_type in _index_model_types(index):
for target_suffix, source_suffix, dim, force_cpu in _wildcard_concat_converters_for(model_type):
if not full_name.endswith(target_suffix):
continue
prefix = full_name[: -len(target_suffix)]
# Require a clean path boundary so a suffix does not match mid-token.
if prefix and not prefix.endswith("."):
continue
# Build the concrete ``shard_*`` glob for this parameter and match the index.
source_regex = re.compile("^" + re.escape(prefix + source_suffix).replace(r"\*", r"(.+)") + "$")
matched = [name for name in index.weight_map if source_regex.match(name)]
if not matched:
continue
matched.sort(key=_dot_natural_key)
read_device = "cpu" if force_cpu else device
read = index.read_tensors(matched, device=read_device)
assembled = torch.cat([read[name] for name in matched], dim=dim).contiguous()
return assembled, read_device
return None
# ── General converter-driven materialization ─────────────────────────────────
# The helpers above cover the common single-op cases explicitly (direct name,
# reverse renaming, expert fusion, literal/wildcard concat). Some model families
# -- especially the MoE ones we target on the meta path (Qwen3-Next "Flash",
# GLM-5-next, DeepSeek-V4, ...) -- also carry *other* WeightConverters in their
# transformers mapping: fused-qkv ``Chunk``, ``Interleave``/``Transpose`` gate-up,
# ``PermuteForRope``, and multi-op chains. Rather than reimplement each op, the
# fallback below reuses the real transformers ``WeightConverter`` objects: it
# routes the checkpoint keys through ``rename_source_key`` exactly like
# ``from_pretrained`` (all renamings, then one converter), feeds the collected
# source tensors into a fresh converter instance, and runs ``convert`` -- the same
# path transformers' own ``revert_weight_conversion`` uses. Output is accepted
# only when its shape matches the destination parameter, so a bad route can never
# silently corrupt a weight (it just falls back to leaving the param on meta).
@lru_cache(maxsize=8)
def _conversion_transforms_for_dir(checkpoint_dir: str):
"""Collect ``(renamings, converters)`` from every model_type in the config.
Deduplicated by ``(source_patterns, target_patterns)`` so families that share
a base mapping (e.g. ``qwen3_5_moe_text`` embeds ``qwen2_moe``) do not route a
checkpoint key through the same converter twice (which would duplicate tensors
inside a concat/merge).
"""
try:
from transformers.conversion_mapping import get_checkpoint_conversion_mapping
from transformers.core_model_loading import WeightConverter, WeightRenaming
except ImportError: # pragma: no cover - transformers < 5
return (), ()
renamings: list = []
converters: list = []
seen: set = set()
for model_type in _model_types_for_dir(checkpoint_dir):
mapping = get_checkpoint_conversion_mapping(model_type)
if not mapping:
continue
for entry in mapping:
if not isinstance(entry, (WeightConverter, WeightRenaming)):
continue
sig = (tuple(entry.source_patterns), tuple(entry.target_patterns))
if sig in seen:
continue
seen.add(sig)
(converters if isinstance(entry, WeightConverter) else renamings).append(entry)
return tuple(renamings), tuple(converters)
@lru_cache(maxsize=8)
def _converter_routing_table(checkpoint_dir: str):
"""Map each converter-produced model-side name to its checkpoint sources.
Mirrors the ``from_pretrained`` routing loop: every checkpoint key is renamed
via ``rename_source_key`` (all ``WeightRenaming``s, then at most one
``WeightConverter``). Returns ``(table, pattern_to_converter)`` where ``table``
maps ``renamed_full_name -> [(source_pattern, checkpoint_key), ...]``.
"""
renamings, converters = _conversion_transforms_for_dir(checkpoint_dir)
if not converters:
return {}, {}
try:
from transformers.core_model_loading import rename_source_key
except ImportError: # pragma: no cover - transformers < 5
return {}, {}
index = get_safetensors_index(checkpoint_dir)
pattern_to_converter = {pattern: conv for conv in converters for pattern in conv.source_patterns}
renaming_list = list(renamings)
converter_list = list(converters)
table: dict = {}
for key in index.weight_map:
try:
renamed, source_pattern = rename_source_key(key, renaming_list, converter_list)
except Exception: # pragma: no cover - never fail the whole load on one key
continue
if source_pattern is None:
continue
table.setdefault(renamed, []).append((source_pattern, key))
# ``rename_source_key`` renames a 1->N converter (e.g. fused-QKV ``Chunk``) to its FIRST
# target only, so the table above holds just that one name. ``convert`` emits every target
# from the same source tensors, so register the sibling target names too (each pointing at
# the same collected sources); the caller passes whichever it needs and picks it out.
expanded: dict = {}
for renamed, entries in table.items():
converter = pattern_to_converter.get(entries[0][0])
targets = list(getattr(converter, "target_patterns", []) or [])
first = targets[0] if targets else None
if len(targets) <= 1 or first is None or first not in renamed:
expanded[renamed] = entries
continue
for target in targets:
expanded[renamed.replace(first, target, 1)] = entries
return expanded, pattern_to_converter
@lru_cache(maxsize=8)
def _auto_config_for_dir(checkpoint_dir: str):
"""Best-effort model config for converter ops that need it (PermuteForRope, ...).
``local_files_only=True`` keeps this fully offline -- the checkpoint is already a local
snapshot dir, and a hub round-trip here would stall the whole materialization.
"""
try:
from transformers import AutoConfig
return AutoConfig.from_pretrained(checkpoint_dir, trust_remote_code=True, local_files_only=True)
except Exception as exc: # pragma: no cover - config-less ops still work
logger.debug("could not load config for converter ops from %s: %s", checkpoint_dir, exc)
return None
def _convert_param_via_converters(index, full_name: str, target_shape, device: str):
"""Materialize ``full_name`` by replaying its transformers ``WeightConverter``.
Returns ``(tensor, dst_device)`` when a converter produces a tensor whose shape
matches ``target_shape``; otherwise ``None`` (the caller then leaves the param
on meta). ``dst_device`` is ``"cpu"`` when the converter declares ``force_cpu``.
"""
checkpoint_dir = str(getattr(index, "checkpoint_dir", "") or "")
if not checkpoint_dir:
return None
try:
table, pattern_to_converter = _converter_routing_table(checkpoint_dir)
except Exception: # pragma: no cover
return None
entries = table.get(full_name)
if not entries:
return None
try:
import copy as _copy
from types import SimpleNamespace
base_converter = pattern_to_converter.get(entries[0][0])
if base_converter is None:
return None
force_cpu = bool(getattr(base_converter, "force_cpu", False))
read_device = "cpu" if force_cpu else device
# Group checkpoint keys per source pattern; order within each group by the
# same natural key transformers uses, so shard/expert indices line up.
keys_by_pattern: dict = {}
for source_pattern, key in entries:
keys_by_pattern.setdefault(source_pattern, []).append(key)
converter = _copy.deepcopy(base_converter)
all_keys = [k for keys in keys_by_pattern.values() for k in keys]
read = index.read_tensors(all_keys, device=read_device)
for source_pattern in converter.source_patterns:
keys = keys_by_pattern.get(source_pattern)
if not keys:
continue
keys.sort(key=_dot_natural_key)
for key in keys:
converter.add_tensor(full_name, key, source_pattern, read[key])
config = _auto_config_for_dir(checkpoint_dir)
target_size = torch.Size(tuple(target_shape))
model_shim = SimpleNamespace(
config=config,
get_parameter=lambda _name, _s=target_size: SimpleNamespace(shape=_s),
)
result = converter.convert(full_name, model=model_shim, config=config)
except Exception as exc:
logger.debug("converter-driven materialization of %s failed: %s", full_name, exc)
return None
if not isinstance(result, dict):
return None
value = result.get(full_name)
if isinstance(value, list):
value = value[0] if value else None
if value is None or tuple(value.shape) != tuple(target_shape):
return None
return value.contiguous(), read_device
def materialize_module(module: nn.Module, module_name: str, index: SafetensorsIndex, device: str) -> None:
"""Populate `module`'s (currently meta) parameters/buffers with real data read
directly from the checkpoint, onto `device`. `module_name` is `module`'s dotted
path in the full model (used as the tensor-name prefix in the checkpoint).
AutoScheme's scoring
wraps quantized layers in ``AutoSchemeWrapperLinear``, which replaces a plain
``nn.Linear`` with a wrapper holding the real layer at ``.orig_layer`` --
inserting an extra ``.orig_layer`` path segment that doesn't exist in the
checkpoint's own tensor names. Strip it back out before looking up the name.
"""
import re as _re
# Fused-MoE replacement modules
# (SequentialQwen3_5MoeExperts and friends) expose UNFUSED per-expert
# parameter names (experts.{i}.gate_proj.weight ...) that don't exist in a
# checkpoint whose on-disk layout is the fused 3D one
# (experts.gate_up_proj [N, 2*inter, hidden] / experts.down_proj). The
# compressor's own tuning loop handles this via OffloadManager.reload +
# materialize_model_, but every bare materialize_module() consumer
# (AutoScheme's delta_loss scoring streams, stream_block_forward for the
# calibration/eval forwards) previously left those params on meta -- the
# meta-ness then propagated silently until a crash far downstream. Map
# each unfused name onto its fused on-disk tensor and slice.
#
# The exact slicing depends on the family's fused layout: qwen3_moe stores
# ``[N, 2*inter, hidden]`` (contiguous halves, no transpose), while gpt_oss /
# llama4 store ``[N, hidden, 2*inter]`` (matmul layout, needs transpose) and
# gpt_oss additionally interleaves the gate/up columns. Rather than key the
# whole decision off ``model_type`` (which does not generalise to new
# families), the transpose is inferred from the *target* parameter shape and
# only the interleave axis (invisible to shapes) falls back to a small
# model_type hint table (see ``_slice_fused_expert``). Biases (gpt_oss) follow
# the same split as their weight.
_FUSED_RE = _re.compile(r"^(.*\.experts)\.(\d+)\.(gate_proj|up_proj|down_proj)\.(weight|bias)$")
_fused_cache: dict = {}
_layout_cache: list = [] # lazily-resolved (transposed_hint, interleaved)
def _fused_lookup(full_name: str, target_shape):
m = _FUSED_RE.match(full_name)
if not m:
return None
prefix, expert_idx, proj, attr = m.group(1), int(m.group(2)), m.group(3), m.group(4)
is_gate_up = proj in ("gate_proj", "up_proj")
if is_gate_up:
fused_name = f"{prefix}.gate_up_proj" if attr == "weight" else f"{prefix}.gate_up_proj_bias"
else:
fused_name = f"{prefix}.down_proj" if attr == "weight" else f"{prefix}.down_proj_bias"
if not index.has_tensor(fused_name):
return None
if fused_name not in _fused_cache:
_fused_cache[fused_name] = index.read_tensors([fused_name], device=device)[fused_name]
fused = _fused_cache[fused_name][expert_idx]
# Resolve the model_type layout hint lazily and only once, and only when a
# fused expert tensor is actually being sliced -- reading the model type
# touches ``index.checkpoint_dir``/config, which cheap/fake indices used by
# non-fused (dense) materialization do not populate.
if not _layout_cache:
_layout_cache.append(_fused_expert_layout(index))
transposed_hint, interleaved = _layout_cache[0]
value = _slice_fused_expert(fused, proj, attr, is_gate_up, tuple(target_shape), transposed_hint, interleaved)
# Guard against a slice that does not match the destination parameter: it
# is safer to leave the param on meta (the caller then raises an
# actionable "fall back to a full CPU load" error) than to hand accelerate
# a mismatched tensor and crash with a cryptic ValueError.
if value is not None and tuple(value.shape) != tuple(target_shape):
return None
return value
def _concat_lookup(full_name: str):
"""Assemble a model-side param that the checkpoint stores as several tensors
concatenated by a ``WeightConverter`` (e.g. GLM-5.3 ``q/k/v_conv1d`` ->
``conv1d``). Returns the concatenated tensor, or ``None`` when no converter
applies or a source tensor is missing."""
for model_type in _index_model_types(index):
for target_suffix, source_suffixes, dim in _concat_converters_for(model_type):
if not full_name.endswith(target_suffix):
continue
prefix = full_name[: -len(target_suffix)]
# Guard against a spurious mid-token match (require a clean boundary).
if prefix and not prefix.endswith("."):
continue
source_names = [f"{prefix}{suffix}" for suffix in source_suffixes]
if not all(index.has_tensor(name) for name in source_names):
continue
read = index.read_tensors(source_names, device=device)
return torch.cat([read[name] for name in source_names], dim=dim).contiguous()
return None
targets = [] # (param_name, full_checkpoint_name, declared_meta_dtype)
fused_targets = [] # (param_name, sliced_value)
buffer_names = {name for name, _ in module.named_buffers()}
for name, tensor in list(module.named_parameters()) + list(module.named_buffers()):
is_meta = str(tensor.device) == "meta"
is_checkpoint_backed_buffer = name in buffer_names
if not is_meta and not is_checkpoint_backed_buffer:
continue # already materialized parameter (e.g. shared/tied weights)
full_name = f"{module_name}.{name}".replace(".orig_layer.", ".")
resolved_name = _resolve_checkpoint_name(index, full_name)
if not is_meta and resolved_name is None:
continue # runtime buffer without a checkpoint counterpart
if resolved_name is None:
sliced = _fused_lookup(full_name, tensor.shape)
if sliced is None:
sliced = _concat_lookup(full_name)
if sliced is not None:
fused_targets.append((name, sliced))
continue
assembled = _assemble_sharded_tensor(index, full_name, device)
if assembled is not None:
value, dst_device = assembled
set_module_tensor_to_device(module, name, dst_device, value=value, dtype=value.dtype)
continue
converted = _convert_param_via_converters(index, full_name, tensor.shape, device)
if converted is not None:
value, dst_device = converted
set_module_tensor_to_device(module, name, dst_device, value=value, dtype=value.dtype)
continue
logger.warning("No checkpoint tensor found for %s, leaving on meta", full_name)
continue
targets.append((name, resolved_name, tensor.dtype))
for name, value in fused_targets:
try:
set_module_tensor_to_device(module, name, device, value=value, dtype=value.dtype)
except (ValueError, RuntimeError) as exc:
# A fused-expert slice whose shape does not match its destination
# cannot be reconciled from the checkpoint; rather than surface the
# raw accelerate error, point the user at the non-meta fallback.
raise RuntimeError(
f"Failed to materialize fused MoE parameter '{module_name}.{name}' from the checkpoint "
f"(value shape {tuple(value.shape)}). This usually means the model's fused-expert layout "
"is not yet recognised by the meta-skeleton loader. Set AR_DISABLE_META_LOAD=1 to load "
"the whole model on CPU instead (uses more RAM)."
) from exc
_fused_cache.clear()
if not targets:
return
values = index.read_tensors([full_name for _, full_name, _ in targets], device=device)
for name, full_name, declared_dtype in targets:
# Prefer the meta parameter's already-declared dtype: it reflects
# whatever compute dtype the caller already promoted the (still-meta)
# model to (e.g. ModelContext._set_amp_dtype()'s `model.to(amp_dtype)`),
# and materializing to a different dtype than sibling non-block params
# that were promoted while still real breaks ops mixing the two (e.g.
# LayerNorm on bf16 activations with fp16 weight/bias). The one
# exception: a meta skeleton built without an enclosing dtype context
# (e.g. Qwen3_5MoeExperts' per-expert Linears, built under
# `torch.device("meta")` alone) defaults the declared dtype to
# float32 regardless of the checkpoint's real dtype -- found via a
# real 8-layer MoE fixture crashing with "expected m1 and m2 to have
# the same dtype" inside AutoScheme scoring. Detect that case (declared
# float32 but checkpoint isn't) and fall back to the checkpoint's own
# dtype instead.
target_dtype = declared_dtype
if declared_dtype == torch.float32 and values[full_name].dtype != torch.float32:
target_dtype = values[full_name].dtype
set_module_tensor_to_device(module, name, device, value=values[full_name], dtype=target_dtype)
def free_module(module: nn.Module) -> None:
"""Release a module's real tensors back to the meta device, freeing memory."""
for name, tensor in list(module.named_parameters()) + list(module.named_buffers()):
if str(tensor.device) == "meta":
continue
set_module_tensor_to_device(module, name, "meta")
def total_resident_bytes(model: nn.Module) -> int:
"""Debug helper: sum the byte size of every non-meta parameter/buffer in
`model`. Used to diagnose whether blocks are genuinely returning to meta
after free_module(), or something else is holding real memory."""
total = 0
for _, tensor in list(model.named_parameters()) + list(model.named_buffers()):
if str(tensor.device) != "meta":
total += tensor.numel() * tensor.element_size()
return total
def unfuse_meta_moe_(model: nn.Module) -> list[str]:
"""Turn ``transformers>=5`` fused 3D expert parameters into per-expert ``nn.Linear``.
``transformers>=5`` declares MoE experts as a single fused ``nn.Parameter`` of shape
``(num_experts, ...)``, which AutoRound cannot quantize (it is not an ``nn.Linear``).
The regular load pipeline fixes this *after* a full CPU load; on a meta skeleton we
can do it up-front for free -- meta tensors own no storage, so unfusing here only
rewires ``nn.Module`` objects and costs ~0 RAM.
Doing it before any weight is read is also what keeps the later per-block
materialization cheap: the model-side names become ``experts.<i>.<proj>.weight``,
which is exactly the layout the checkpoints use, so ``materialize_module`` reads one
small 2D tensor per expert instead of forcing a whole fused ``(num_experts, ...)``
tensor to be assembled in RAM.
No modeling class is monkey-patched: the per-expert forward is dispatched through
``transformers.integrations.moe.ALL_EXPERTS_FUNCTIONS`` (the public experts-interface
registry) via ``config._experts_implementation``.
Returns the list of unfused module names (empty when there is nothing to do).
"""
try:
from auto_round.modeling.fused_moe.replace_modules import (
_handle_moe_modules,
is_custom_model,
log_moe_block_transition,
)
except ImportError: # pragma: no cover - transformers < 5 or partial install
return []
# Model families with a dedicated replacement (llama4, qwen3_5_moe, step3p5, ...) must
# go through `apply_replacements`, which runs its custom `from_original` against the
# *fused* module. Unfusing them here with the generic linear_loop path would leave
# that replacement facing a module whose fused params no longer exist.
if is_custom_model(model):
logger.debug(
"meta skeleton: %s has a dedicated MoE replacement, deferring the unfuse to apply_replacements",
getattr(getattr(model, "config", None), "model_type", None),
)
return []
try:
# This is where the structural change actually happens for a meta-built model, so
# report the before/after here rather than in `apply_replacements` (which by then
# has nothing left to do).
with log_moe_block_transition(model, "unfuse"):
unfused = _handle_moe_modules(model)
except Exception: # pragma: no cover - never block the meta build on this
logger.warning("Structural MoE unfuse on the meta skeleton failed", exc_info=True)
return []
if unfused:
logger.info("meta skeleton: structural MoE unfuse produced %d unfused experts modules", len(unfused))
return unfused
def build_meta_model(model_name: str, trust_remote_code: bool = True, unfuse_moe: bool = True):
"""Build a meta-device model skeleton (~0 RAM) plus its tokenizer and a
SafetensorsIndex for on-demand materialization, instead of AutoRound's own
``llm_load_model(model_name, device_map="cpu")`` which fully materializes the
checkpoint on CPU RAM in one shot. Deliberately narrower than ``llm_load_model``:
only covers the common local-directory ``AutoModelForCausalLM`` case (no
bagel/glm/mxfp4/HPU special-casing) -- callers should fall back to
``llm_load_model`` for anything this doesn't handle.
When ``unfuse_moe`` is set (the default) the fused ``transformers>=5`` MoE experts
are split into per-expert ``nn.Linear`` while still on meta, so the returned skeleton
is immediately quantizable and every later materialization is per-expert.
"""
from accelerate import init_empty_weights
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
config = AutoConfig.from_pretrained(model_name, trust_remote_code=trust_remote_code)
# Prefer the exact class named in config.architectures: AutoModelForCausalLM
# cannot resolve multimodal architectures (e.g.
# Qwen3_5MoeForConditionalGeneration), which previously forced VLM
# checkpoints down the full-CPU-load path. Same resolution strategy as
# reap/layerwise_prune.py's disk-streamed model builder.
import transformers as _transformers
from auto_round.modeling.nemotron_h_omni import (
prefer_remote_omni_class,
register_nemotron_h_omni_checkpoint_mapping,
)
register_nemotron_h_omni_checkpoint_mapping(config, trust_remote_code)
archs = getattr(config, "architectures", None) or []
model_cls = None
if not prefer_remote_omni_class(config, trust_remote_code):
model_cls = next((getattr(_transformers, a) for a in archs if hasattr(_transformers, a)), None)
with init_empty_weights():
if model_cls is not None:
model = model_cls(config)
else:
model = AutoModelForCausalLM.from_config(config, trust_remote_code=trust_remote_code)
# Unfuse while everything is still meta: free, and it makes the model-side
# parameter names line up 1:1 with the per-expert checkpoint tensors.
if unfuse_moe:
unfuse_meta_moe_(model)
# Constructing from config does not tie weights (that happens in
# from_pretrained), so tied params such as lm_head.weight -- absent from
# the checkpoint by design -- stay separate meta tensors and later crash
# with "Cannot copy out of meta tensor" on the first device move. Tie
# here: the shared tensor materializes once from the checkpoint side it
# is stored under and both names become real together.
if getattr(model.config, "tie_word_embeddings", False):
model.tie_weights()
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=trust_remote_code)
index = SafetensorsIndex(model_name)
return model, tokenizer, index
def materialize_non_block_params(
model: nn.Module, block_prefixes: list[str], index: SafetensorsIndex, device: str
) -> None:
"""Materialize every real (non-meta) parameter/buffer NOT under one of
``block_prefixes`` -- i.e. embeddings, final norm, lm_head, and similar small
top-level modules -- leaving the (typically 100+GB combined) decoder blocks on
meta for later per-block materialize/free. These non-block modules are needed
continuously throughout scoring and are comparatively small even for large
vocabularies, so it's simplest to load them once, for real, up front.
"""
def _in_block(name: str) -> bool:
return any(name == p or name.startswith(p + ".") for p in block_prefixes)
# Tied output embeddings (lm_head.weight tied to embed_tokens.weight) are absent from
# the checkpoint by design -- the tie is re-established below, which makes them real.
# Do not warn about those, or every tied model reports a scary "leaving on meta" line
# for a parameter that is fine a few statements later.
tied_keys = set(getattr(model, "_tied_weights_keys", None) or ())
def _is_tied(name: str) -> bool:
return name in tied_keys or any(name.startswith(key.rstrip("*")) for key in tied_keys if key.endswith("*"))
targets = [] # (param_name, full_checkpoint_name)
deferred_tied = []
for name, tensor in list(model.named_parameters()) + list(model.named_buffers()):
if str(tensor.device) != "meta" or _in_block(name):
continue
full_name = name.replace(".orig_layer.", ".")
resolved_name = _resolve_checkpoint_name(index, full_name)
if resolved_name is None:
assembled = _assemble_sharded_tensor(index, full_name, device)
if assembled is not None:
value, dst_device = assembled
set_module_tensor_to_device(model, name, dst_device, value=value, dtype=value.dtype)
continue
converted = _convert_param_via_converters(index, full_name, tensor.shape, device)
if converted is not None:
value, dst_device = converted
set_module_tensor_to_device(model, name, dst_device, value=value, dtype=value.dtype)
continue
if _is_tied(name):
deferred_tied.append(name)
else:
logger.warning("No checkpoint tensor found for %s, leaving on meta", full_name)
continue
targets.append((name, resolved_name))
if targets:
values = index.read_tensors([full_name for _, full_name in targets], device=device)
for name, full_name in targets:
# See the matching comment in materialize_module() -- explicit dtype=
# is required so the checkpoint's real dtype wins over whatever the
# meta skeleton happened to declare.
set_module_tensor_to_device(model, name, device, value=values[full_name], dtype=values[full_name].dtype)
# set_module_tensor_to_device replaces parameter objects, which breaks
# weight tying: a tied lm_head still references the old (meta) parameter
# while the embedding points at the new real one. Re-tie so every tied
# name follows its checkpoint-backed source tensor.
if getattr(getattr(model, "config", None), "tie_word_embeddings", False):
model.tie_weights()
# Anything declared tied that the re-tie did not actually make real is a genuine
# problem (quantizing it would fail on a meta tensor), so surface it now.
still_meta = [name for name in deferred_tied if _param_or_buffer(model, name) is None]
if still_meta:
logger.warning("No checkpoint tensor found for tied %s, leaving on meta", ", ".join(still_meta))
def _param_or_buffer(model: nn.Module, name: str):
"""Return the named parameter/buffer when it is materialized, else ``None``."""
for candidate_name, tensor in list(model.named_parameters()) + list(model.named_buffers()):
if candidate_name == name:
return None if tensor.device.type == "meta" else tensor
return None
class stream_block_forward:
"""Context manager: wrap every top-level decoder block's ``forward`` so it
materializes its own real weights from ``index`` right before running and
frees them back to meta right after -- letting a plain ``model(...)`` call
(e.g. for computing held-out loss) drive the model exactly as normal while
only ever one block's weights are resident at a time.
Deliberately much lighter than the auto_scheme delta_loss.py streaming
forward it's modeled on (``prepare_model_low_gpu``/``model_forward_low_gpu``):
no input-caching for later backward replay, no grad-mode bookkeeping -- this
is for a plain inference-only forward pass (e.g. eval loss), not tuning.
"""
def __init__(self, model: nn.Module, index: SafetensorsIndex, device: str, block_names: list[str] | None = None):
self.model = model
self.index = index
self.device = device
self.block_names = block_names if block_names is not None else _default_block_names(model)
self._originals: dict[str, callable] = {}
def __enter__(self):
for block_name in self.block_names:
module = _get_module(self.model, block_name)
self._originals[block_name] = module.forward
def make_wrapped(module=module, block_name=block_name, original_forward=module.forward):
def wrapped(*args, **kwargs):
materialize_module(module, block_name, self.index, device=self.device)
try:
return original_forward(*args, **kwargs)
finally:
free_module(module)
return wrapped
module.forward = make_wrapped()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
for block_name, original_forward in self._originals.items():
_get_module(self.model, block_name).forward = original_forward
return False
def _default_block_names(model: nn.Module) -> list[str]:
from auto_round.utils import get_block_names
return get_block_names(model)[0]
def _get_module(model: nn.Module, name: str) -> nn.Module:
from auto_round.utils import get_module
return get_module(model, name)