PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Tune-loop memory model, MoE-implementation policy, and pool pulls.
Shared by the quantizer base class (the ``maybe_adapt_moe_implementation``
and ``pull_tuning_pool`` hooks) so individual algorithm implementations stay
free of fleet-memory policy: everything here reads only the block's wrapper
layout, the batch size, the run's config, and the devices' free memory.
"""
from typing import Any
import torch
from auto_round.logger import logger
def _tuning_state_bytes(block: torch.nn.Module, target_dev: str) -> int:
"""In-loop tuning state for parameters homed on ``target_dev`` (logical count).
14 B per logical parameter: fp32 value + fp32 grad (first backward) +
best-params snapshot + bf16 copy. At tune time ``block.parameters()``
yields BOTH the original weights and the wrappers' registered value
params -- counting both would price every wrapped parameter twice.
Wrapper tuning tensors are identified by identity (any ``.params``
dict on a module) and excluded; the original weights carry the charge.
"""
return _logical_state_by_device(block).get(str(target_dev), 0)
def _logical_state_by_device(block: torch.nn.Module) -> dict[str, int]:
"""Deduped per-device form of :func:`_tuning_state_bytes` ({device: bytes}).
``_state_bytes_by_device`` counts ``block.parameters()`` raw, which at
tune time includes BOTH the original weights and the wrappers' fp32
value params -- every wrapped parameter priced twice.
"""
tuning = set()
for m in block.modules():
_p = getattr(m, "params", None)
if isinstance(_p, dict):
for _v in _p.values():
if torch.is_tensor(_v):
tuning.add(id(_v))
out = {}
for p in block.parameters():
if id(p) in tuning:
continue
out[str(p.device)] = out.get(str(p.device), 0) + p.numel() * 14
return out
def _ensure_routed_shape_recorders_(block: torch.nn.Module) -> None:
"""Attach fire-once recorders on MoE containers for the activation budget.
Records ``top_k = top_k_index.shape[-1]`` and ``routed_rows = index.numel()``
from the first NATURAL dispatch (forced all-experts routing -- tracing /
warmup -- is skipped: it would record top_k = num_experts and over-reserve).
Layout-agnostic: no attribute spellings, only arg shapes. The experts
interface passes ``(hidden_states, top_k_index, top_k_weights)``; the
recorder reads the two leading tensor args. The hook removes itself after
the first record.
"""
from auto_round.modeling.fused_moe.utils import force_all_experts_routing_enabled
if force_all_experts_routing_enabled():
return
from auto_round.utils import SUPPORTED_LAYER_TYPES
from auto_round.utils.model import is_moe_layer
def _record(module: torch.nn.Module, args: Any) -> None:
recorded = False
try:
tensors = [a for a in args if torch.is_tensor(a)]
if len(tensors) < 2:
return
hidden, index = tensors[0], tensors[1]
if hidden.ndim < 2 or index.ndim < 1:
return
module._routed_shape_rec_ = (int(index.shape[-1]), int(index.numel()))
recorded = True
logger.debug(
"[routed-shape] recorded top_k=%d routed_rows=%d",
module._routed_shape_rec_[0],
module._routed_shape_rec_[1],
)
except Exception as e: # pragma: no cover - diagnostics only, never break the forward
logger.debug("[routed-shape] recorder skipped (%s)", e)
finally:
if recorded:
# self-remove ONLY on a successful record: removing on a
# miss (non-dispatch arg shapes) silently disarms the
# recorder before it ever sees a natural dispatch
for handle in list(getattr(module, "_routed_rec_handles_", [])):
handle.remove()
module._routed_rec_handles_ = []
def _is_experts_container(name: str, module: torch.nn.Module) -> bool:
# container may carry the marker itself, expose num_experts, be a
# ModuleList, or carry the MoE marker only on an ANCESTOR container
if is_moe_layer(module) or isinstance(getattr(module, "num_experts", None), int):
return True
stem = name
while "." in stem:
stem = stem.rsplit(".", 1)[0]
mod = mods.get(stem)
if mod is not None and (is_moe_layer(mod) or isinstance(getattr(mod, "num_experts", None), int)):
return True
return False
def _can_host_recorder(module: torch.nn.Module) -> bool:
# Only dispatch containers receive (hidden, top_k_index,
# top_k_weights). Leaf projections (plain Linear/Conv1D) and tuning
# wrappers (orig_layer child) see per-row activations only: their
# pre-hook can never record, never self-removes, and lingers for the
# whole block -- which disqualifies the projection in the
# grouped-experts support gate and forces every tune iteration onto
# the per-expert linear_loop.
if hasattr(module, "orig_layer"):
return False
return not isinstance(module, SUPPORTED_LAYER_TYPES)
mods = dict(block.named_modules())
for _name, module in mods.items():
if (
"expert" in _name.lower()
and _is_experts_container(_name, module)
and _can_host_recorder(module)
and not hasattr(module, "_routed_shape_rec_")
):
module._routed_rec_handles_ = [module.register_forward_pre_hook(_record)]
def _grouped_stack_bytes(block: torch.nn.Module, device: str, config: Any = None) -> int:
"""Qdq-stack bytes the grouped experts modes materialize on ``device``."""
d = _grouped_stack_bytes_detail(block, device, config=config)
return d["retention"] + d["transient"]
def _grouped_stack_bytes_detail(block: torch.nn.Module, device: str, config: Any = None) -> dict[str, int]:
"""Per-term split of :func:`_grouped_stack_bytes` (retention + transient).
Two chunk-aware terms (retention is chunk-independent; transient is
chunk-sized) for ``auto``/``linear_grouped``/
``linear_grouped_sliced`` (linear_loop stacks nothing):
- retention (chunk-independent): autograd keeps EVERY chunk's stacked
fp32 values (straight-through mask) and qdq output (GEMM backward) --
6 B per weight element of the experts HOMED on the device, observed as
31 simultaneous 16-expert stacks on the OOMed lane;
- transient (chunk-sized): the quant func streams the stacked chunk
through several passes (~2x chunk bytes). Chunk size comes from the
same source the grouped path uses (_qdq_chunk_size: AR_MOE_CHUNK or
the auto working-set budget; 0 = fuse everything = all-experts chunk).
Mode resolution matches the dispatch: an EXPLICIT env value governs,
otherwise ``config._experts_implementation`` (so gates stay consistent
with a mid-run auto switch to linear_loop instead of charging stacks
for a lane that no longer stacks any). Mode-set-but-loop-ran fallbacks
can only overcharge -- safe direction.
"""
import auto_round.envs as _envs
from auto_round.modeling.fused_moe.moe_experts_interface import GROUPED_LINEAR_IMPL
mode = str(getattr(_envs, "AR_MOE_EXPERTS_IMPL", "auto") or "auto").lower()
if mode in ("", "auto"):
mode = str(getattr(config, "_experts_implementation", GROUPED_LINEAR_IMPL) or GROUPED_LINEAR_IMPL)
if mode not in ("", "auto", "linear_grouped", "linear_grouped_sliced"):
return {"retention": 0, "transient": 0}
retention = 0
transient = 0
try:
from auto_round.modeling.fused_moe.grouped_experts import _qdq_chunk_size
from auto_round.utils.model import is_moe_layer
def _is_expert_leaf(name, leaf):
# expert leaves are named "experts.N.proj" in every unfused layout;
# the container may carry num_experts, be a ModuleList, or carry
# the MoE marker only on an ANCESTOR container -- a name-only
# walk can return 0 stacks and silently disable the activation
# charge and the auto linear_loop pick
if "expert" not in name.lower() or "shared" in name.lower():
# shared experts are plain modules outside the dispatch (the
# repo's own is_moe_expert predicate excludes them); stacking
# or routing charges on them are phantom bytes
return False
stem = name
while "." in stem:
stem = stem.rsplit(".", 1)[0]
mod = mods.get(stem)
if mod is None:
continue
if (
isinstance(mod, torch.nn.ModuleList)
or isinstance(getattr(mod, "num_experts", None), int)
or is_moe_layer(mod)
):
return True
return False
# group leaves by slot name so per-slot chunking matches the real
# grouped path (gate/up/down are chunked independently)
mods = dict(block.named_modules())
by_slot = {}
seen_weight_ids = set()
for name, leaf in mods.items():
# at tune time named_modules yields BOTH the wrapper
# (...gate_proj) and its orig_layer child (...gate_proj.orig_
# layer), each exposing a same-shape weight -- counting both
# doubles retention
if name.endswith(".orig_layer"):
continue
weight = getattr(leaf, "weight", None)
if weight is None or not hasattr(weight, "numel"):
continue
if id(weight) in seen_weight_ids:
continue
if not _is_expert_leaf(name, leaf):
continue
seen_weight_ids.add(id(weight))
by_slot.setdefault(type(leaf).__name__ + "." + getattr(leaf, "slot_tag", ""), []).append((leaf, weight))
for _slot, pairs in by_slot.items():
local_pairs = [(l, w) for l, w in pairs if str(w.device) == str(device)]
if not local_pairs:
continue
elems = sum(int(w.numel()) for _l, w in local_pairs)
retention += elems * 6 # fp32 value stack + bf16 qdq output
try:
chunk = _qdq_chunk_size([l for l, _w in local_pairs]) # env/budget-aware; 0 = all
except Exception as e:
logger.warning("[tune] chunk sizing failed for slot %r on %s (%s); using 16", _slot, device, e)
chunk = 16
per_leaf = elems // max(1, len(local_pairs))
chunk_elems = elems if chunk <= 0 else min(elems, chunk * per_leaf)
transient += chunk_elems * 6 # ~2 streaming passes over the chunk
except Exception as e: # pragma: no cover - never break the gate
logger.warning(
"[tune] grouped-stack accounting unavailable (%s); continuing with a zero stack charge "
"(an OOM in the tuned loop after this line is explained by the missing charge)",
e,
)
return {"retention": 0, "transient": 0}
return {"retention": retention, "transient": transient}
def _routed_budget_bytes(block: torch.nn.Module, tensors: Any, batch_size: int, config: Any = None) -> int | None:
"""tokens x top_k routed working set x6 (in/out accumulators + bwd grad).
Top_k resolution order: config values (maintained spelling list) ->
recorded NATURAL-dispatch arg shapes (layout/spelling agnostic) ->
container attrs. None when nothing resolves (callers decline).
"""
from auto_round.utils.device import get_first_available_attr
from auto_round.utils.model import is_moe_layer
top_k = None
if config is not None:
top_k = get_first_available_attr(config, ["num_experts_per_tok", "moe_num_active_primary_experts"])
if top_k is None:
moe_topk = getattr(config, "moe_topk", None) # list-form top_k
if isinstance(moe_topk, (list, tuple)) and moe_topk:
top_k = moe_topk[0]
if top_k is None:
for m in block.modules():
rec = getattr(m, "_routed_shape_rec_", None)
if rec is not None:
top_k, _rows = rec
break
if top_k is None:
for _name, module in block.named_modules():
if is_moe_layer(module) or isinstance(getattr(module, "num_experts", None), int):
top_k = getattr(module, "top_k", None) or getattr(module, "num_experts_per_tok", None)
if top_k is not None:
break
if top_k is None:
return None
ref = (
tensors[0]
if isinstance(tensors, list) and tensors
else next(
(t for t in (tensors.values() if isinstance(tensors, dict) else []) if isinstance(t, torch.Tensor)), None
)
)
if ref is None or ref.ndim < 2:
return None
row_len = int(ref.shape[-2]) if ref.ndim >= 3 else int(ref.shape[0])
hidden_bytes = int(ref.shape[-1]) * ref.element_size()
tune_tokens = max(1, int(batch_size)) * max(1, row_len)
routed_rows = None
for m in block.modules():
rec = getattr(m, "_routed_shape_rec_", None)
if rec is not None:
top_k_rec, routed_rows = rec
break
if routed_rows is not None:
# recorded rows scale per-token to this batch's tokens (collection may
# have seen a larger batch; a floor-div here silently kept the
# oversized count)
seen_tokens = max(1, routed_rows // max(1, int(top_k)))
routed_rows = int(routed_rows * tune_tokens / seen_tokens)
else:
routed_rows = tune_tokens * int(top_k)
return int(routed_rows * hidden_bytes * 6)
def _activation_bytes_by_device(
block: torch.nn.Module, tensors: Any, batch_size: int, config: Any = None
) -> dict[str, int] | None:
"""Per-DEVICE activation charge for the tune loop: {device: bytes}.
The loop's forward/backward executes on every device the block spans --
each module's saved output + grad lands on the module's WEIGHT HOME, and
each expert's routed-row caches land on the expert's home. Charging the
whole activation budget to the entry alone (the previous form) left the
MoE stages' transients unpriced while the entry's own charge was
accurate.
Terms per device:
- est split: the estimator's per-module output bytes (x2 grads, expert
outputs ratio-scaled) bucketed by weight home -- counts shared experts
wherever they are PLAIN modules outside the dispatch.
- routed split: tokens x top_k x hidden x 6 distributed across devices
by homed-expert count (first-order static split; per-expert token
counts vary with routing draws).
- grouped stacks: chunk-INDEPENDENT qdq retention, additive (a different
tensor class entirely).
Per-device composition is max(est_d, routed_d) -- the terms overlap on
the routed rows exactly as in the scalar form, per device. None only
when neither term resolves anywhere (callers decline).
"""
try:
from auto_round.utils.device import estimate_tuning_block_mem
from auto_round.utils.model import is_moe_layer
est_by_dev = {}
expert_counts = {}
est_failed = False
has_moe = any(is_moe_layer(m) or isinstance(getattr(m, "num_experts", None), int) for m in block.modules())
try:
_ld, _la, _io, _ad, est_by_dev, expert_counts = estimate_tuning_block_mem(
block, tensors, batch_size, config
)
except Exception as e: # per-module accounting is best-effort for dense
logger.warning("[tune] per-module activation estimate unavailable (%s)", e)
est_failed = True
if est_failed and has_moe:
# an MoE block without the estimator loses the expert homes that
# attribute routed bytes and stacks to devices; the composed
# charge would collapse toward state-only and keep an
# over-budget grouped lane -- decline instead
return None
if not has_moe:
expert_counts = {}
routed_by_dev = {}
routed_total = None
# routed x6 was calibrated on the per-expert-loop implementation
# (per-expert route caches + batch cats). Grouped mode builds no
# route caches -- its routed working set (sorted flat buffers) is
# covered by the estimator's ratio-scaled expert outputs, and the
# weight-side retention arrives via the stacks term. Charging both
# under grouped mode double-charges and can falsely switch a
# working grouped lane to the per-expert loop.
stacks_any = any(
_grouped_stack_bytes(block, d, config=config) > 0 for d in set(est_by_dev) | set(expert_counts)
)
if has_moe and not stacks_any:
routed_total = _routed_budget_bytes(block, tensors, batch_size, config)
total_experts = sum(expert_counts.values())
if routed_total is not None and total_experts > 0:
routed_by_dev = {d: int(routed_total * c / total_experts) for d, c in expert_counts.items()}
elif routed_total is not None:
# no expert homes resolved (detection is name-based): never drop
# the budget silently -- split evenly across the graph's devices,
# or onto the pool reference's device when no module info at all
if est_by_dev:
n = len(est_by_dev)
routed_by_dev = {d: int(routed_total / n) for d in est_by_dev}
logger.debug(
"[tune] routed budget %.2fGiB split evenly over %d devices (no expert homes)",
routed_total / 2**30,
n,
)
else:
ref_dev = None
if isinstance(tensors, list) and tensors and isinstance(tensors[0], torch.Tensor):
ref_dev = str(tensors[0].device)
elif isinstance(tensors, dict):
for t in tensors.values():
if isinstance(t, torch.Tensor):
ref_dev = str(t.device)
break
if ref_dev is not None:
routed_by_dev = {ref_dev: routed_total}
out = {}
for d in set(est_by_dev) | set(routed_by_dev):
# est split is GiB floats (estimator), routed split is bytes
m = max(int(est_by_dev.get(d, 0.0) * 2**30), int(routed_by_dev.get(d, 0)))
out[d] = m + _grouped_stack_bytes(block, d, config=config)
return out or None
except Exception as e: # pragma: no cover - placement must never break tuning
logger.warning("[tune] activation estimate failed (%s); treating as unknown", e)
return None
def _block_activation_bytes(
block: torch.nn.Module, tensors: Any, batch_size: int, config: Any = None, device: str | None = None
) -> int | None:
"""Back-compat scalar view: this DEVICE's slice of the per-device charge."""
by_dev = _activation_bytes_by_device(block, tensors, batch_size, config)
if by_dev is None:
return None
if device is None:
return max(by_dev.values())
return by_dev.get(str(device))
_MOE_IMPL_AUTO_LINEAR_LOOP_DONE = False
_MOE_IMPL_AUTO_DONE_REF = None # weakref to the deciding run's config
def _maybe_auto_linear_loop_for_tuning(
block: torch.nn.Module, tensors: Any, batch_size: int, iters: int, config: Any, model: torch.nn.Module
) -> None:
"""One-shot auto pick: ``linear_loop`` when grouped tuning cannot fit.
Grouped tuning stacks per-expert fp32 value/grad tensors on every
expert-homing device (chunk-INDEPENDENT autograd retention) ON TOP of the
in-loop tuning state; when a device cannot hold both beside the reserve,
the first backward OOMs only after a wasted grouped_mm attempt plus a
sliced-fallback re-staging. The charges are
the same ones the placement gates use: per-device tuning state charged
at 6 of the 14 B/param layout (grads + bf16 are guaranteed in-loop; the
values are already inside the probed free and the best-params snapshot
parks to host under pressure) plus the per-device activation budget
(routed transients on loop lanes, grouped stacks on grouped lanes).
Any expert-homing device over budget -> switch the WHOLE run to
linear_loop once, loudly. Explicit AR_MOE_EXPERTS_IMPL choices are never
overridden.
"""
global _MOE_IMPL_AUTO_LINEAR_LOOP_DONE, _MOE_IMPL_AUTO_DONE_REF
import weakref
_key_obj = config if config is not None else model
try:
_key = ("w", weakref.ref(_key_obj)) if _key_obj is not None else ("n", None)
except TypeError: # non-weakrefable object: fall back to identity
_key = ("i", id(_key_obj))
_prev = _MOE_IMPL_AUTO_DONE_REF
_same = False
if _prev is not None and _prev[0] == _key[0]:
if _prev[0] == "w":
_a, _b = _prev[1](), _key[1]()
_same = _a is not None and _a is _b # dead ref -> re-decide
else:
_same = _prev[1] == _key[1]
if not _same:
# a new (or garbage-collected) run's config re-decides: no silent
# skip for a second model quantized in the same process
_MOE_IMPL_AUTO_DONE_REF = _key
_MOE_IMPL_AUTO_LINEAR_LOOP_DONE = False
if _MOE_IMPL_AUTO_LINEAR_LOOP_DONE or iters is None or int(iters) <= 0:
return
from auto_round import envs as _envs_mod
from auto_round.modeling.fused_moe.moe_experts_interface import GROUPED_LINEAR_IMPL, LINEAR_LOOP_IMPL
from auto_round.utils.device import probe_usable_bytes
from auto_round.utils.pool_placement import _RESERVE_BYTES
requested = str(getattr(_envs_mod, "AR_MOE_EXPERTS_IMPL", "auto") or "auto").lower()
if requested not in ("", "auto"):
_MOE_IMPL_AUTO_LINEAR_LOOP_DONE = True # explicit choice governs
logger.debug("[moe-impl] auto check skipped: explicit AR_MOE_EXPERTS_IMPL=%s", requested)
return
# The dispatch reads the MODEL's config; model_context.config may be a
# separate object that never received _experts_implementation (a silent
# no-op). Prefer whichever candidate
# actually carries the attr; switch ALL of them when we switch.
candidates = [c for c in (config, getattr(model, "config", None)) if c is not None]
cfg = next((c for c in candidates if getattr(c, "_experts_implementation", None)), None)
impl = str(getattr(cfg, "_experts_implementation", GROUPED_LINEAR_IMPL)) if cfg is not None else GROUPED_LINEAR_IMPL
if impl != GROUPED_LINEAR_IMPL:
logger.debug("[moe-impl] auto check skipped: current impl is %s (not grouped)", impl)
return
try:
state = _logical_state_by_device(block) or {}
if not state:
logger.debug("[moe-impl] auto check skipped: no per-device tuning state resolved")
return
act = _activation_bytes_by_device(block, tensors, batch_size, cfg) or {}
if not any(_grouped_stack_bytes(block, d) > 0 for d in state):
# dense block, loop impl already, or no experts homed: not the
# decision point yet -- retry on the next block
logger.debug("[moe-impl] auto check deferred: no grouped stacks homed on any state device")
return
over = []
max_ratio_dev, max_ratio = None, -1.0
for d, state_bytes in state.items():
# values (4 B/param) are already inside the probed free here; of
# the remainder, grads (4) + bf16 copy (2) are guaranteed in-loop,
# while the best-params snapshot (4) PARKS TO HOST under pressure
# -- charging it as guaranteed over-declines tight grouped lanes
# that were working
remaining_state = state_bytes * 6 // 14
demand = remaining_state + act.get(d, 0) + _RESERVE_BYTES
free = probe_usable_bytes(d)
_st = _grouped_stack_bytes_detail(block, d)
_est_routed = max(act.get(d, 0) - _st["retention"] - _st["transient"], 0)
logger.debug(
"[moe-impl] auto check %s: demand %.2f GiB (state %.2f + act %.2f "
"[stacks retention %.2f + transient %.2f, est/routed %.2f] + reserve) vs free %.2f GiB",
d,
demand / 2**30,
remaining_state / 2**30,
act.get(d, 0) / 2**30,
_st["retention"] / 2**30,
_st["transient"] / 2**30,
_est_routed / 2**30,
-1.0 if free is None else free / 2**30,
)
if free is not None:
ratio = demand / max(int(free), 1)
if ratio > max_ratio:
max_ratio, max_ratio_dev = ratio, d
if demand > int(free):
over.append((d, demand, int(free), remaining_state, act.get(d, 0)))
_MOE_IMPL_AUTO_LINEAR_LOOP_DONE = True
if over:
d, demand, free, st, ac = over[0]
logger.info(
"[moe-impl] auto: grouped tuning needs %.1f GiB on %s (free %.1f: state %.1f + "
"stacks/transients %.1f + reserve) -- switching this run to %s",
demand / 2**30,
d,
free / 2**30,
st / 2**30,
ac / 2**30,
LINEAR_LOOP_IMPL,
)
seen_cfg = {id(c) for c in candidates}
_nm = getattr(block, "named_modules", None)
for _n, m in (_nm() if callable(_nm) else []):
# some archs deepcopy the config per layer; the dispatch
# reads each module's own config, so switch those too
c = getattr(m, "config", None)
if c is not None and id(c) not in seen_cfg:
candidates.append(c)
seen_cfg.add(id(c))
for c in candidates:
c._experts_implementation = LINEAR_LOOP_IMPL
elif max_ratio_dev is not None:
logger.debug(
"[moe-impl] auto: grouped fits (max demand/free ratio %.2f on %s)",
max_ratio,
max_ratio_dev,
)
except Exception as e: # never break tuning over an advisory auto-pick
logger.debug("[moe-impl] auto feasibility check failed (%s); keeping grouped", e)
def _pull_pool_if_fits(
pool: Any,
target_dev: str,
block: torch.nn.Module,
batch_size: int,
iters: int,
label: str,
charge_activation: bool = False,
config: Any = None,
) -> Any:
"""Bulk-move a whole tune-loop pool onto the device that reads it every iteration.
iters>0 strategy only (loop-amortized): ``active_inputs`` -> the entry
device the forward gathers onto (BlockForwardRunner.device, home of the
block's starting modules), ``fp_outputs`` -> the loss device. The iters=0
lane never calls this -- its pools are single-pass streamed, so a bulk
move buys nothing. Declines (pool stays sharded behind the per-batch
gather) unless free(target) covers the pool beside the loop transients
and the target's own in-loop tuning state (14 B/param), which
materializes after this probe -- the term that separates the 8-GPU lane
(pulls) from the 4-GPU lane (declines; peers completed the loop at the
same state density without the pool). The loss side has no per-iteration
graph retention (one cat per iteration, freed after the loss -- validated
by the full 80-block pull run).
``charge_activation`` adds the block's ACTUAL activation accounting
(per-module output shapes, MoE-ratio scaled -- see
``_block_activation_bytes``) and is set only for the input pull: the
entry device carries the loop's forward graph (batch cats + routed-row
caches materialize along the hidden flow regardless of weight homes).
"""
if pool is None or target_dev is None:
return pool
_act_bytes = _block_activation_bytes(block, pool, batch_size, config, str(target_dev)) if charge_activation else 0
if charge_activation and _act_bytes is None:
return pool # unknown activation cost: never pull blind
_tgt = torch.device(target_dev)
try:
tensors = pool if isinstance(pool, list) else None
if tensors is None or not tensors or not any(hasattr(t, "device") for t in tensors):
return pool
if all(t.device == _tgt for t in tensors):
return pool
pool_b = sum(t.numel() * t.element_size() for t in tensors)
if charge_activation:
pool_b += _act_bytes
try:
from auto_round.utils.device import probe_usable_bytes
free = probe_usable_bytes(str(_tgt))
except Exception as e: # pragma: no cover - diagnostics only
logger.warning("[%s] free-memory probe failed for %s (%s); keeping pool sharded", label, _tgt, e)
free = None
try:
from auto_round.utils.pool_placement import _RESERVE_BYTES, _working_allowance_bytes
reserve = _working_allowance_bytes(block, tensors, batch_size) + _RESERVE_BYTES
reserve += _tuning_state_bytes(block, _tgt)
except Exception as e: # pragma: no cover - gate must never break tuning
logger.warning("[%s] working-set estimate failed (%s); using flat 4GiB reserve", label, e)
reserve = 4 << 30
if free is not None and free - reserve >= pool_b:
moved = [t.to(_tgt) for t in tensors]
# splice in place: the tune loop rebinds only its local handle
# while the pool containers (release bookkeeping, snapshots)
# still hold the ORIGINAL list -- returning a fresh list would
# keep the source tensors resident next to the pulled copies
pool[:] = moved
logger.debug("[%s bulk pull -> %s (%.2f GiB, free %.2f GiB)", label, _tgt, pool_b / 2**30, free / 2**30)
return pool
if free is not None:
logger.debug(
"[%s stays sharded, per-batch gather (data %.2f GiB + reserve %.2f GiB vs free %.2f GiB on %s)",
label,
pool_b / 2**30,
reserve / 2**30,
free / 2**30,
_tgt,
)
except Exception as e: # pragma: no cover - placement must never break tuning
logger.warning("[%s bulk pull failed (%s); keeping data sharded", label, e)
return pool