Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: AutoScheme parallel scoring OOMs (MoE memory, VLM gate) and disk-stream tuning crash (#2220)

avtc7da9bea890b2
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index 3f405fb3e..c7207bea6 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -84,6 +84,43 @@ from auto_round.wrapper import WrapperLinear
__all__ = ["gen_layer_config"]
+class _ScoreLinear(torch.autograd.Function):
+ """Linear for scoring passes whose weight lives in the wrapper's CPU cache.
+
+ Saves only the input activation: the dequantized weight is transferred
+ from the cache again wherever needed, so the block's autograd graph holds
+ no weight-shaped tensors. The weight score is accumulated from the
+ explicit gradient instead of a tensor hook.
+ """
+
+ @staticmethod
+ def forward(ctx, x, wrapper, bias): # pylint: disable=arguments-differ
+ ctx.wrapper = wrapper
+ ctx.save_for_backward(x)
+ weight = wrapper._score_weight_for_device(x.device)
+ return torch.nn.functional.linear(x, weight, bias)
+
+ @staticmethod
+ def backward(ctx, grad_out): # pylint: disable=arguments-differ
+ (x,) = ctx.saved_tensors
+ wrapper = ctx.wrapper
+ weight = wrapper._score_weight_for_device(x.device)
+ grad_x = None
+ if ctx.needs_input_grad[0]:
+ grad_x = torch.matmul(grad_out, weight)
+ if wrapper.grad_mode and wrapper.need_weight_grad:
+ grad_out_2d = grad_out.reshape(-1, grad_out.shape[-1])
+ x_2d = x.reshape(-1, x.shape[-1])
+ grad_w = torch.mm(grad_out_2d.t(), x_2d)
+ weight_ref = wrapper.orig_layer.weight
+ if weight_ref.device.type == "meta":
+ weight_ref = wrapper.orig_layer.get_weight().to(x.device)
+ w_diff = weight_ref.to(x.device) - weight
+ wrapper.weight_score += torch.abs(grad_w * w_diff).sum().item()
+ wrapper.mix_score = wrapper.weight_score + wrapper.act_score
+ return grad_x, None, None
+
+
class AutoSchemeWrapperLinear(WrapperLinear):
def __init__(
@@ -121,6 +158,12 @@ class AutoSchemeWrapperLinear(WrapperLinear):
self.grad_mode = False
if self.need_weight_grad:
self.orig_layer.weight.requires_grad = True
+ # scoring cache: the dequantized weight, computed once per wrapper
+ # visit and stored on CPU; bounded to the block being replayed so
+ # parallel workers' caches stay within host RAM
+ self._score_qdq_cpu = None
+ # use the recompute-weight scoring forward (see _ScoreLinear)
+ self._custom_score_forward = True
def _qdq_act(self, x, act_min_scale=1.0, act_max_scale=1.0, act_max=None):
"""Quant-dequant the activation and, in ``grad_mode``, register a backward hook that
@@ -161,38 +204,85 @@ class AutoSchemeWrapperLinear(WrapperLinear):
qdq_x.register_hook(save_grad)
return qdq_x, scale, zp
+ def _ensure_score_cache(self):
+ """Build the per-visit CPU cache of the dequantized weight (scoring layers only)."""
+ if self._score_qdq_cpu is not None:
+ return
+ device = self.device
+ with torch.no_grad():
+ qdq_w, _, _ = super(AutoSchemeWrapperLinear, self)._qdq_weight(
+ torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device)
+ )
+ self._score_qdq_cpu = qdq_w.detach().to("cpu")
+
+ def _score_weight_for_device(self, device):
+ """Return the cached dequantized weight on ``device`` (transfers from the CPU cache)."""
+ self._ensure_score_cache()
+ return self._score_qdq_cpu.to(device)
+
+ def forward(self, x):
+ """In the scoring replay, run the linear through ``_ScoreLinear`` so the block's
+ autograd graph saves activations only; every other phase uses the base forward."""
+ if getattr(self, "_custom_score_forward", False) and self.grad_mode and torch.is_grad_enabled():
+ x = x.to(self.device)
+ for hook in self.orig_layer._forward_pre_hooks.values():
+ result = hook(self.orig_layer, (x,))
+ if result is not None:
+ x = result[0] if isinstance(result, tuple) else result
+ bias = self.orig_layer.bias
+ if bias is not None and bias.device.type == "meta":
+ bias = self.orig_layer.get_bias().to(self.device)
+ output = _ScoreLinear.apply(x, self, bias)
+ for hook in self.orig_layer._forward_hooks.values():
+ hook_result = hook(self.orig_layer, (x,), output)
+ if hook_result is not None:
+ output = hook_result
+ return output.to(self.output_device)
+ return super().forward(x)
+
def _qdq_weight(self, value, min_scale, max_scale):
- """Quant-dequant the weight and, in ``grad_mode``, register a backward hook that
- accumulates ``weight_score`` from ``|grad * (weight - qdq_w)|``. Weight quantization
- for the hook is only recomputed lazily inside the hook itself, so layers whose
- forward never runs (e.g. unrouted MoE experts) never pay this cost.
+ """Quant-dequant the weight once per scoring pass, cache the result, and, in
+ ``grad_mode``, route the score hook through a scalar anchor.
+
+ The cached quantization is recomputed lazily on the first call, so layers
+ whose forward never runs (e.g. unrouted MoE experts) never pay this cost.
+ In ``grad_mode`` the anchor makes the returned tensor a graph node whose
+ gradient is exactly d(loss)/d(qdq_w): the score hook fires without building
+ the quant chain into the graph and without materializing weight-shaped
+ gradient buffers, which on expert-heavy blocks dominated replay VRAM.
"""
device = self.device
- if self.orig_layer.bits > 8 or not self.need_weight_grad:
- qdq_w, scale, zp = super()._qdq_weight(
+ if self.orig_layer.bits > 8:
+ return super()._qdq_weight(
torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device)
)
- return qdq_w, 1.0, None
-
- qdq_w, scale, zp = super()._qdq_weight(
- torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device)
- )
+ scoring = self.grad_mode and self.need_weight_grad
+ if not scoring:
+ # one-shot layers (capture pass, non-grad phases) never reuse the
+ # result, so do not grow a CPU-side cache for them
+ with torch.no_grad():
+ return super()._qdq_weight(
+ torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device)
+ )
- if self.grad_mode:
+ self._ensure_score_cache()
+ qdq_w = self._score_qdq_cpu.to(device)
+ if scoring:
+ anchor = torch.zeros((), dtype=qdq_w.dtype, device=qdq_w.device, requires_grad=True)
+ qdq_w = qdq_w + 0.0 * anchor
def save_grad(grad):
"""Backward hook: accumulate weight score from grad * (weight - qdq_w)."""
- qdq_w, scale, zp = self.super_qdq_func(
- torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device)
- )
- w_diff = self.orig_layer.weight - qdq_w.to(self.orig_layer.weight.device)
- self.weight_score += torch.abs((grad.to(w_diff.device) * w_diff)).sum().item()
+ weight = self.orig_layer.weight
+ if weight.device.type == "meta":
+ weight = self.orig_layer.get_weight().to(grad.device)
+ w_diff = weight.to(grad.device) - self._score_qdq_cpu.to(grad.device)
+ self.weight_score += torch.abs(grad.to(w_diff.device) * w_diff).sum().item()
self.mix_score = self.weight_score + self.act_score
return None
- if qdq_w.requires_grad:
- qdq_w.register_hook(save_grad)
+ qdq_w.register_hook(save_grad)
return qdq_w, 1.0, None
@@ -240,6 +330,7 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
group_size=orig_layer.group_size,
iters=0,
)
+ self._custom_score_forward = False
self.post_init_qdqw(device)
@torch.no_grad()
@@ -342,6 +433,7 @@ class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):
need_weight_grad,
**kwargs,
)
+ self._custom_score_forward = False
self.post_init_qdqw(device)
@torch.no_grad()
@@ -400,6 +492,7 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
enable_torch_compile=enable_torch_compile,
**kwargs,
)
+ self._custom_score_forward = False
self.post_init_qdqw(device)
@torch.no_grad()
@@ -596,9 +689,12 @@ def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_dev
materialize_module(module, module_name, disk_index, device=major_device)
move_module_to_tuning_device(module, major_device=major_device)
- # for n,m in module.named_modules():
- # if hasattr(m, "post_init_qdqw"):
- # m.post_init_qdqw()
+ # The block now sits on major_device; its incoming tensors may have
+ # been emitted on another device (e.g. CPU-resident embeddings, or
+ # CPU-recorded replay inputs), so align them before the forward --
+ # a same-device .to() is a no-op reference return.
+ args = tuple(a.to(major_device) if isinstance(a, torch.Tensor) else a for a in args)
+ kwargs = {k: v.to(major_device) if isinstance(v, torch.Tensor) else v for k, v in kwargs.items()}
# Call the original forward
with torch.no_grad():
@@ -695,6 +791,34 @@ def model_forward(model, data, **forward_kwargs):
return model(**prepared, **forward_kwargs), prepared
+def _clear_wrapper_score_caches(block_module):
+ """Drop the per-wrapper scoring caches of every layer in ``block_module`."
+
+ Wrappers persist for a whole scoring pass while their caches are only valid
+ while the owning layer's weights are materialized; without this the CPU-side
+ caches accumulate across blocks of the model.
+ """
+ for module in block_module.modules():
+ if getattr(module, "_score_qdq_cpu", None) is not None:
+ module._score_qdq_cpu = None
+
+
+def _replay_retain_graph(block_module) -> bool:
+ """Whether this block's backward must keep its autograd graph alive.
+
+ MX-family data types re-run quantization code inside backward hooks that
+ expect the block graph to still exist. Every other data type frees the
+ graph per block: the reverse replay holds one block's saved activations at
+ a time, and retaining graphs across blocks accumulates them for the whole
+ pass, which is what pushes streamed scoring workers OOM mid-pass.
+ """
+ for _, module in block_module.named_modules():
+ data_type = getattr(module, "data_type", None)
+ if isinstance(data_type, str) and data_type.startswith("mx"):
+ return True
+ return False
+
+
def _prepare_replay_input(block_input_args, block_input_kwargs, block_name):
"""Find the floating hidden-state tensor whose gradient feeds the preceding block."""
candidates = []
@@ -709,7 +833,184 @@ def _prepare_replay_input(block_input_args, block_input_kwargs, block_name):
raise RuntimeError(f"No floating replay input found for block {block_name}")
-def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, scheme_tag=None, disk_index=None):
+def _vram_inventory_text(top_k: int = 12) -> str:
+ """Return the live-CUDA-tensor census as text (see ``_vram_inventory``).
+
+ Grouped by (shape, dtype) signature, largest first -- block weight shapes
+ are recognizable, which is what makes an at-failure census actionable.
+ """
+ import gc as _gc
+ from collections import defaultdict
+
+ if not torch.cuda.is_available():
+ return "cuda unavailable; no census"
+ groups = defaultdict(lambda: [0, 0])
+ for obj in _gc.get_objects():
+ try:
+ if torch.is_tensor(obj) and obj.is_cuda:
+ key = (tuple(obj.shape), str(obj.dtype))
+ groups[key][0] += 1
+ groups[key][1] += obj.element_size() * obj.numel()
+ except Exception: # noqa: BLE001
+ continue
+ ranked = sorted(groups.items(), key=lambda kv: -kv[1][1])[:top_k]
+ total = sum(v[1] for v in groups.values())
+ alloc = torch.cuda.memory_allocated() / 2**30
+ lines = [f"live tensors {total / 2**30:.2f} GiB (allocator {alloc:.2f} GiB)"]
+ for (shape, dtype), (cnt, nb) in ranked:
+ lines.append(f" {nb / 2**30:6.2f} GiB x{cnt:<4} {dtype} {shape}")
+ # name the retaining objects for the largest group so the census points at
+ # the retainer, not only the retained shape
+ for rank, ((shape, dtype), _) in enumerate(ranked[:2]):
+ target = None
+ objs = _gc.get_objects()
+ for obj in objs:
+ try:
+ if torch.is_tensor(obj) and obj.is_cuda and tuple(obj.shape) == shape:
+ target = obj
+ break
+ except Exception: # noqa: BLE001
+ continue
+ # The enumeration list itself references every object it yielded; drop
+ # it before asking for referrers or it shadows the real retainers.
+ del objs
+ if target is not None:
+ trace = _tensor_referrer_snippet(target)
+ if trace:
+ lines.append(f" group#{rank} referrers {shape}:\n{trace}")
+ return "\n".join(lines)
+
+
+def _tensor_referrer_snippet(tensor, max_depth: int = 3, max_entries: int = 4) -> str:
+ """Describe what holds ``tensor`` alive, as a compact referrer chain.
+
+ Walks ``gc.get_referrers`` breadth-first, reporting container types (list /
+ dict sizes, module class names) so a retention census names the retaining
+ object rather than only the retained shape.
+ """
+ import gc as _gc
+
+ import torch.nn as _nn
+
+ def _describe(obj, refers_to=None):
+ if isinstance(obj, (list, tuple)):
+ return f"{type(obj).__name__}[{len(obj)}]"
+ if isinstance(obj, dict):
+ keys = list(obj.keys())[:max_entries]
+ shown = [k if isinstance(k, str) else type(k).__name__ for k in keys]
+ desc = "dict{" + ", ".join(map(str, shown)) + "}"
+ if refers_to is not None:
+ # name the holding attribute deterministically: identity-search
+ # the walked object among the dict's values, so the snippet does
+ # not depend on key order or gc traversal order across platforms
+ for key, value in obj.items():
+ if value is refers_to and isinstance(key, str):
+ return f"{desc}[holding {key!r}]"
+ return desc
+ if isinstance(obj, _nn.Module):
+ desc = f"module:{type(obj).__name__}"
+ if refers_to is not None:
+ for store_name in ("_parameters", "_buffers", "_modules"):
+ store = getattr(obj, store_name, None)
+ if isinstance(store, dict):
+ for attr_name, value in store.items():
+ if value is refers_to:
+ return f"{desc}.{store_name}[{attr_name!r}]"
+ for attr_name, value in obj.__dict__.items():
+ if value is refers_to:
+ return f"{desc}.{attr_name}"
+ return desc
+ return type(obj).__name__
+
+ seen = {id(tensor)}
+ frontier = [tensor]
+ lines = []
+ for _depth in range(max_depth):
+ next_frontier = []
+ for obj in frontier:
+ try:
+ referrers = _gc.get_referrers(obj)
+ except Exception: # noqa: BLE001
+ continue
+ for ref in referrers:
+ if id(ref) in seen:
+ continue
+ seen.add(id(ref))
+ desc = _describe(ref)
+ if desc in ("frame", "builtin_function_or_method", "function"):
+ continue
+ if isinstance(ref, (list, tuple)) and len(ref) > 10000:
+ # giant containers at this depth are bookkeeping artifacts
+ # (object registries, gc internals), not model state
+ continue
+ lines.append(f" depth{_depth + 1} <- {_describe(ref, refers_to=obj)}")
+ if isinstance(ref, (list, tuple, dict)):
+ next_frontier.append(ref)
+ if len(lines) >= max_entries * 3:
+ break
+ frontier = next_frontier
+ if not frontier or len(lines) >= max_entries * 3:
+ break
+ return "\n".join(lines[: max_entries * 3]) if lines else "no external referrers found"
+
+
+def _annotate_worker_oom(worker_index, exc):
+ """Attach a live-tensor census and the failing op to a CUDA OOM in a worker."""
+ import traceback as _tb
+
+ try:
+ census = _vram_inventory_text(top_k=20)
+ except Exception: # noqa: BLE001 the census must never mask the OOM
+ census = "census unavailable"
+ tb_lines = _tb.format_exc().splitlines()
+ tb_head = "\n".join(tb_lines[:4])
+ tb_tail = "\n".join(tb_lines[-8:])
+ return RuntimeError(
+ f"_score_scheme_worker[{worker_index}] CUDA OOM during scoring. "
+ f"Live-tensor census at failure:\n{census}\n\n"
+ f"Traceback head:\n{tb_head}\n...\nTraceback tail:\n{tb_tail}"
+ f"Original error: {exc}"
+ )
+
+
+def _vram_inventory(tag: str, top_k: int = 12):
+ """Env-gated (AR_SCHEME_MEM_INVENTORY=1) per-device live-tensor census:
+ resident CUDA tensors grouped by (shape, dtype) signature, largest first --
+ used to spot unreleased module weights or retained graphs during streamed
+ scoring (block shapes are recognizable: e.g. [5120, 17408] = mlp.down)."""
+ import gc as _gc
+ import os as _os
+ from collections import defaultdict
+
+ if _os.getenv("AR_SCHEME_MEM_INVENTORY", "0") != "1":
+ return
+ groups = defaultdict(lambda: [0, 0])
+ for obj in _gc.get_objects():
+ try:
+ if torch.is_tensor(obj) and obj.is_cuda:
+ key = (tuple(obj.shape), str(obj.dtype))
+ groups[key][0] += 1
+ groups[key][1] += obj.element_size() * obj.numel()
+ except Exception: # noqa: BLE001
+ continue
+ ranked = sorted(groups.items(), key=lambda kv: -kv[1][1])[:top_k]
+ total = sum(v[1] for v in groups.values())
+ alloc = torch.cuda.memory_allocated() / 2**30
+ print(f"[mem-inv] {tag}: live tensors {total / 2**30:.2f} GiB " f"(allocator {alloc:.2f} GiB)", flush=True)
+ for (shape, dtype), (cnt, nb) in ranked:
+ print(f"[mem-inv] {nb / 2**30:6.2f} GiB x{cnt:<4} {dtype} {shape}", flush=True)
+
+
+def model_forward_low_gpu(
+ model,
+ dataloader,
+ major_device="cuda",
+ pbar=None,
+ scheme_tag=None,
+ disk_index=None,
+ skip_batches=0,
+ batch_checkpoint=None,
+):
"""Run one full scoring pass (all calibration batches) in low-GPU-memory mode.
For each batch: capture per-block inputs via ``prepare_model_low_gpu``, run a forward
@@ -741,6 +1042,12 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, sch
raise MyCustomError("Interrupt backward pass")
for batch_idx, data in enumerate(dataloader, start=1):
+ if batch_idx <= skip_batches:
+ # resumed run: these batches' contributions are already in the
+ # restored accumulators; advance the loader and the progress bar only
+ if pbar is not None:
+ pbar.update(len(block_names) * 2)
+ continue
captured_grad = None
interrupted = False
last_block_backward_hook = None
@@ -835,7 +1142,7 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, sch
torch.autograd.backward(
tensors=main_output,
grad_tensors=current_grad,
- retain_graph=True, # False may lead to zero gradients for some cases (e.g., MXFP4)
+ retain_graph=_replay_retain_graph(block_module),
)
if replay_input.grad is None:
@@ -845,6 +1152,7 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, sch
finally:
for parameter in block_module.parameters():
parameter.grad = None
+ _clear_wrapper_score_caches(block_module)
if disk_index is not None and materialized:
from auto_round.utils.disk_stream_util import free_module
@@ -858,6 +1166,11 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, sch
if pbar is not None:
pbar.update(1)
+ _vram_inventory(f"scheme={scheme_tag} batch={batch_idx}/{total_batches} post-replay")
+
+ if batch_checkpoint is not None:
+ batch_checkpoint(batch_idx, total_batches or -1)
+
_log_batch_avg_loss(
model,
batch_idx,
@@ -877,6 +1190,8 @@ def get_score_for_scheme(
ignore_scale_zp_bits=False,
nsamples=16,
seqlen=256,
+ skip_batches=0,
+ batch_checkpoint=None,
pbar=None,
shared_layers=None,
need_weight_grad=False,
@@ -1114,7 +1429,14 @@ def get_score_for_scheme(
"Provide a `processor` and a multimodal `dataset`."
)
model_forward_low_gpu(
- model, mllm_loader, major_device=major_device, pbar=pbar, scheme_tag=scheme_tag, disk_index=disk_index
+ model,
+ mllm_loader,
+ major_device=major_device,
+ pbar=pbar,
+ scheme_tag=scheme_tag,
+ disk_index=disk_index,
+ skip_batches=skip_batches,
+ batch_checkpoint=batch_checkpoint,
)
else:
try:
@@ -1126,6 +1448,8 @@ def get_score_for_scheme(
pbar=pbar,
scheme_tag=scheme_tag,
disk_index=disk_index,
+ skip_batches=skip_batches,
+ batch_checkpoint=batch_checkpoint,
)
except Exception as exc: # noqa: BLE001
if not is_vlm:
@@ -1636,6 +1960,57 @@ def _autoscheme_cache_path(cache_key, scheme_index):
return os.path.join(cache_dir, f"scheme_{scheme_index:02d}_{cache_key}.json")
+def _extract_score_accumulators(model):
+ """Snapshot the additive per-layer scoring accumulators for mid-scheme resume."""
+ state = {}
+ for n, m in model.named_modules():
+ if hasattr(m, "mix_score"):
+ state[n] = [float(m.act_score), float(m.weight_score), float(m.act_cnt)]
+ return state
+
+
+def _inject_score_accumulators(model, state):
+ """Restore scoring accumulators snapshotted by ``_extract_score_accumulators``."""
+ for n, m in model.named_modules():
+ if n in state and hasattr(m, "mix_score"):
+ m.act_score, m.weight_score, m.act_cnt = state[n]
+
+
+def _partial_scores_path(cache_path):
+ return cache_path + ".partial"
+
+
+def _save_partial_scores(cache_path, batch_idx, total_batches, state):
+ """Atomically persist a batch-granularity scoring checkpoint (JSON)."""
+ payload = {
+ "version": 1,
+ "batches_done": batch_idx,
+ "total_batches": total_batches,
+ "scores_state": state,
+ }
+ tmp = _partial_scores_path(cache_path) + ".tmp"
+ with open(tmp, "w", encoding="utf-8") as f:
+ json.dump(payload, f)
+ os.replace(tmp, _partial_scores_path(cache_path))
+
+
+def _load_partial_scores(cache_path, expected_total_batches):
+ """Load a batch checkpoint if present and still applicable."""
+ path = _partial_scores_path(cache_path)
+ if cache_path is None or not os.path.exists(path):
+ return None
+ try:
+ with open(path, encoding="utf-8") as f:
+ data = json.load(f)
+ if data.get("total_batches") != expected_total_batches:
+ return None
+ if not 0 < data.get("batches_done", 0) < expected_total_batches:
+ return None
+ return data
+ except Exception: # noqa: BLE001
+ return None
+
+
def _save_autoscheme_scores(
cache_path,
cache_key,
@@ -1786,6 +2161,21 @@ def _refresh_cached_layer_bits(
return refreshed_scores
+def _parallel_scoring_must_raise(parallel_error: Exception) -> bool:
+ """Decide whether a parallel-scoring failure aborts instead of falling back to serial.
+
+ Set AR_AUTO_SCHEME_NO_SERIAL_FALLBACK=1 to fail fast: the serial fallback is
+ ~workers-count times slower and, on models where it cannot run at all, only
+ burns hours before crashing. Schemes lost with their worker always raise
+ regardless of the env, since their scores are unrecoverable.
+ """
+ from auto_round import envs as _envs
+
+ if "was lost with its worker" in str(parallel_error):
+ return True
+ return bool(_envs.AR_AUTO_SCHEME_NO_SERIAL_FALLBACK)
+
+
def _assign_scheme_worker_devices(worker_count, available_devices):
"""Assign workers round-robin within the devices selected by the caller."""
if not available_devices:
@@ -1857,6 +2247,40 @@ def _get_scheme_worker_count(num_schemes, num_gpus):
return num_schemes
+def _weights_span_multiple_gpus(devices) -> bool:
+ """Whether a device collection places weights on more than one GPU."""
+ cuda_devices = {str(device) for device in devices if str(device).startswith("cuda")}
+ return len(cuda_devices) > 1
+
+
+def _serial_scoring_device_safe(model, visible_cuda_devices=None) -> bool:
+ """Whether in-process (serial) full-model scoring can run on ``model``.
+
+ Serial scoring runs the whole-model forward through per-layer wrappers.
+ Two placement sources can spread work over several GPUs: live parameter
+ devices (a materialized model), and ``hf_device_map`` - which is what a
+ disk-stream (meta-skeleton) model consults when its blocks materialize
+ block-by-block during scoring. Either spanning more than one GPU makes
+ the first cross-device layer raise a tensor-device error mid-forward.
+
+ Additionally, a disk-stream (meta-skeleton) model scored serially with
+ more than one visible CUDA device is not considered safe: block
+ materialization and wrapper bookkeeping are only validated with a
+ single visible GPU, and workers pin one GPU each. Such models are
+ routed through disk-stream workers instead.
+ """
+ if _weights_span_multiple_gpus({parameter.device for parameter in model.parameters()}):
+ return False
+ hf_device_map = getattr(model, "hf_device_map", None)
+ if hf_device_map:
+ if _weights_span_multiple_gpus(set(hf_device_map.values())):
+ return False
+ if visible_cuda_devices is not None and len(visible_cuda_devices) > 1:
+ if getattr(model, "_disk_stream_index", None) is not None:
+ return False
+ return True
+
+
def _can_parallel_scheme_scoring(
parallel_enabled,
model_id,
@@ -1866,16 +2290,26 @@ def _can_parallel_scheme_scoring(
disk_stream_model,
is_vlm,
low_gpu_mem_usage=True,
+ force_mllm=False,
+ min_uncached=2,
):
- """Return whether candidate schemes can be scored in separate workers."""
+ """Return whether candidate schemes can be scored in separate workers.
+
+ ``is_vlm`` is accepted for call-site compatibility; text-only scoring of a
+ VLM's language tower streams blocks just like a text model. Only vision
+ scoring (``force_mllm``) requires a full-model backward and is excluded
+ from the block-wise materialize/free path used by disk streaming.
+ ``min_uncached`` is the parallel-worthiness floor (2 by default); the call
+ site lowers it to 1 when serial scoring cannot run on the loaded model and
+ a disk-stream worker can."""
return (
parallel_enabled
and low_gpu_mem_usage
and model_id is not None
and num_gpus >= 1
- and uncached_count >= 2
+ and uncached_count >= min_uncached
and not need_imatrix
- and (not disk_stream_model or not is_vlm)
+ and (not disk_stream_model or not force_mllm)
)
@@ -1895,6 +2329,15 @@ def _load_disk_stream_scheme_worker_model(model_name, use_model_replacements=Fal
from auto_round.utils.disk_stream_util import build_meta_model
model, tokenizer, disk_index = build_meta_model(model_name)
+ # The regular load pipeline applies custom replacements, among them the
+ # structural MoE unfusing that turns fused expert containers into per-expert
+ # Linear modules; the meta-skeleton build skips that pipeline. Unfuse here
+ # so per-expert quant layers resolve -- weights stay on meta and are filled
+ # per-block from the checkpoint index during scoring.
+ from auto_round.modeling.fused_moe.replace_modules import _handle_moe_modules
+
+ unfused = _handle_moe_modules(model)
+ logger.info("disk-stream scoring worker: structural MoE unfuse produced %d unfused experts modules", len(unfused))
if use_model_replacements:
from auto_round.special_model_handler import _handle_special_model, update_module
@@ -1904,8 +2347,12 @@ def _load_disk_stream_scheme_worker_model(model_name, use_model_replacements=Fal
def _prefer_disk_stream_scheme_worker(model_id, is_vlm, low_gpu_mem_usage):
- """Prefer block-wise disk streaming whenever the worker scoring path supports it."""
- return model_id is not None and not is_vlm and low_gpu_mem_usage
+ """Prefer block-wise disk streaming whenever the worker scoring path supports it.
+
+ Multimodal archs are covered: build_meta_model resolves the class from
+ config.architectures and the workers materialize non-block params (vision
+ tower included) via materialize_non_block_params."""
+ return model_id is not None and low_gpu_mem_usage
def _score_scheme_worker(args):
@@ -1934,6 +2381,7 @@ def _score_scheme_worker(args):
total_schemes,
progress_queue,
disk_stream_model,
+ worker_cache_path,
) = args
from auto_round.auto_scheme.utils import _scheme_short_name as _short_name
@@ -1997,8 +2445,13 @@ def _score_scheme_worker(args):
for layer_name in quant_layer_names:
layer = _get_module(model, layer_name)
if layer is None:
+ parent_name = layer_name.rsplit(".", 1)[0]
+ parent = _get_module(model, parent_name)
+ expert_containers = sum(1 for _, m in model.named_modules() if m.__class__.__name__ == "_ExpertContainer")
raise RuntimeError(
- f"_score_scheme_worker[{index}]: layer {layer_name!r} is missing after model preprocessing"
+ f"_score_scheme_worker[{index}]: layer {layer_name!r} is missing after model preprocessing "
+ f"(parent {parent_name!r} resolves to {type(parent).__name__ if parent is not None else None}; "
+ f"expert containers in model: {expert_containers}; disk-stream build: {disk_index is not None})"
)
if not hasattr(layer, "tuning_device"):
layer.tuning_device = worker_device
@@ -2011,6 +2464,27 @@ def _score_scheme_worker(args):
scheme=scheme,
)
+ # mid-scheme resume: reload additive accumulators and skip completed batches
+ skip_batches = 0
+ if worker_cache_path is not None:
+ expected_total = max(1, math.ceil(nsamples / max(1, batch_size)))
+ partial = _load_partial_scores(worker_cache_path, expected_total)
+ if partial is not None:
+ _inject_score_accumulators(model, partial["scores_state"])
+ skip_batches = partial["batches_done"]
+ logger.info(
+ "_score_scheme_worker[%d]: resuming from batch checkpoint %d/%d",
+ index,
+ skip_batches,
+ expected_total,
+ )
+
+ def batch_checkpoint(batch_idx, total_batches):
+ _save_partial_scores(worker_cache_path, batch_idx, total_batches, _extract_score_accumulators(model))
+
+ else:
+ batch_checkpoint = None
+
is_bf16 = isinstance(scheme, str) and scheme.upper() == "BF16"
if isinstance(scheme, dict):
is_bf16 = scheme.get("bits", 16) >= 16 and scheme.get("act_bits", 16) >= 16
@@ -2025,29 +2499,36 @@ def _score_scheme_worker(args):
from auto_round.auto_scheme.delta_loss import get_score_for_scheme
- scores = get_score_for_scheme(
- model,
- tokenizer,
- quant_layer_names,
- fixed_layer_scheme,
- dataset,
- ignore_scale_zp_bits=ignore_scale_zp_bits,
- pbar=_ProgressQueueProxy(progress_queue),
- nsamples=nsamples,
- seqlen=seqlen,
- need_weight_grad=need_weight_grad,
- enable_torch_compile=enable_torch_compile,
- low_gpu_mem_usage=low_gpu_mem_usage,
- major_device=worker_device,
- batch_size=batch_size,
- offload_context=None,
- processor=processor,
- is_vlm=is_vlm,
- force_mllm=force_mllm,
- model_name=model_name,
- scheme_tag=f"{index + 1}/{total_schemes} {_short_name(scheme)}",
- disk_index=disk_index,
- )
+ try:
+ scores = get_score_for_scheme(
+ model,
+ tokenizer,
+ quant_layer_names,
+ fixed_layer_scheme,
+ dataset,
+ ignore_scale_zp_bits=ignore_scale_zp_bits,
+ pbar=_ProgressQueueProxy(progress_queue),
+ nsamples=nsamples,
+ seqlen=seqlen,
+ skip_batches=skip_batches,
+ batch_checkpoint=batch_checkpoint,
+ need_weight_grad=need_weight_grad,
+ enable_torch_compile=enable_torch_compile,
+ low_gpu_mem_usage=low_gpu_mem_usage,
+ major_device=worker_device,
+ batch_size=batch_size,
+ offload_context=None,
+ processor=processor,
+ is_vlm=is_vlm,
+ force_mllm=force_mllm,
+ model_name=model_name,
+ scheme_tag=f"{index + 1}/{total_schemes} {_short_name(scheme)}",
+ disk_index=disk_index,
+ )
+ except RuntimeError as exc:
+ if "out of memory" not in str(exc).lower():
+ raise
+ raise _annotate_worker_oom(index, exc) from exc
return index, scores, _get_worker_memory_report(worker_device)
@@ -2256,7 +2737,11 @@ def _gen_layer_config(
act_bits = scheme.get("act_bits", 16)
if scheme.get("super_group_size"):
need_imatrix = True
- if bits <= 8 < act_bits:
+ # Weight scores must accumulate for every weight-quantized scheme, not
+ # only A16 ones: MX options (W4A4/W8A8) quantify weight error solely via
+ # the weight score, and skipping it zeroes every layer score, so the
+ # bit allocation collapses to the cheapest option.
+ if bits <= 8:
need_weight_grad = True
if not auto_scheme.low_gpu_mem_usage:
pbar_cnt += nsamples
@@ -2403,6 +2888,10 @@ def _gen_layer_config(
def _save_per_op_scores(index, scheme, cache_key, cache_path, per_op_scores):
if cache_key is None or cache_path is None:
return
+ try:
+ os.remove(_partial_scores_path(cache_path))
+ except OSError:
+ pass
if isinstance(scheme, str):
scheme_dict = asdict(preset_name_to_scheme(scheme))
elif isinstance(scheme, QuantizationScheme):
@@ -2469,6 +2958,46 @@ def _gen_layer_config(
worker_disk_stream_model = _prefer_disk_stream_scheme_worker(
_model_id_for_cache, is_vlm, auto_scheme.low_gpu_mem_usage
)
+ # Serial scoring runs the full-model forward, which cannot cross GPUs on
+ # a manually sharded model. When that is the loaded configuration and a
+ # disk-stream worker is possible, lower the parallel floor to 1 so even
+ # a single uncached scheme is scored through a worker instead of the
+ # broken in-process pass.
+ serial_device_safe = _serial_scoring_device_safe(model, visible_cuda_devices=worker_device_pool)
+ min_uncached = 2
+ if not serial_device_safe and worker_disk_stream_model:
+ min_uncached = 1
+ logger.info(
+ "AutoScheme: serial in-process scoring is unsafe (multi-GPU weights, "
+ "hf_device_map, or a streamed model with several visible GPUs); "
+ "routing even a single uncached scheme through a disk-stream scoring worker."
+ )
+ elif not serial_device_safe and uncached_indices:
+ raise RuntimeError(
+ "AutoScheme serial scoring cannot run on a model whose weights span "
+ "multiple GPUs without accelerate dispatch (the full-model forward has "
+ "no cross-device activation transfer). Score via disk-stream workers: "
+ "point AR_DISK_STREAM_MODEL at a local checkpoint directory and keep "
+ "AR_ENABLE_AUTO_SCHEME_PARALLEL enabled, or load the model on a single "
+ "GPU / with device_map='auto'."
+ )
+ if __debug__:
+ from collections import Counter
+
+ _param_devs = Counter(str(p_.device) for p_ in model.parameters())
+ _hf_map = getattr(model, "hf_device_map", None)
+ _hf_devs = sorted({str(v_) for v_ in _hf_map.values()}) if _hf_map else None
+ logger.info(
+ "AutoScheme serial-safety census: serial_device_safe=%s "
+ "param_devices=%s hf_device_map_devices=%s disk_stream_index=%s meta_params=%d "
+ "cuda_visible=%s",
+ serial_device_safe,
+ dict(_param_devs),
+ _hf_devs,
+ getattr(model, "_disk_stream_index", None) is not None,
+ sum(1 for p_ in model.parameters() if p_.device.type == "meta"),
+ [str(d_) for d_ in worker_device_pool],
+ )
# Vision scoring requires a full-model backward and therefore cannot use
# the block-wise materialize/free path used by disk streaming.
can_parallel = _can_parallel_scheme_scoring(
@@ -2480,6 +3009,8 @@ def _gen_layer_config(
worker_disk_stream_model,
is_vlm,
low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage,
+ force_mllm=force_mllm,
+ min_uncached=min_uncached,
)
logger.info(
"AutoScheme scoring mode: parallel_configured=%s, parallel_enabled=%s, disk_stream_enabled=%s",
@@ -2519,6 +3050,41 @@ def _gen_layer_config(
"AutoScheme: if parallel scoring runs out of RAM/VRAM or automatic serial fallback cannot "
"recover, set AR_ENABLE_AUTO_SCHEME_PARALLEL=0 and rerun."
)
+ # free the parent's GPU before spawning workers: non-block params
+ # (embed/lm_head/vision) parked on a device by earlier setup are
+ # dead weight during parallel scoring -- the parent never runs a
+ # forward, and each worker owns its private copy. A shared-GPU
+ # worker then gets the card's full budget.
+ _moved = 0
+ _parked = []
+ for _p in model.parameters():
+ if _p.device.type == "cuda":
+ _parked.append((_p, _p.device))
+ _p.data = _p.data.to("cpu")
+ _moved += 1
+ for _b in model.buffers():
+ if _b.device.type == "cuda":
+ _parked.append((_b, _b.device))
+ _b.data = _b.data.to("cpu")
+
+ def _restore_parked_tensors():
+ """Move tensors parked before spawning back to their devices.
+
+ The serial fallback and the post-parallel phases run
+ full-model forwards; leaving non-block params on CPU while
+ ``model.device`` still reports the GPU crashes the first
+ non-block module on a device mismatch (e.g. final norm)."""
+ for _tensor, _device in _parked:
+ if _tensor.device.type == "cpu" and _device.type != "cpu":
+ _tensor.data = _tensor.data.to(_device)
+ _parked.clear()
+
+ if _moved:
+ logger.info("AutoScheme: parked %d parent-side GPU tensors on CPU before spawning workers", _moved)
+ clear_memory()
+
+ _vram_inventory("parent pre-spawn")
+
spawn_context = multiprocessing.get_context("spawn")
with spawn_context.Manager() as manager:
progress_queue = manager.Queue()
@@ -2545,17 +3111,44 @@ def _gen_layer_config(
len(schemes),
progress_queue,
worker_disk_stream_model,
+ scheme_cache_meta[index][1], # cache_path for batch checkpoints
)
for slot, index in enumerate(uncached_indices)
]
with spawn_context.Pool(processes=num_workers) as pool:
- async_results = pool.map_async(_score_scheme_worker, worker_args)
- while not async_results.ready():
- _drain_progress_queue(progress_queue, pbar)
- memory_monitor.update_cpu()
- async_results.wait(timeout=0.1)
+ # incremental consumption: persist each scheme's scores to
+ # the per-scheme cache as its worker returns, so a partial
+ # failure (or serial-fallback crash) never discards
+ # completed schemes -- the rerun picks them up from cache
+ result_iter = pool.imap_unordered(_score_scheme_worker, worker_args)
+ worker_results = []
+ failed_workers = 0
+ while True:
+ try:
+ worker_results.append(result_iter.next(timeout=0.5))
+ except multiprocessing.TimeoutError:
+ _drain_progress_queue(progress_queue, pbar)
+ memory_monitor.update_cpu()
+ continue
+ except StopIteration:
+ break
+ except Exception as worker_error: # noqa: BLE001
+ # one worker's failure must not abort its siblings:
+ # keep consuming their results (each already-saved
+ # scheme survives in the per-scheme cache)
+ failed_workers += 1
+ logger.warning(
+ "AutoScheme: a scoring worker failed (%s); collecting remaining workers' results.",
+ worker_error,
+ )
+ _drain_progress_queue(progress_queue, pbar)
+ continue
+ index, scores, _report = worker_results[-1]
+ cache_key, cache_path, _ = scheme_cache_meta[index]
+ _save_per_op_scores(index, schemes[index], cache_key, cache_path, scores)
_drain_progress_queue(progress_queue, pbar)
- worker_results = async_results.get()
+ if not worker_results:
+ raise RuntimeError("all parallel scoring workers failed") from None
parallel_results = {index: scores for index, scores, _ in worker_results}
_merge_worker_memory_reports(memory_monitor, [report for _, _, report in worker_results])
@@ -2599,8 +3192,17 @@ def _gen_layer_config(
if not check_bf16_scheme(scheme):
pbar.update(pbar_cnt // effective_scheme_num if effective_scheme_num > 0 else 1)
else:
- per_op_scores = parallel_results[index]
- _save_per_op_scores(index, scheme, cache_key, cache_path, per_op_scores)
+ if index in parallel_results:
+ per_op_scores = parallel_results[index]
+ _save_per_op_scores(index, scheme, cache_key, cache_path, per_op_scores)
+ else:
+ # worker died for this scheme; siblings' results are saved.
+ # Rerunning resumes/scores only the missing schemes.
+ raise RuntimeError(
+ f"AutoScheme: scheme {index} ({_scheme_short_name(scheme)}) was lost with its "
+ "worker; completed schemes are persisted in the per-scheme cache -- rerun to "
+ "score only the failed schemes."
+ )
total_loss = _record_scheme_scores(index, per_op_scores)
logger.info(
"AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.6f)",
@@ -2610,8 +3212,19 @@ def _gen_layer_config(
)
parallel_done = True
logger.info("AutoScheme: parallel scoring completed.")
+ _restore_parked_tensors()
post_scoring_started = time.perf_counter()
except Exception as parallel_error: # noqa: BLE001
+ _restore_parked_tensors()
+ if _parallel_scoring_must_raise(parallel_error):
+ logger.error(
+ "AutoScheme: keeping the parallel scoring failure as a hard error "
+ "(AR_AUTO_SCHEME_NO_SERIAL_FALLBACK is set, or a scheme was lost with its worker); "
+ "completed schemes and batches are persisted in the per-scheme cache -- "
+ "rerun to score only the failed parts, e.g. with a smaller "
+ "AR_AUTO_SCHEME_BATCH_SIZE / AR_AUTO_SCHEME_NSAMPLES."
+ )
+ raise
logger.warning(
"AutoScheme: parallel scoring failed, falling back to serial: %s. "
"If fallback cannot recover (for example after RAM/VRAM exhaustion), rerun with "
diff --git a/auto_round/envs.py b/auto_round/envs.py
index 2401c8ce2..5eaf8351c 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
AR_AUTO_SCHEME_NSAMPLES: Optional[int] = None
AR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = None
AR_AUTO_SCHEME_CACHE: Optional[str] = None
+ AR_AUTO_SCHEME_NO_SERIAL_FALLBACK: bool = False
AR_ENABLE_AUTO_SCHEME_PARALLEL: bool = True
AR_NVFP4_E5M3_CACHE_HP_WEIGHT: bool = False
AR_DISK_STREAM_MODEL: bool = False
@@ -104,6 +105,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Stores persistent AutoScheme scoring results independently from AR_WORK_SPACE,
# whose contents are temporary working data and may be cleaned after a run.
"AR_AUTO_SCHEME_CACHE": lambda: os.getenv("AR_AUTO_SCHEME_CACHE", None),
+ "AR_AUTO_SCHEME_NO_SERIAL_FALLBACK": lambda: os.getenv("AR_AUTO_SCHEME_NO_SERIAL_FALLBACK", "0").lower()
+ in ("1", "true", "yes"),
# Enables AutoScheme to score schemes in parallel. Enabled by default;
# set it to 0 when workers could exhaust host RAM or device memory.
"AR_ENABLE_AUTO_SCHEME_PARALLEL": lambda: os.getenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "1").lower()
diff --git a/auto_round/utils/disk_stream_util.py b/auto_round/utils/disk_stream_util.py
index 5115b300f..3da7864e6 100644
--- a/auto_round/utils/disk_stream_util.py
+++ b/auto_round/utils/disk_stream_util.py
@@ -12,6 +12,7 @@ from __future__ import annotations
import json
import logging
+import re
from pathlib import Path
from typing import Dict
@@ -76,6 +77,65 @@ class SafetensorsIndex:
return [n for n in self.weight_map if n == prefix or n.startswith(dotted)]
+# Model-side module names sometimes differ from the checkpoint-side tensor
+# names. Transformers already maintains the authoritative per-family renames
+# (``transformers/conversion_mapping.py``, checkpoint -> model direction);
+# the disk-stream materializers know only model-side names, so those entries
+# are inverted here. A rename is applied only when the model-side name is
+# absent from the index AND the renamed candidate exists in it, so checkpoints
+# whose names already match the model are untouched.
+from functools import lru_cache
+
+
+@lru_cache(maxsize=None)
+def _reverse_renamings_for(model_type):
+ """Invert the checkpoint-conversion WeightRenaming entries for one family."""
+ if not model_type:
+ return ()
+ try:
+ from transformers.conversion_mapping import WeightRenaming, get_checkpoint_conversion_mapping
+ except ImportError: # pragma: no cover - transformers is a hard dep, guarded for safety
+ return ()
+ mapping = get_checkpoint_conversion_mapping(model_type)
+ if not mapping:
+ return ()
+ renamings = []
+ for entry in mapping:
+ if not isinstance(entry, WeightRenaming):
+ # Other converter types (fusions, prefix moves) are not invertible
+ # by simple regex substitution and are skipped.
+ continue
+ for target in entry.target_patterns:
+ for source in entry.source_patterns:
+ renamings.append((target, source))
+ return tuple(renamings)
+
+
+@lru_cache(maxsize=None)
+def _index_model_type(index):
+ config_path = Path(index.checkpoint_dir) / "config.json"
+ try:
+ with open(config_path) as f:
+ return json.load(f).get("model_type")
+ except OSError:
+ return None
+
+
+def _resolve_checkpoint_name(index, full_name: str):
+ """Map a model-side parameter name to its checkpoint-side tensor name."""
+ if index.has_tensor(full_name):
+ return full_name
+ for target_pattern, source_replacement in _reverse_renamings_for(_index_model_type(index)):
+ # ``source_replacement`` arrives regex-escaped (``mlp\.router\.gate``);
+ # as a ``re.sub`` replacement those backslashes would land in the string
+ # verbatim, so unescape it to the plain checkpoint-side spelling.
+ replacement = re.sub(r"\\(.)", r"\1", source_replacement)
+ candidate = re.sub(target_pattern, lambda _m: replacement, full_name)
+ if candidate != full_name and index.has_tensor(candidate):
+ return candidate
+ return None
+
+
def materialize_module(module: nn.Module, module_name: str, index: SafetensorsIndex, device: str) -> None:
"""Populate `module`'s (currently meta) parameters/buffers with real data read
directly from the checkpoint, onto `device`. `module_name` is `module`'s dotted
@@ -125,14 +185,15 @@ def materialize_module(module: nn.Module, module_name: str, index: SafetensorsIn
if str(tensor.device) != "meta":
continue # already materialized (e.g. shared/tied weights)
full_name = f"{module_name}.{name}".replace(".orig_layer.", ".")
- if not index.has_tensor(full_name):
+ resolved_name = _resolve_checkpoint_name(index, full_name)
+ if resolved_name is None:
sliced = _fused_lookup(full_name)
if sliced is not None:
fused_targets.append((name, sliced))
continue
logger.warning("No checkpoint tensor found for %s, leaving on meta", full_name)
continue
- targets.append((name, full_name, tensor.dtype))
+ targets.append((name, resolved_name, tensor.dtype))
for name, value in fused_targets:
set_module_tensor_to_device(module, name, device, value=value, dtype=value.dtype)
@@ -208,6 +269,14 @@ def build_meta_model(model_name: str, trust_remote_code: bool = True):
model = model_cls(config)
else:
model = AutoModelForCausalLM.from_config(config, trust_remote_code=trust_remote_code)
+ # Constructing from config does not tie weights (that happens in
+ # from_pretrained), so tied params such as lm_head.weight -- absent from
+ # the checkpoint by design -- stay separate meta tensors and later crash
+ # with "Cannot copy out of meta tensor" on the first device move. Tie
+ # here: the shared tensor materializes once from the checkpoint side it
+ # is stored under and both names become real together.
+ if getattr(model.config, "tie_word_embeddings", False):
+ model.tie_weights()
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=trust_remote_code)
index = SafetensorsIndex(model_name)
return model, tokenizer, index
@@ -232,10 +301,11 @@ def materialize_non_block_params(
if str(tensor.device) != "meta" or _in_block(name):
continue
full_name = name.replace(".orig_layer.", ".")
- if not index.has_tensor(full_name):
+ resolved_name = _resolve_checkpoint_name(index, full_name)
+ if resolved_name is None:
logger.warning("No checkpoint tensor found for %s, leaving on meta", full_name)
continue
- targets.append((name, full_name))
+ targets.append((name, resolved_name))
if not targets:
return
@@ -245,6 +315,12 @@ def materialize_non_block_params(
# is required so the checkpoint's real dtype wins over whatever the
# meta skeleton happened to declare.
set_module_tensor_to_device(model, name, device, value=values[full_name], dtype=values[full_name].dtype)
+ # set_module_tensor_to_device replaces parameter objects, which breaks
+ # weight tying: a tied lm_head still references the old (meta) parameter
+ # while the embedding points at the new real one. Re-tie so every tied
+ # name follows its checkpoint-backed source tensor.
+ if getattr(getattr(model, "config", None), "tie_word_embeddings", False):
+ model.tie_weights()
class stream_block_forward:
diff --git a/auto_round/utils/offload.py b/auto_round/utils/offload.py
index bc3d72f35..4f5ee3481 100644
--- a/auto_round/utils/offload.py
+++ b/auto_round/utils/offload.py
@@ -243,6 +243,26 @@ def load_block_from_model_files(model_dir: str, block_name: str, block: torch.nn
block: The ``nn.Module`` to load weights into.
"""
model_dir = _resolve_model_dir(model_dir)
+
+ def _has_meta(module: torch.nn.Module) -> bool:
+ return any(tensor.device.type == "meta" for tensor in list(module.parameters()) + list(module.buffers()))
+
+ if _has_meta(block):
+ # A meta-skeleton block (disk-streamed model): resolve each parameter's
+ # checkpoint-side name (checkpoint-conversion renames, fused expert
+ # layouts) instead of raw prefix matching, which silently skips tensors
+ # whose checkpoint spelling differs (e.g. the router stored as
+ # ``mlp.router.gate`` behind a ``mlp.gate`` module) and leaves them on
+ # meta until a downstream ``.to(device)`` crashes.
+ try:
+ from auto_round.utils.disk_stream_util import SafetensorsIndex, materialize_module
+
+ materialize_module(block, block_name, SafetensorsIndex(model_dir), device="cpu")
+ if not _has_meta(block):
+ return
+ except FileNotFoundError:
+ pass # non-safetensors checkpoint: fall through to prefix loading
+
weight_map = _build_weight_map(model_dir)
prefix = block_name + "."
diff --git a/docs/environments.md b/docs/environments.md
index 6d16b3130..84a6fcdef 100644
--- a/docs/environments.md
+++ b/docs/environments.md
@@ -161,6 +161,16 @@ export AR_AUTO_SCHEME_BATCH_SIZE=1
export AR_AUTO_SCHEME_SEQLEN=1024
```
+### AR_AUTO_SCHEME_NO_SERIAL_FALLBACK
+- **Description**: Turn a parallel-scoring failure into a hard error instead of falling back to serial scoring. Useful when the serial pass is known to be unable to run (or would take workers-count times longer): completed schemes and batches are persisted in the per-scheme cache, so a rerun scores only the failed parts.
+- **Default**: unset -> parallel scoring failure falls back to serial
+- **Valid Values**: `1`, `true`, `yes`
+- **Usage**: Set this to fail fast on parallel scoring errors
+
+```bash
+export AR_AUTO_SCHEME_NO_SERIAL_FALLBACK=1
+```
+
### AR_AUTO_SCHEME_CACHE
- **Description**: Stores persistent per-scheme AutoScheme scoring JSON files. This directory is independent of `AR_WORK_SPACE`, which is reserved for temporary working data.
- **Default**: `~/.cache/auto_round`
@@ -181,6 +191,16 @@ export AR_AUTO_SCHEME_CACHE=/path/to/auto_scheme_cache
export AR_ENABLE_AUTO_SCHEME_PARALLEL=0
```
+### AR_SCHEME_MEM_INVENTORY
+- **Description**: When enabled, AutoScheme streaming scoring prints a live CUDA-tensor census (`[mem-inv]` lines) at block boundaries -- tensors grouped by (shape, dtype), largest first -- which makes unreleased module weights or retained autograd graphs visible as they accumulate. Independently of this variable, a richer census (live tensors, the retaining containers, and the failing op's traceback) is always attached to scoring-worker CUDA OOM errors.
+- **Default**: `False`
+- **Valid Values**: `"1"`, `"true"`, `"yes"` (case-insensitive) for enabling; any other value for disabling
+- **Usage**: Enable when investigating VRAM growth during AutoScheme scoring
+
+```bash
+export AR_SCHEME_MEM_INVENTORY=1
+```
+
### AR_NVFP4_E5M3_CACHE_HP_WEIGHT
- **Description**: Controls whether `NVFP4E5M3QuantLinear` caches a dequantized high-precision weight after the first forward pass, instead of dequantizing the packed FP4 weight on every call.
- **Default**: `False` (equivalent to `"0"`)
diff --git a/docs/environments_CN.md b/docs/environments_CN.md
index 8648b4d76..a9c1f4aea 100644
--- a/docs/environments_CN.md
+++ b/docs/environments_CN.md
@@ -161,6 +161,16 @@ export AR_AUTO_SCHEME_BATCH_SIZE=1
export AR_AUTO_SCHEME_SEQLEN=1024
```
+### AR_AUTO_SCHEME_NO_SERIAL_FALLBACK
+- **描述**:将并行评分失败变为硬错误,而不是回退到串行评分。当串行路径已知无法运行(或耗时会是 worker 数量的倍数)时非常有用:已完成的方案和批次会持久化在单方案缓存中,重新运行时只补打失败的部分。
+- **默认值**:未设置时,并行评分失败会回退到串行评分
+- **有效值**:`1`、`true`、`yes`(不区分大小写)
+- **用法**:设置该变量以在并行评分出错时快速失败
+
+```bash
+export AR_AUTO_SCHEME_NO_SERIAL_FALLBACK=1
+```
+
### AR_AUTO_SCHEME_CACHE
- **描述**:存放可持久复用的 AutoScheme 单方案评分 JSON 文件。该目录独立于用于临时工作数据的 `AR_WORK_SPACE`。
- **默认值**:`~/.cache/auto_round`
@@ -181,6 +191,16 @@ export AR_AUTO_SCHEME_CACHE=/path/to/auto_scheme_cache
export AR_ENABLE_AUTO_SCHEME_PARALLEL=0
```
+### AR_SCHEME_MEM_INVENTORY
+- **描述**:启用后,AutoScheme 流式评分会在 block 边界打印当前 CUDA tensor 清单(`[mem-inv]` 行)——按 (shape, dtype) 分组、按大小降序——便于发现未释放的模块权重或被保留的计算图。与该变量无关,打分 worker 的 CUDA OOM 错误始终会附带更详细的清单(活动 tensor、持有它们的容器以及出错算子的 traceback)。
+- **默认值**:`False`
+- **有效值**:`"1"`、`"true"`、`"yes"`(不区分大小写)表示启用;其他值表示禁用
+- **用法**:排查 AutoScheme 打分期间显存增长问题时启用
+
+```bash
+export AR_SCHEME_MEM_INVENTORY=1
+```
+
### AR_NVFP4_E5M3_CACHE_HP_WEIGHT
- **描述**:控制 `NVFP4E5M3QuantLinear` 是否在首次前向后缓存解量化得到的高精度权重,而不是每次调用都从打包的 FP4 权重重新解量化。
- **默认值**:`False`(等价于 `"0"`)
diff --git a/test/unit/test_cpu/schemes/test_auto_scheme.py b/test/unit/test_cpu/schemes/test_auto_scheme.py
index 43b90815d..652affa4a 100644
--- a/test/unit/test_cpu/schemes/test_auto_scheme.py
+++ b/test/unit/test_cpu/schemes/test_auto_scheme.py
@@ -3,8 +3,18 @@ import os
import shutil
import pytest
+import torch
from auto_round import AutoRound, AutoScheme
+from auto_round.auto_scheme.delta_loss import (
+ AutoSchemeWrapperLinear,
+ _annotate_worker_oom,
+ _clear_wrapper_score_caches,
+ _parallel_scoring_must_raise,
+ _replay_retain_graph,
+ _tensor_referrer_snippet,
+ _vram_inventory_text,
+)
from auto_round.auto_scheme.utils import _build_layer_config_header_rows, _short_summary_name
@@ -457,6 +467,21 @@ class TestAutoScheme:
f"cache may have stored merged group scores instead of individual scores"
)
+ def test_mxfp_options_produce_mixed_layer_config(self, tiny_opt_model_path):
+ """MX schemes score weight error via the weight-score path, so their
+ layer scores must be nonzero and the allocator must mix MXFP4/MXFP8
+ (a zeroed score set collapses the config onto the cheapest option)."""
+ scheme = AutoScheme(
+ avg_bits=7,
+ options=("MXFP4", "MXFP8"),
+ shared_layers=["q_proj", "k_proj", "v_proj"],
+ nsamples=1,
+ )
+ ar = AutoRound(model=tiny_opt_model_path, scheme=scheme, iters=0, nsamples=1)
+ _, config = ar.quantize()
+ bits_used = {v["bits"] for v in config.values() if "bits" in v}
+ assert bits_used == {4, 8}, f"expected a mixed MXFP4/MXFP8 config, got {sorted(bits_used)}"
+
def test_different_avg_bits_produces_different_layer_config(self, tiny_opt_model_path):
"""Changing avg_bits should change the resulting layer_config."""
scheme_low = AutoScheme(
@@ -787,12 +812,18 @@ def test_autoscheme_cache_path_is_independent_from_workspace(tmp_path, monkeypat
from auto_round.auto_scheme.delta_loss import _autoscheme_cache_path
monkeypatch.setenv("HOME", str(tmp_path / "home"))
+ # Windows expanduser prefers USERPROFILE over HOME; patch both so the
+ # default-cache assertion holds on every platform (no-op on Linux).
+ monkeypatch.setenv("USERPROFILE", str(tmp_path / "home"))
monkeypatch.setenv("AR_WORK_SPACE", str(tmp_path / "workspace"))
monkeypatch.delenv("AR_AUTO_SCHEME_CACHE", raising=False)
cache_path = _autoscheme_cache_path("cache_key", 1)
- assert cache_path == str(tmp_path / "home" / ".cache" / "auto_round" / "scheme_01_cache_key.json")
+ # normpath: expanduser keeps the forward slashes of "~/.cache" on Windows
+ assert os.path.normpath(cache_path) == os.path.normpath(
+ str(tmp_path / "home" / ".cache" / "auto_round" / "scheme_01_cache_key.json")
+ )
def test_autoscheme_cache_path_can_be_overridden(tmp_path, monkeypatch):
@@ -884,17 +915,26 @@ def test_parallel_scheme_scoring_supports_disk_stream_model():
assert _can_parallel_scheme_scoring(True, "local-model", 1, 2, False, True, False)
-def test_parallel_scheme_scoring_rejects_disk_stream_vlm():
+def test_parallel_scheme_scoring_allows_text_only_disk_stream_vlm():
+ """c34538d2: text-only scoring of a VLM language tower streams blocks like
+ a text model; only vision scoring (force_mllm) is excluded from disk
+ streaming."""
from auto_round.auto_scheme.delta_loss import _can_parallel_scheme_scoring
- assert not _can_parallel_scheme_scoring(True, "local-model", 1, 2, False, True, True)
+ # is_vlm alone no longer disqualifies disk-stream parallel scoring
+ assert _can_parallel_scheme_scoring(True, "local-model", 1, 2, False, True, True)
+ # force_mllm (vision scoring needs a full-model backward) still does
+ assert not _can_parallel_scheme_scoring(True, "local-model", 1, 2, False, True, False, force_mllm=True)
@pytest.mark.parametrize(
"model_id,is_vlm,low_gpu_mem_usage,expected",
[
+ # c34538d2: is_vlm no longer disqualifies disk-stream workers -- the
+ # language tower streams blocks like a text model; multimodal archs
+ # are covered via materialize_non_block_params.
("local-model", False, True, True),
- ("local-model", True, True, False),
+ ("local-model", True, True, True),
("local-model", False, False, False),
(None, False, True, False),
],
@@ -935,7 +975,15 @@ def test_disk_stream_scheme_worker_builds_meta_model(monkeypatch):
from auto_round.auto_scheme import delta_loss
from auto_round.utils import disk_stream_util
- expected = (object(), object(), object())
+ class FakeMetaModel:
+ """Minimal module stand-in: the loader structurally unfuses MoE modules
+ (named_modules walk) before weights are streamed."""
+
+ def named_modules(self, memo=None):
+ return iter(())
+
+ model, tokenizer, disk_index = FakeMetaModel(), object(), object()
+ expected = (model, tokenizer, disk_index)
monkeypatch.setattr(disk_stream_util, "build_meta_model", lambda model_name: expected)
assert delta_loss._load_disk_stream_scheme_worker_model("local-model") == expected
@@ -946,7 +994,11 @@ def test_disk_stream_scheme_worker_applies_model_replacements(monkeypatch):
from auto_round.auto_scheme import delta_loss
from auto_round.utils import disk_stream_util
- model = object()
+ class FakeMetaModel:
+ def named_modules(self, memo=None):
+ return iter(())
+
+ model = FakeMetaModel()
updated_model = object()
handled_model = object()
tokenizer = object()
@@ -1053,3 +1105,386 @@ def test_replacement_wrapper_without_tuning_device_uses_major_device():
move_module_to_tuning_device(wrapper, major_device="cpu")
assert wrapper.orig_layer.weight.device.type == "cpu"
+
+
+def test_env_ar_auto_scheme_no_serial_fallback(monkeypatch):
+ import auto_round.envs as envs
+
+ monkeypatch.delenv("AR_AUTO_SCHEME_NO_SERIAL_FALLBACK", raising=False)
+ assert envs.AR_AUTO_SCHEME_NO_SERIAL_FALLBACK is False
+ monkeypatch.setenv("AR_AUTO_SCHEME_NO_SERIAL_FALLBACK", "1")
+ assert envs.AR_AUTO_SCHEME_NO_SERIAL_FALLBACK is True
+
+
+def test_parallel_error_must_raise_respects_env_and_lost_worker(monkeypatch):
+ import auto_round.envs as envs
+
+ monkeypatch.delenv("AR_AUTO_SCHEME_NO_SERIAL_FALLBACK", raising=False)
+ # plain failure, env off -> serial fallback proceeds
+ assert not _parallel_scoring_must_raise(RuntimeError("CUDA out of memory"))
+ # scheme lost with its worker always raises
+ assert _parallel_scoring_must_raise(RuntimeError("scheme 2 (W4A16) was lost with its worker"))
+ # env on -> any failure raises
+ monkeypatch.setenv("AR_AUTO_SCHEME_NO_SERIAL_FALLBACK", "1")
+ assert _parallel_scoring_must_raise(RuntimeError("CUDA out of memory"))
+
+
+class TestScoreAnchorWeightScoring:
+ """The scoring wrapper quantizes each layer once and routes the score hook
+ through a scalar anchor, so block backwards never materialize weight-shaped
+ gradients or re-run the quant search."""
+
+ def _make_wrapper(self):
+ """Minimal scoring wrapper: a Linear with the attributes _qdq_weight reads."""
+ layer = torch.nn.Linear(8, 8, bias=False)
+ layer.bits, layer.group_size, layer.sym = 4, 8, True
+ layer.scale_dtype = None
+ layer.data_type = "int"
+ wrapper = AutoSchemeWrapperLinear.__new__(AutoSchemeWrapperLinear)
+ torch.nn.Module.__init__(wrapper)
+ wrapper.orig_layer = layer
+ wrapper.device = "cpu"
+ wrapper.data_type = "int"
+ wrapper.q_scale_thresh = None
+ wrapper.grad_mode = True
+ wrapper.need_weight_grad = True
+ wrapper.enable_act_quant = False
+ wrapper.minmax_scale_bound = (1e-4, 1e3)
+ wrapper.weight_min = None
+ wrapper.weight_max = None
+ from auto_round.wrapper import WrapperLinear
+
+ wrapper.super_qdq_func = WrapperLinear._qdq_weight.__get__(wrapper)
+ wrapper.act_score = 0.0
+ wrapper.avg_act_score = 0.0
+ wrapper.act_cnt = 0.0
+ wrapper.weight_score = 0.0
+ wrapper.mix_score = 0.0
+ wrapper.max_act_value = 0
+ wrapper._score_qdq_cpu = None
+
+ def _fake_quant(weight, **kwargs):
+ scale = weight.abs().amax().clamp(min=1e-4) / 7.0
+ return ((weight / scale).round().clamp(-7, 7) * scale), scale, None
+
+ wrapper.weight_quant_func = _fake_quant
+ layer.weight.requires_grad = True
+ return wrapper, layer
+
+ def test_qdq_weight_quantizes_once_and_reuses_cache(self):
+ wrapper, layer = self._make_wrapper()
+ calls = {"n": 0}
+ real_quant = wrapper.weight_quant_func
+
+ def counting_quant(weight, **kwargs):
+ calls["n"] += 1
+ return real_quant(weight, **kwargs)
+
+ wrapper.weight_quant_func = counting_quant
+ args = (torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))
+ first, _, _ = wrapper._qdq_weight(*args)
+ second, _, _ = wrapper._qdq_weight(*args)
+ assert calls["n"] == 1
+ assert torch.equal(first, second)
+
+ def test_score_anchor_accumulates_expected_weight_score(self):
+ torch.manual_seed(0)
+ wrapper, layer = self._make_wrapper()
+ with torch.no_grad():
+ layer.weight.copy_(torch.randn(8, 8) * 0.5)
+ x = torch.randn(4, 8)
+ qdq_w, _, _ = wrapper._qdq_weight(torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))
+ assert qdq_w.requires_grad
+ torch.nn.functional.linear(x, qdq_w).sum().backward()
+ # dL/d(qdq_w)[j, k] = sum_i x[i, k], so the expected score is
+ # sum_{j,k} |x.sum(0)[k]| * |w_diff[j, k]|
+ w_diff = layer.weight - wrapper._score_qdq_cpu
+ expected = (w_diff.abs().sum(dim=0) * x.sum(dim=0).abs()).sum().item()
+ assert wrapper.weight_score > 0
+ assert abs(wrapper.weight_score - expected) < 1e-5
+
+ def test_orig_weight_gets_no_grad(self):
+ wrapper, layer = self._make_wrapper()
+ qdq_w, _, _ = wrapper._qdq_weight(torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))
+ torch.nn.functional.linear(torch.randn(2, 8), qdq_w).sum().backward()
+ assert layer.weight.grad is None
+
+ def test_capture_mode_returns_fresh_tensor_without_cache(self):
+ wrapper, layer = self._make_wrapper()
+ wrapper.grad_mode = False
+ out, _, _ = wrapper._qdq_weight(torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))
+ assert not out.requires_grad
+ assert wrapper._score_qdq_cpu is None # one-shot phases must not grow CPU caches
+
+ def test_clear_wrapper_score_caches_drops_block_cache(self):
+ import torch.nn as nn
+
+ wrapper, layer = self._make_wrapper()
+ block = nn.Sequential(wrapper)
+ wrapper._qdq_weight(torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))
+ assert wrapper._score_qdq_cpu is not None
+ _clear_wrapper_score_caches(block)
+ assert wrapper._score_qdq_cpu is None
+
+
+class TestOomCensus:
+ def test_vram_inventory_text_cpu_only_returns_graceful_string(self):
+ text = _vram_inventory_text(top_k=4)
+ assert isinstance(text, str)
+ assert len(text) > 0
+
+ def test_annotate_worker_oom_embeds_census_and_original(self, monkeypatch):
+ monkeypatch.setattr("auto_round.auto_scheme.delta_loss._vram_inventory_text", lambda top_k=12: "CENSUS-LINE")
+ out = _annotate_worker_oom(2, RuntimeError("CUDA out of memory. Tried to allocate 12.00 MiB"))
+ assert "CENSUS-LINE" in str(out)
+ assert "Tried to allocate 12.00 MiB" in str(out)
+
+ def test_non_oom_runtime_errors_are_not_annotated(self, monkeypatch):
+ called = {"n": 0}
+
+ def boom(top_k=12):
+ called["n"] += 1
+ return "CENSUS-LINE"
+
+ monkeypatch.setattr("auto_round.auto_scheme.delta_loss._vram_inventory_text", boom)
+ exc = RuntimeError("something else entirely")
+ try:
+ if "out of memory" not in str(exc).lower():
+ raise _annotate_worker_oom(0, exc) if False else exc
+ except RuntimeError as caught:
+ assert caught is exc
+ assert called["n"] == 0
+
+
+class TestVramReferrerTrace:
+ def test_referrer_snippet_names_container_types(self):
+ import torch
+
+ holder = {"box": [torch.zeros(2, 3)]}
+
+ def _find():
+ import gc
+
+ for obj in gc.get_objects():
+ if torch.is_tensor(obj) and tuple(obj.shape) == (2, 3):
+ return obj
+ return None
+
+ t = _find()
+ assert t is not None
+ snippet = _tensor_referrer_snippet(t, max_depth=3)
+ assert isinstance(snippet, str)
+ assert "list" in snippet or "dict" in snippet
+
+
+class TestReferrerAttributeNaming:
+ def test_module_referrer_names_the_holding_attribute(self):
+ import torch.nn as nn
+
+ class Holder(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.cache_box = []
+
+ t = torch.zeros(2, 2)
+ holder = Holder()
+ holder.cache_box.append(t)
+ snippet = _tensor_referrer_snippet(t, max_depth=3)
+ assert "Holder" in snippet
+ assert "cache_box" in snippet
+
+
+class TestReplayRetainGraph:
+ def test_int_data_type_frees_graph(self):
+ import torch.nn as nn
+
+ block = nn.Sequential(nn.Linear(4, 4))
+ block[0].data_type = "int"
+ assert _replay_retain_graph(block) is False
+
+ def test_mx_family_data_type_retains_graph(self):
+ import torch.nn as nn
+
+ block = nn.Sequential(nn.Linear(4, 4))
+ block[0].data_type = "mxfp4"
+ assert _replay_retain_graph(block) is True
+
+ def test_grads_flow_with_freed_graph(self):
+ lin = torch.nn.Linear(4, 4)
+ x = torch.randn(2, 4, requires_grad=True)
+ out = lin(x).sum()
+ torch.autograd.backward(out, retain_graph=False)
+ assert x.grad is not None
+ assert x.grad.abs().sum() > 0
+
+
+class TestScoreLinearRecompute:
+ """The scoring forward runs the linear through a custom Function that saves
+ only the input activation: the weight is re-transferred from the CPU cache
+ in backward, so the block graph never retains weight-shaped copies."""
+
+ def _make_wrapper(self):
+ layer = torch.nn.Linear(8, 6, bias=False)
+ layer.bits, layer.group_size, layer.sym = 4, 8, True
+ layer.scale_dtype = None
+ layer.data_type = "int"
+ wrapper = AutoSchemeWrapperLinear.__new__(AutoSchemeWrapperLinear)
+ torch.nn.Module.__init__(wrapper)
+ wrapper.orig_layer = layer
+ wrapper.device = "cpu"
+ wrapper.output_device = "cpu"
+ wrapper.data_type = "int"
+ wrapper.q_scale_thresh = None
+ wrapper.grad_mode = True
+ wrapper.need_weight_grad = True
+ wrapper.enable_act_quant = False
+ wrapper.enable_norm_bias_tuning = False
+ wrapper.minmax_scale_bound = (1e-4, 1e3)
+ wrapper.weight_min = None
+ wrapper.weight_max = None
+ from auto_round.wrapper import WrapperLinear
+
+ wrapper.super_qdq_func = WrapperLinear._qdq_weight.__get__(wrapper)
+ wrapper.act_qdq_func = WrapperLinear._qdq_act.__get__(wrapper)
+ wrapper.act_score = 0.0
+ wrapper.avg_act_score = 0.0
+ wrapper.act_cnt = 0.0
+ wrapper.weight_score = 0.0
+ wrapper.mix_score = 0.0
+ wrapper.max_act_value = 0
+ wrapper._score_qdq_cpu = None
+ wrapper._custom_score_forward = True
+ wrapper.orig_forward = torch.nn.functional.linear
+ wrapper.value = torch.tensor(0.0)
+ wrapper.min_scale = torch.tensor(1.0)
+ wrapper.max_scale = torch.tensor(1.0)
+
+ def _fake_quant(weight, **kwargs):
+ scale = weight.abs().amax().clamp(min=1e-4) / 7.0
+ return (weight / scale).round().clamp(-7, 7) * scale, scale, None
+
+ wrapper.weight_quant_func = _fake_quant
+ layer.weight.requires_grad = True
+ return wrapper, layer
+
+ def test_forward_matches_reference_linear(self):
+ torch.manual_seed(0)
+ wrapper, layer = self._make_wrapper()
+ with torch.no_grad():
+ layer.weight.copy_(torch.randn(6, 8) * 0.5)
+ x = torch.randn(4, 8, requires_grad=True)
+ out = wrapper.forward(x)
+ ref_w, _, _ = wrapper._qdq_weight(torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))
+ ref = torch.nn.functional.linear(x, ref_w.detach())
+ assert torch.equal(out, ref)
+
+ def test_input_grads_match_reference(self):
+ torch.manual_seed(0)
+ wrapper, layer = self._make_wrapper()
+ x = torch.randn(4, 8, requires_grad=True)
+ out = wrapper.forward(x)
+ grad_out = torch.randn_like(out)
+ out.backward(grad_out)
+ ref_w = wrapper._score_qdq_cpu
+ ref_x = x.detach().clone().requires_grad_(True)
+ ref = torch.nn.functional.linear(ref_x, ref_w)
+ ref.backward(grad_out)
+ assert torch.allclose(x.grad, ref_x.grad, atol=1e-6)
+
+ def test_weight_score_accumulated_in_backward(self):
+ torch.manual_seed(0)
+ wrapper, layer = self._make_wrapper()
+ with torch.no_grad():
+ layer.weight.copy_(torch.randn(6, 8) * 0.5)
+ x = torch.randn(4, 8, requires_grad=True)
+ out = wrapper.forward(x)
+ grad_out = torch.randn_like(out)
+ out.backward(grad_out)
+ qdq_w = wrapper._score_qdq_cpu
+ grad_w = grad_out.t() @ x.detach()
+ expected = (grad_w.abs() * (layer.weight - qdq_w).abs()).sum().item()
+ assert wrapper.weight_score > 0
+ assert abs(wrapper.weight_score - expected) / expected < 1e-5
+
+ def test_weight_refetched_in_backward_and_no_weight_grad(self):
+ wrapper, layer = self._make_wrapper()
+ x = torch.randn(4, 8, requires_grad=True)
+ wrapper.forward(x).sum().backward()
+ assert layer.weight.grad is None
+ assert wrapper.weight_score > 0
+
+ def test_capture_mode_uses_base_forward(self):
+ torch.manual_seed(1)
+ wrapper, layer = self._make_wrapper()
+ wrapper.grad_mode = False
+ x = torch.randn(4, 8)
+ out = wrapper.forward(x)
+ assert not out.requires_grad
+ assert wrapper._score_qdq_cpu is None
+
+
+def test_parallel_scheme_scoring_min_uncached_overrides_single_scheme():
+ """A single uncached scheme can still be scored through a worker when the
+ serial full-model pass cannot run (weights spanning several GPUs)."""
+ from auto_round.auto_scheme.delta_loss import _can_parallel_scheme_scoring
+
+ # default keeps the >=2 heuristic: one scheme stays serial
+ assert not _can_parallel_scheme_scoring(True, "local-model", 1, 1, False, True, False)
+ # an explicit lower floor admits the single-scheme worker run
+ assert _can_parallel_scheme_scoring(True, "local-model", 1, 1, False, True, False, min_uncached=1)
+
+
+def test_serial_scoring_device_safe():
+ """Serial full-model scoring is device-safe only on single-GPU/CPU models;
+ any multi-GPU weight span is unsafe regardless of leftover hook markers
+ (partial dispatch / manual placement does not shuttle activations)."""
+ import torch
+ import torch.nn as nn
+
+ from auto_round.auto_scheme.delta_loss import _serial_scoring_device_safe, _weights_span_multiple_gpus
+
+ class Tiny(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.fc1 = nn.Linear(4, 4)
+
+ assert _serial_scoring_device_safe(Tiny()) is True
+
+ # a meta-skeleton (disk-stream) model has all params on meta - the live
+ # scan sees nothing; the placement plan in hf_device_map is what the
+ # serial streaming path materializes blocks onto
+ class MetaSkeleton(Tiny):
+ hf_device_map = {"": "cuda:0", "model.layers.1": "cuda:3"}
+
+ assert _serial_scoring_device_safe(MetaSkeleton()) is False
+
+ class SingleDeviceMap(Tiny):
+ hf_device_map = {"": "cuda:1"}
+
+ assert _serial_scoring_device_safe(SingleDeviceMap()) is True
+
+ # a streamed (meta-skeleton) model with several visible CUDA devices is
+ # serial-unsafe: the in-process streaming pass is only validated with a
+ # single visible GPU; workers pin one GPU each
+ class StreamedSkeleton(Tiny):
+ _disk_stream_index = object()
+
+ assert _serial_scoring_device_safe(StreamedSkeleton(), visible_cuda_devices=["cuda:0", "cuda:1"]) is False
+ assert _serial_scoring_device_safe(StreamedSkeleton(), visible_cuda_devices=["cuda:0"]) is True
+ assert _serial_scoring_device_safe(Tiny(), visible_cuda_devices=["cuda:0", "cuda:1"]) is True
+
+ assert _weights_span_multiple_gpus([torch.device("cuda:0"), torch.device("cuda:0")]) is False
+ assert _weights_span_multiple_gpus([torch.device("cuda:0"), torch.device("cuda:3")]) is True
+ assert _weights_span_multiple_gpus([torch.device("cpu"), torch.device("cpu")]) is False
+
+ # a multi-GPU span is unsafe even when an _hf_hook marker is present:
+ # the marker alone does not guarantee activation transfer under the
+ # scoring wrappers
+ class FakeHooked(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self._hf_hook = object()
+
+ assert (
+ _weights_span_multiple_gpus([torch.device("cuda:0"), torch.device("cuda:1")]) is True
+ ) # the unsafe case _serial_scoring_device_safe keys on
diff --git a/test/unit/test_cpu/schemes/test_auto_scheme_disk_stream.py b/test/unit/test_cpu/schemes/test_auto_scheme_disk_stream.py
index 4205bde95..27d67b6da 100644
--- a/test/unit/test_cpu/schemes/test_auto_scheme_disk_stream.py
+++ b/test/unit/test_cpu/schemes/test_auto_scheme_disk_stream.py
@@ -99,3 +99,30 @@ class TestDiskStreamUtilRoundTrip:
for _, tensor in list(block.named_parameters()):
assert str(tensor.device) == "meta"
assert total_resident_bytes(block) == 0
+
+
+class TestDiskStreamWorkerModelPrep:
+ def test_disk_stream_build_unfuses_moe_modules(self, monkeypatch):
+ """The disk-stream worker build must structurally unfuse fused MoE experts
+ (the regular load pipeline does this via custom replacements; the
+ meta-skeleton build skips them, leaving per-expert quant layers
+ unresolvable)."""
+ import auto_round.auto_scheme.delta_loss as dl
+
+ seen = {}
+ sentinel_model, sentinel_tokenizer, sentinel_index = object(), object(), object()
+
+ def _fake_build_meta_model(model_name):
+ return sentinel_model, sentinel_tokenizer, sentinel_index
+
+ def _fake_handle_moe(model):
+ seen["model"] = model
+ return []
+
+ monkeypatch.setattr("auto_round.utils.disk_stream_util.build_meta_model", _fake_build_meta_model)
+ monkeypatch.setattr("auto_round.modeling.fused_moe.replace_modules._handle_moe_modules", _fake_handle_moe)
+
+ model, _tokenizer, _index = dl._load_disk_stream_scheme_worker_model("dummy-model")
+
+ assert model is sentinel_model
+ assert seen["model"] is sentinel_model
diff --git a/test/unit/test_cpu/utils/test_disk_stream_util.py b/test/unit/test_cpu/utils/test_disk_stream_util.py
index bfc9949f8..bc20165b3 100644
--- a/test/unit/test_cpu/utils/test_disk_stream_util.py
+++ b/test/unit/test_cpu/utils/test_disk_stream_util.py
@@ -14,6 +14,7 @@
"""Unit tests for auto_round.utils.disk_stream_util (AR_DISK_STREAM_MODEL primitives)."""
+import pytest
import torch
import torch.nn as nn
from accelerate import init_empty_weights
@@ -120,3 +121,171 @@ class TestBuildMetaModel:
# Decoder blocks must still be untouched (meta).
for name, tensor in model.model.decoder.layers[0].named_parameters():
assert str(tensor.device) == "meta", f"{name} was unexpectedly materialized"
+
+ def test_tied_lm_head_shares_the_embedding_tensor(self, tiny_opt_model_path):
+ """A tied lm_head.weight has no checkpoint tensor of its own; the meta
+ skeleton must tie it to embed_tokens.weight so materializing the
+ embedding makes both real (otherwise it stays a separate meta tensor
+ and the first device move crashes)."""
+ model, _tokenizer, index = build_meta_model(tiny_opt_model_path)
+ if not getattr(model.config, "tie_word_embeddings", False):
+ pytest.skip("fixture model does not tie word embeddings")
+ assert model.lm_head.weight is model.model.decoder.embed_tokens.weight
+
+ block_names = ["model.decoder.layers.0", "model.decoder.layers.1"]
+ materialize_non_block_params(model, block_names, index, device="cpu")
+ assert str(model.lm_head.weight.device) != "meta", "tied lm_head stayed on meta"
+
+
+class TestCheckpointNameAliases:
+ """Model-side names that differ from checkpoint-side tensor names must
+ still materialize, resolving through transformers' checkpoint-conversion
+ renames (the router stored as ``mlp.router.gate`` behind a ``mlp.gate``
+ module, ``shared_mlp`` tensors behind ``shared_experts``)."""
+
+ def _make_index(self, tensors, tmp_path):
+ import json
+
+ from auto_round.utils.disk_stream_util import SafetensorsIndex
+
+ checkpoint_dir = tmp_path / "ckpt"
+ checkpoint_dir.mkdir(exist_ok=True)
+ (checkpoint_dir / "config.json").write_text(json.dumps({"model_type": "hy_v3"}))
+
+ class _AliasIndex(SafetensorsIndex):
+ def __init__(self):
+ self.checkpoint_dir = checkpoint_dir
+ self.weight_map = {name: "dummy" for name in tensors}
+ self._tensors = tensors
+
+ def has_tensor(self, name):
+ return name in self._tensors
+
+ def read_tensors(self, names, device="cpu"):
+ return {n: self._tensors[n].clone() for n in names}
+
+ return _AliasIndex()
+
+ def _make_block(self):
+ import torch.nn as nn
+
+ class _Shared(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate_proj = nn.Linear(4, 4, bias=False)
+ self.up_proj = nn.Linear(4, 4, bias=False)
+
+ class _Mlp(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate = nn.Linear(4, 4, bias=False)
+ self.shared_experts = _Shared()
+ self.up_proj = nn.Linear(4, 4, bias=False)
+
+ with init_empty_weights():
+ return _Mlp()
+
+ def test_aliased_names_materialize_from_checkpoint_names(self, tmp_path):
+ with torch.no_grad():
+ tensors = {
+ "model.layers.1.mlp.router.gate.weight": torch.randn(4, 4, dtype=torch.float16),
+ "model.layers.1.mlp.shared_mlp.gate_proj.weight": torch.randn(4, 4, dtype=torch.float16),
+ "model.layers.1.mlp.shared_mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float16),
+ }
+ block = self._make_block()
+ materialize_module(block, "model.layers.1.mlp", self._make_index(tensors, tmp_path), device="cpu")
+ assert block.gate.weight.device.type != "meta"
+ assert block.shared_experts.gate_proj.weight.device.type != "meta"
+ assert block.shared_experts.up_proj.weight.device.type != "meta"
+
+ def test_exact_names_still_resolve_without_alias(self, tmp_path):
+ with torch.no_grad():
+ tensors = {"model.layers.1.mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float16)}
+ block = self._make_block()
+ materialize_module(block, "model.layers.1.mlp", self._make_index(tensors, tmp_path), device="cpu")
+ assert block.up_proj.weight.device.type != "meta"
+
+ def test_non_block_params_use_aliases_too(self, tmp_path):
+ import torch.nn as nn
+
+ class _Top(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.mlp = nn.Module()
+ self.mlp.gate = nn.Linear(4, 4, bias=False)
+
+ with torch.no_grad():
+ tensors = {"mlp.router.gate.weight": torch.randn(4, 4, dtype=torch.float16)}
+ with init_empty_weights():
+ top = _Top()
+ materialize_non_block_params(top, ["unrelated.block"], self._make_index(tensors, tmp_path), device="cpu")
+ assert top.mlp.gate.weight.device.type != "meta"
+
+
+class TestOffloadLoadBlockAliases:
+ """The offloader's block loader must resolve checkpoint-conversion renames
+ for meta-skeleton blocks (disk-streamed models), matching what the scoring
+ streams do -- otherwise aliased tensors (router, shared experts) silently
+ stay on meta and crash the tuning loop's device placement."""
+
+ def _write_checkpoint(self, tmp_path, tensors):
+ import json
+
+ from safetensors.torch import save_file
+
+ model_dir = tmp_path / "offload_ckpt"
+ model_dir.mkdir(exist_ok=True)
+ (model_dir / "config.json").write_text(json.dumps({"model_type": "hy_v3"}))
+ save_file(tensors, str(model_dir / "model.safetensors"))
+ weight_map = {name: "model.safetensors" for name in tensors}
+ (model_dir / "model.safetensors.index.json").write_text(json.dumps({"weight_map": weight_map}))
+ return str(model_dir)
+
+ def _make_block(self):
+ import torch.nn as nn
+
+ class _Shared(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate_proj = nn.Linear(4, 4, bias=False)
+ self.up_proj = nn.Linear(4, 4, bias=False)
+
+ class _Mlp(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate = nn.Linear(4, 4, bias=False)
+ self.shared_experts = _Shared()
+ self.up_proj = nn.Linear(4, 4, bias=False)
+
+ with init_empty_weights():
+ return _Mlp()
+
+ def test_aliased_meta_block_materializes(self, tmp_path):
+ from auto_round.utils.offload import load_block_from_model_files
+
+ with torch.no_grad():
+ tensors = {
+ "model.layers.1.mlp.router.gate.weight": torch.randn(4, 4, dtype=torch.float16),
+ "model.layers.1.mlp.shared_mlp.gate_proj.weight": torch.randn(4, 4, dtype=torch.float16),
+ "model.layers.1.mlp.shared_mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float16),
+ "model.layers.1.mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float16),
+ }
+ model_dir = self._write_checkpoint(tmp_path, tensors)
+ block = self._make_block()
+ load_block_from_model_files(model_dir, "model.layers.1.mlp", block)
+ assert block.gate.weight.device.type != "meta"
+ assert block.shared_experts.gate_proj.weight.device.type != "meta"
+ assert block.shared_experts.up_proj.weight.device.type != "meta"
+ assert block.up_proj.weight.device.type != "meta"
+ assert torch.equal(block.gate.weight, tensors["model.layers.1.mlp.router.gate.weight"])
+
+ def test_real_block_still_uses_prefix_loading(self, tmp_path):
+ from auto_round.utils.offload import load_block_from_model_files
+
+ with torch.no_grad():
+ tensors = {"model.layers.1.mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float32)}
+ model_dir = self._write_checkpoint(tmp_path, tensors)
+ block = self._make_block()
+ load_block_from_model_files(model_dir, "model.layers.1.mlp", block)
+ assert block.up_proj.weight.device.type != "meta"
+ assert torch.equal(block.up_proj.weight, tensors["model.layers.1.mlp.up_proj.weight"])