PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: AutoScheme parallel scoring OOMs (MoE memory, VLM gate) and disk-stream tuning crash (#2220)
avtc7da9bea890b2
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex 3f405fb3e..c7207bea6 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -84,6 +84,43 @@ from auto_round.wrapper import WrapperLinear__all__ = ["gen_layer_config"]+class _ScoreLinear(torch.autograd.Function):+ """Linear for scoring passes whose weight lives in the wrapper's CPU cache.++ Saves only the input activation: the dequantized weight is transferred+ from the cache again wherever needed, so the block's autograd graph holds+ no weight-shaped tensors. The weight score is accumulated from the+ explicit gradient instead of a tensor hook.+ """++ @staticmethod+ def forward(ctx, x, wrapper, bias): # pylint: disable=arguments-differ+ ctx.wrapper = wrapper+ ctx.save_for_backward(x)+ weight = wrapper._score_weight_for_device(x.device)+ return torch.nn.functional.linear(x, weight, bias)++ @staticmethod+ def backward(ctx, grad_out): # pylint: disable=arguments-differ+ (x,) = ctx.saved_tensors+ wrapper = ctx.wrapper+ weight = wrapper._score_weight_for_device(x.device)+ grad_x = None+ if ctx.needs_input_grad[0]:+ grad_x = torch.matmul(grad_out, weight)+ if wrapper.grad_mode and wrapper.need_weight_grad:+ grad_out_2d = grad_out.reshape(-1, grad_out.shape[-1])+ x_2d = x.reshape(-1, x.shape[-1])+ grad_w = torch.mm(grad_out_2d.t(), x_2d)+ weight_ref = wrapper.orig_layer.weight+ if weight_ref.device.type == "meta":+ weight_ref = wrapper.orig_layer.get_weight().to(x.device)+ w_diff = weight_ref.to(x.device) - weight+ wrapper.weight_score += torch.abs(grad_w * w_diff).sum().item()+ wrapper.mix_score = wrapper.weight_score + wrapper.act_score+ return grad_x, None, None++class AutoSchemeWrapperLinear(WrapperLinear):def __init__(@@ -121,6 +158,12 @@ class AutoSchemeWrapperLinear(WrapperLinear):self.grad_mode = Falseif self.need_weight_grad:self.orig_layer.weight.requires_grad = True+ # scoring cache: the dequantized weight, computed once per wrapper+ # visit and stored on CPU; bounded to the block being replayed so+ # parallel workers' caches stay within host RAM+ self._score_qdq_cpu = None+ # use the recompute-weight scoring forward (see _ScoreLinear)+ self._custom_score_forward = Truedef _qdq_act(self, x, act_min_scale=1.0, act_max_scale=1.0, act_max=None):"""Quant-dequant the activation and, in ``grad_mode``, register a backward hook that@@ -161,38 +204,85 @@ class AutoSchemeWrapperLinear(WrapperLinear):qdq_x.register_hook(save_grad)return qdq_x, scale, zp+ def _ensure_score_cache(self):+ """Build the per-visit CPU cache of the dequantized weight (scoring layers only)."""+ if self._score_qdq_cpu is not None:+ return+ device = self.device+ with torch.no_grad():+ qdq_w, _, _ = super(AutoSchemeWrapperLinear, self)._qdq_weight(+ torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device)+ )+ self._score_qdq_cpu = qdq_w.detach().to("cpu")++ def _score_weight_for_device(self, device):+ """Return the cached dequantized weight on ``device`` (transfers from the CPU cache)."""+ self._ensure_score_cache()+ return self._score_qdq_cpu.to(device)++ def forward(self, x):+ """In the scoring replay, run the linear through ``_ScoreLinear`` so the block's+ autograd graph saves activations only; every other phase uses the base forward."""+ if getattr(self, "_custom_score_forward", False) and self.grad_mode and torch.is_grad_enabled():+ x = x.to(self.device)+ for hook in self.orig_layer._forward_pre_hooks.values():+ result = hook(self.orig_layer, (x,))+ if result is not None:+ x = result[0] if isinstance(result, tuple) else result+ bias = self.orig_layer.bias+ if bias is not None and bias.device.type == "meta":+ bias = self.orig_layer.get_bias().to(self.device)+ output = _ScoreLinear.apply(x, self, bias)+ for hook in self.orig_layer._forward_hooks.values():+ hook_result = hook(self.orig_layer, (x,), output)+ if hook_result is not None:+ output = hook_result+ return output.to(self.output_device)+ return super().forward(x)+def _qdq_weight(self, value, min_scale, max_scale):- """Quant-dequant the weight and, in ``grad_mode``, register a backward hook that- accumulates ``weight_score`` from ``|grad * (weight - qdq_w)|``. Weight quantization- for the hook is only recomputed lazily inside the hook itself, so layers whose- forward never runs (e.g. unrouted MoE experts) never pay this cost.+ """Quant-dequant the weight once per scoring pass, cache the result, and, in+ ``grad_mode``, route the score hook through a scalar anchor.++ The cached quantization is recomputed lazily on the first call, so layers+ whose forward never runs (e.g. unrouted MoE experts) never pay this cost.+ In ``grad_mode`` the anchor makes the returned tensor a graph node whose+ gradient is exactly d(loss)/d(qdq_w): the score hook fires without building+ the quant chain into the graph and without materializing weight-shaped+ gradient buffers, which on expert-heavy blocks dominated replay VRAM."""device = self.device- if self.orig_layer.bits > 8 or not self.need_weight_grad:- qdq_w, scale, zp = super()._qdq_weight(+ if self.orig_layer.bits > 8:+ return super()._qdq_weight(torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device))- return qdq_w, 1.0, None-- qdq_w, scale, zp = super()._qdq_weight(- torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device)- )+ scoring = self.grad_mode and self.need_weight_grad+ if not scoring:+ # one-shot layers (capture pass, non-grad phases) never reuse the+ # result, so do not grow a CPU-side cache for them+ with torch.no_grad():+ return super()._qdq_weight(+ torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device)+ )- if self.grad_mode:+ self._ensure_score_cache()+ qdq_w = self._score_qdq_cpu.to(device)+ if scoring:+ anchor = torch.zeros((), dtype=qdq_w.dtype, device=qdq_w.device, requires_grad=True)+ qdq_w = qdq_w + 0.0 * anchordef save_grad(grad):"""Backward hook: accumulate weight score from grad * (weight - qdq_w)."""- qdq_w, scale, zp = self.super_qdq_func(- torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device)- )- w_diff = self.orig_layer.weight - qdq_w.to(self.orig_layer.weight.device)- self.weight_score += torch.abs((grad.to(w_diff.device) * w_diff)).sum().item()+ weight = self.orig_layer.weight+ if weight.device.type == "meta":+ weight = self.orig_layer.get_weight().to(grad.device)+ w_diff = weight.to(grad.device) - self._score_qdq_cpu.to(grad.device)+ self.weight_score += torch.abs(grad.to(w_diff.device) * w_diff).sum().item()self.mix_score = self.weight_score + self.act_scorereturn None- if qdq_w.requires_grad:- qdq_w.register_hook(save_grad)+ qdq_w.register_hook(save_grad)return qdq_w, 1.0, None@@ -240,6 +330,7 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):group_size=orig_layer.group_size,iters=0,)+ self._custom_score_forward = Falseself.post_init_qdqw(device)@torch.no_grad()@@ -342,6 +433,7 @@ class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):need_weight_grad,**kwargs,)+ self._custom_score_forward = Falseself.post_init_qdqw(device)@torch.no_grad()@@ -400,6 +492,7 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):enable_torch_compile=enable_torch_compile,**kwargs,)+ self._custom_score_forward = Falseself.post_init_qdqw(device)@torch.no_grad()@@ -596,9 +689,12 @@ def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_devmaterialize_module(module, module_name, disk_index, device=major_device)move_module_to_tuning_device(module, major_device=major_device)- # for n,m in module.named_modules():- # if hasattr(m, "post_init_qdqw"):- # m.post_init_qdqw()+ # The block now sits on major_device; its incoming tensors may have+ # been emitted on another device (e.g. CPU-resident embeddings, or+ # CPU-recorded replay inputs), so align them before the forward --+ # a same-device .to() is a no-op reference return.+ args = tuple(a.to(major_device) if isinstance(a, torch.Tensor) else a for a in args)+ kwargs = {k: v.to(major_device) if isinstance(v, torch.Tensor) else v for k, v in kwargs.items()}# Call the original forwardwith torch.no_grad():@@ -695,6 +791,34 @@ def model_forward(model, data, **forward_kwargs):return model(**prepared, **forward_kwargs), prepared+def _clear_wrapper_score_caches(block_module):+ """Drop the per-wrapper scoring caches of every layer in ``block_module`."++ Wrappers persist for a whole scoring pass while their caches are only valid+ while the owning layer's weights are materialized; without this the CPU-side+ caches accumulate across blocks of the model.+ """+ for module in block_module.modules():+ if getattr(module, "_score_qdq_cpu", None) is not None:+ module._score_qdq_cpu = None+++def _replay_retain_graph(block_module) -> bool:+ """Whether this block's backward must keep its autograd graph alive.++ MX-family data types re-run quantization code inside backward hooks that+ expect the block graph to still exist. Every other data type frees the+ graph per block: the reverse replay holds one block's saved activations at+ a time, and retaining graphs across blocks accumulates them for the whole+ pass, which is what pushes streamed scoring workers OOM mid-pass.+ """+ for _, module in block_module.named_modules():+ data_type = getattr(module, "data_type", None)+ if isinstance(data_type, str) and data_type.startswith("mx"):+ return True+ return False++def _prepare_replay_input(block_input_args, block_input_kwargs, block_name):"""Find the floating hidden-state tensor whose gradient feeds the preceding block."""candidates = []@@ -709,7 +833,184 @@ def _prepare_replay_input(block_input_args, block_input_kwargs, block_name):raise RuntimeError(f"No floating replay input found for block {block_name}")-def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, scheme_tag=None, disk_index=None):+def _vram_inventory_text(top_k: int = 12) -> str:+ """Return the live-CUDA-tensor census as text (see ``_vram_inventory``).++ Grouped by (shape, dtype) signature, largest first -- block weight shapes+ are recognizable, which is what makes an at-failure census actionable.+ """+ import gc as _gc+ from collections import defaultdict++ if not torch.cuda.is_available():+ return "cuda unavailable; no census"+ groups = defaultdict(lambda: [0, 0])+ for obj in _gc.get_objects():+ try:+ if torch.is_tensor(obj) and obj.is_cuda:+ key = (tuple(obj.shape), str(obj.dtype))+ groups[key][0] += 1+ groups[key][1] += obj.element_size() * obj.numel()+ except Exception: # noqa: BLE001+ continue+ ranked = sorted(groups.items(), key=lambda kv: -kv[1][1])[:top_k]+ total = sum(v[1] for v in groups.values())+ alloc = torch.cuda.memory_allocated() / 2**30+ lines = [f"live tensors {total / 2**30:.2f} GiB (allocator {alloc:.2f} GiB)"]+ for (shape, dtype), (cnt, nb) in ranked:+ lines.append(f" {nb / 2**30:6.2f} GiB x{cnt:<4} {dtype} {shape}")+ # name the retaining objects for the largest group so the census points at+ # the retainer, not only the retained shape+ for rank, ((shape, dtype), _) in enumerate(ranked[:2]):+ target = None+ objs = _gc.get_objects()+ for obj in objs:+ try:+ if torch.is_tensor(obj) and obj.is_cuda and tuple(obj.shape) == shape:+ target = obj+ break+ except Exception: # noqa: BLE001+ continue+ # The enumeration list itself references every object it yielded; drop+ # it before asking for referrers or it shadows the real retainers.+ del objs+ if target is not None:+ trace = _tensor_referrer_snippet(target)+ if trace:+ lines.append(f" group#{rank} referrers {shape}:\n{trace}")+ return "\n".join(lines)+++def _tensor_referrer_snippet(tensor, max_depth: int = 3, max_entries: int = 4) -> str:+ """Describe what holds ``tensor`` alive, as a compact referrer chain.++ Walks ``gc.get_referrers`` breadth-first, reporting container types (list /+ dict sizes, module class names) so a retention census names the retaining+ object rather than only the retained shape.+ """+ import gc as _gc++ import torch.nn as _nn++ def _describe(obj, refers_to=None):+ if isinstance(obj, (list, tuple)):+ return f"{type(obj).__name__}[{len(obj)}]"+ if isinstance(obj, dict):+ keys = list(obj.keys())[:max_entries]+ shown = [k if isinstance(k, str) else type(k).__name__ for k in keys]+ desc = "dict{" + ", ".join(map(str, shown)) + "}"+ if refers_to is not None:+ # name the holding attribute deterministically: identity-search+ # the walked object among the dict's values, so the snippet does+ # not depend on key order or gc traversal order across platforms+ for key, value in obj.items():+ if value is refers_to and isinstance(key, str):+ return f"{desc}[holding {key!r}]"+ return desc+ if isinstance(obj, _nn.Module):+ desc = f"module:{type(obj).__name__}"+ if refers_to is not None:+ for store_name in ("_parameters", "_buffers", "_modules"):+ store = getattr(obj, store_name, None)+ if isinstance(store, dict):+ for attr_name, value in store.items():+ if value is refers_to:+ return f"{desc}.{store_name}[{attr_name!r}]"+ for attr_name, value in obj.__dict__.items():+ if value is refers_to:+ return f"{desc}.{attr_name}"+ return desc+ return type(obj).__name__++ seen = {id(tensor)}+ frontier = [tensor]+ lines = []+ for _depth in range(max_depth):+ next_frontier = []+ for obj in frontier:+ try:+ referrers = _gc.get_referrers(obj)+ except Exception: # noqa: BLE001+ continue+ for ref in referrers:+ if id(ref) in seen:+ continue+ seen.add(id(ref))+ desc = _describe(ref)+ if desc in ("frame", "builtin_function_or_method", "function"):+ continue+ if isinstance(ref, (list, tuple)) and len(ref) > 10000:+ # giant containers at this depth are bookkeeping artifacts+ # (object registries, gc internals), not model state+ continue+ lines.append(f" depth{_depth + 1} <- {_describe(ref, refers_to=obj)}")+ if isinstance(ref, (list, tuple, dict)):+ next_frontier.append(ref)+ if len(lines) >= max_entries * 3:+ break+ frontier = next_frontier+ if not frontier or len(lines) >= max_entries * 3:+ break+ return "\n".join(lines[: max_entries * 3]) if lines else "no external referrers found"+++def _annotate_worker_oom(worker_index, exc):+ """Attach a live-tensor census and the failing op to a CUDA OOM in a worker."""+ import traceback as _tb++ try:+ census = _vram_inventory_text(top_k=20)+ except Exception: # noqa: BLE001 the census must never mask the OOM+ census = "census unavailable"+ tb_lines = _tb.format_exc().splitlines()+ tb_head = "\n".join(tb_lines[:4])+ tb_tail = "\n".join(tb_lines[-8:])+ return RuntimeError(+ f"_score_scheme_worker[{worker_index}] CUDA OOM during scoring. "+ f"Live-tensor census at failure:\n{census}\n\n"+ f"Traceback head:\n{tb_head}\n...\nTraceback tail:\n{tb_tail}"+ f"Original error: {exc}"+ )+++def _vram_inventory(tag: str, top_k: int = 12):+ """Env-gated (AR_SCHEME_MEM_INVENTORY=1) per-device live-tensor census:+ resident CUDA tensors grouped by (shape, dtype) signature, largest first --+ used to spot unreleased module weights or retained graphs during streamed+ scoring (block shapes are recognizable: e.g. [5120, 17408] = mlp.down)."""+ import gc as _gc+ import os as _os+ from collections import defaultdict++ if _os.getenv("AR_SCHEME_MEM_INVENTORY", "0") != "1":+ return+ groups = defaultdict(lambda: [0, 0])+ for obj in _gc.get_objects():+ try:+ if torch.is_tensor(obj) and obj.is_cuda:+ key = (tuple(obj.shape), str(obj.dtype))+ groups[key][0] += 1+ groups[key][1] += obj.element_size() * obj.numel()+ except Exception: # noqa: BLE001+ continue+ ranked = sorted(groups.items(), key=lambda kv: -kv[1][1])[:top_k]+ total = sum(v[1] for v in groups.values())+ alloc = torch.cuda.memory_allocated() / 2**30+ print(f"[mem-inv] {tag}: live tensors {total / 2**30:.2f} GiB " f"(allocator {alloc:.2f} GiB)", flush=True)+ for (shape, dtype), (cnt, nb) in ranked:+ print(f"[mem-inv] {nb / 2**30:6.2f} GiB x{cnt:<4} {dtype} {shape}", flush=True)+++def model_forward_low_gpu(+ model,+ dataloader,+ major_device="cuda",+ pbar=None,+ scheme_tag=None,+ disk_index=None,+ skip_batches=0,+ batch_checkpoint=None,+):"""Run one full scoring pass (all calibration batches) in low-GPU-memory mode.For each batch: capture per-block inputs via ``prepare_model_low_gpu``, run a forward@@ -741,6 +1042,12 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, schraise MyCustomError("Interrupt backward pass")for batch_idx, data in enumerate(dataloader, start=1):+ if batch_idx <= skip_batches:+ # resumed run: these batches' contributions are already in the+ # restored accumulators; advance the loader and the progress bar only+ if pbar is not None:+ pbar.update(len(block_names) * 2)+ continuecaptured_grad = Noneinterrupted = Falselast_block_backward_hook = None@@ -835,7 +1142,7 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, schtorch.autograd.backward(tensors=main_output,grad_tensors=current_grad,- retain_graph=True, # False may lead to zero gradients for some cases (e.g., MXFP4)+ retain_graph=_replay_retain_graph(block_module),)if replay_input.grad is None:@@ -845,6 +1152,7 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, schfinally:for parameter in block_module.parameters():parameter.grad = None+ _clear_wrapper_score_caches(block_module)if disk_index is not None and materialized:from auto_round.utils.disk_stream_util import free_module@@ -858,6 +1166,11 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, schif pbar is not None:pbar.update(1)+ _vram_inventory(f"scheme={scheme_tag} batch={batch_idx}/{total_batches} post-replay")++ if batch_checkpoint is not None:+ batch_checkpoint(batch_idx, total_batches or -1)+_log_batch_avg_loss(model,batch_idx,@@ -877,6 +1190,8 @@ def get_score_for_scheme(ignore_scale_zp_bits=False,nsamples=16,seqlen=256,+ skip_batches=0,+ batch_checkpoint=None,pbar=None,shared_layers=None,need_weight_grad=False,@@ -1114,7 +1429,14 @@ def get_score_for_scheme("Provide a `processor` and a multimodal `dataset`.")model_forward_low_gpu(- model, mllm_loader, major_device=major_device, pbar=pbar, scheme_tag=scheme_tag, disk_index=disk_index+ model,+ mllm_loader,+ major_device=major_device,+ pbar=pbar,+ scheme_tag=scheme_tag,+ disk_index=disk_index,+ skip_batches=skip_batches,+ batch_checkpoint=batch_checkpoint,)else:try:@@ -1126,6 +1448,8 @@ def get_score_for_scheme(pbar=pbar,scheme_tag=scheme_tag,disk_index=disk_index,+ skip_batches=skip_batches,+ batch_checkpoint=batch_checkpoint,)except Exception as exc: # noqa: BLE001if not is_vlm:@@ -1636,6 +1960,57 @@ def _autoscheme_cache_path(cache_key, scheme_index):return os.path.join(cache_dir, f"scheme_{scheme_index:02d}_{cache_key}.json")+def _extract_score_accumulators(model):+ """Snapshot the additive per-layer scoring accumulators for mid-scheme resume."""+ state = {}+ for n, m in model.named_modules():+ if hasattr(m, "mix_score"):+ state[n] = [float(m.act_score), float(m.weight_score), float(m.act_cnt)]+ return state+++def _inject_score_accumulators(model, state):+ """Restore scoring accumulators snapshotted by ``_extract_score_accumulators``."""+ for n, m in model.named_modules():+ if n in state and hasattr(m, "mix_score"):+ m.act_score, m.weight_score, m.act_cnt = state[n]+++def _partial_scores_path(cache_path):+ return cache_path + ".partial"+++def _save_partial_scores(cache_path, batch_idx, total_batches, state):+ """Atomically persist a batch-granularity scoring checkpoint (JSON)."""+ payload = {+ "version": 1,+ "batches_done": batch_idx,+ "total_batches": total_batches,+ "scores_state": state,+ }+ tmp = _partial_scores_path(cache_path) + ".tmp"+ with open(tmp, "w", encoding="utf-8") as f:+ json.dump(payload, f)+ os.replace(tmp, _partial_scores_path(cache_path))+++def _load_partial_scores(cache_path, expected_total_batches):+ """Load a batch checkpoint if present and still applicable."""+ path = _partial_scores_path(cache_path)+ if cache_path is None or not os.path.exists(path):+ return None+ try:+ with open(path, encoding="utf-8") as f:+ data = json.load(f)+ if data.get("total_batches") != expected_total_batches:+ return None+ if not 0 < data.get("batches_done", 0) < expected_total_batches:+ return None+ return data+ except Exception: # noqa: BLE001+ return None++def _save_autoscheme_scores(cache_path,cache_key,@@ -1786,6 +2161,21 @@ def _refresh_cached_layer_bits(return refreshed_scores+def _parallel_scoring_must_raise(parallel_error: Exception) -> bool:+ """Decide whether a parallel-scoring failure aborts instead of falling back to serial.++ Set AR_AUTO_SCHEME_NO_SERIAL_FALLBACK=1 to fail fast: the serial fallback is+ ~workers-count times slower and, on models where it cannot run at all, only+ burns hours before crashing. Schemes lost with their worker always raise+ regardless of the env, since their scores are unrecoverable.+ """+ from auto_round import envs as _envs++ if "was lost with its worker" in str(parallel_error):+ return True+ return bool(_envs.AR_AUTO_SCHEME_NO_SERIAL_FALLBACK)++def _assign_scheme_worker_devices(worker_count, available_devices):"""Assign workers round-robin within the devices selected by the caller."""if not available_devices:@@ -1857,6 +2247,40 @@ def _get_scheme_worker_count(num_schemes, num_gpus):return num_schemes+def _weights_span_multiple_gpus(devices) -> bool:+ """Whether a device collection places weights on more than one GPU."""+ cuda_devices = {str(device) for device in devices if str(device).startswith("cuda")}+ return len(cuda_devices) > 1+++def _serial_scoring_device_safe(model, visible_cuda_devices=None) -> bool:+ """Whether in-process (serial) full-model scoring can run on ``model``.++ Serial scoring runs the whole-model forward through per-layer wrappers.+ Two placement sources can spread work over several GPUs: live parameter+ devices (a materialized model), and ``hf_device_map`` - which is what a+ disk-stream (meta-skeleton) model consults when its blocks materialize+ block-by-block during scoring. Either spanning more than one GPU makes+ the first cross-device layer raise a tensor-device error mid-forward.++ Additionally, a disk-stream (meta-skeleton) model scored serially with+ more than one visible CUDA device is not considered safe: block+ materialization and wrapper bookkeeping are only validated with a+ single visible GPU, and workers pin one GPU each. Such models are+ routed through disk-stream workers instead.+ """+ if _weights_span_multiple_gpus({parameter.device for parameter in model.parameters()}):+ return False+ hf_device_map = getattr(model, "hf_device_map", None)+ if hf_device_map:+ if _weights_span_multiple_gpus(set(hf_device_map.values())):+ return False+ if visible_cuda_devices is not None and len(visible_cuda_devices) > 1:+ if getattr(model, "_disk_stream_index", None) is not None:+ return False+ return True++def _can_parallel_scheme_scoring(parallel_enabled,model_id,@@ -1866,16 +2290,26 @@ def _can_parallel_scheme_scoring(disk_stream_model,is_vlm,low_gpu_mem_usage=True,+ force_mllm=False,+ min_uncached=2,):- """Return whether candidate schemes can be scored in separate workers."""+ """Return whether candidate schemes can be scored in separate workers.++ ``is_vlm`` is accepted for call-site compatibility; text-only scoring of a+ VLM's language tower streams blocks just like a text model. Only vision+ scoring (``force_mllm``) requires a full-model backward and is excluded+ from the block-wise materialize/free path used by disk streaming.+ ``min_uncached`` is the parallel-worthiness floor (2 by default); the call+ site lowers it to 1 when serial scoring cannot run on the loaded model and+ a disk-stream worker can."""return (parallel_enabledand low_gpu_mem_usageand model_id is not Noneand num_gpus >= 1- and uncached_count >= 2+ and uncached_count >= min_uncachedand not need_imatrix- and (not disk_stream_model or not is_vlm)+ and (not disk_stream_model or not force_mllm))@@ -1895,6 +2329,15 @@ def _load_disk_stream_scheme_worker_model(model_name, use_model_replacements=Falfrom auto_round.utils.disk_stream_util import build_meta_modelmodel, tokenizer, disk_index = build_meta_model(model_name)+ # The regular load pipeline applies custom replacements, among them the+ # structural MoE unfusing that turns fused expert containers into per-expert+ # Linear modules; the meta-skeleton build skips that pipeline. Unfuse here+ # so per-expert quant layers resolve -- weights stay on meta and are filled+ # per-block from the checkpoint index during scoring.+ from auto_round.modeling.fused_moe.replace_modules import _handle_moe_modules++ unfused = _handle_moe_modules(model)+ logger.info("disk-stream scoring worker: structural MoE unfuse produced %d unfused experts modules", len(unfused))if use_model_replacements:from auto_round.special_model_handler import _handle_special_model, update_module@@ -1904,8 +2347,12 @@ def _load_disk_stream_scheme_worker_model(model_name, use_model_replacements=Faldef _prefer_disk_stream_scheme_worker(model_id, is_vlm, low_gpu_mem_usage):- """Prefer block-wise disk streaming whenever the worker scoring path supports it."""- return model_id is not None and not is_vlm and low_gpu_mem_usage+ """Prefer block-wise disk streaming whenever the worker scoring path supports it.++ Multimodal archs are covered: build_meta_model resolves the class from+ config.architectures and the workers materialize non-block params (vision+ tower included) via materialize_non_block_params."""+ return model_id is not None and low_gpu_mem_usagedef _score_scheme_worker(args):@@ -1934,6 +2381,7 @@ def _score_scheme_worker(args):total_schemes,progress_queue,disk_stream_model,+ worker_cache_path,) = argsfrom auto_round.auto_scheme.utils import _scheme_short_name as _short_name@@ -1997,8 +2445,13 @@ def _score_scheme_worker(args):for layer_name in quant_layer_names:layer = _get_module(model, layer_name)if layer is None:+ parent_name = layer_name.rsplit(".", 1)[0]+ parent = _get_module(model, parent_name)+ expert_containers = sum(1 for _, m in model.named_modules() if m.__class__.__name__ == "_ExpertContainer")raise RuntimeError(- f"_score_scheme_worker[{index}]: layer {layer_name!r} is missing after model preprocessing"+ f"_score_scheme_worker[{index}]: layer {layer_name!r} is missing after model preprocessing "+ f"(parent {parent_name!r} resolves to {type(parent).__name__ if parent is not None else None}; "+ f"expert containers in model: {expert_containers}; disk-stream build: {disk_index is not None})")if not hasattr(layer, "tuning_device"):layer.tuning_device = worker_device@@ -2011,6 +2464,27 @@ def _score_scheme_worker(args):scheme=scheme,)+ # mid-scheme resume: reload additive accumulators and skip completed batches+ skip_batches = 0+ if worker_cache_path is not None:+ expected_total = max(1, math.ceil(nsamples / max(1, batch_size)))+ partial = _load_partial_scores(worker_cache_path, expected_total)+ if partial is not None:+ _inject_score_accumulators(model, partial["scores_state"])+ skip_batches = partial["batches_done"]+ logger.info(+ "_score_scheme_worker[%d]: resuming from batch checkpoint %d/%d",+ index,+ skip_batches,+ expected_total,+ )++ def batch_checkpoint(batch_idx, total_batches):+ _save_partial_scores(worker_cache_path, batch_idx, total_batches, _extract_score_accumulators(model))++ else:+ batch_checkpoint = None+is_bf16 = isinstance(scheme, str) and scheme.upper() == "BF16"if isinstance(scheme, dict):is_bf16 = scheme.get("bits", 16) >= 16 and scheme.get("act_bits", 16) >= 16@@ -2025,29 +2499,36 @@ def _score_scheme_worker(args):from auto_round.auto_scheme.delta_loss import get_score_for_scheme- scores = get_score_for_scheme(- model,- tokenizer,- quant_layer_names,- fixed_layer_scheme,- dataset,- ignore_scale_zp_bits=ignore_scale_zp_bits,- pbar=_ProgressQueueProxy(progress_queue),- nsamples=nsamples,- seqlen=seqlen,- need_weight_grad=need_weight_grad,- enable_torch_compile=enable_torch_compile,- low_gpu_mem_usage=low_gpu_mem_usage,- major_device=worker_device,- batch_size=batch_size,- offload_context=None,- processor=processor,- is_vlm=is_vlm,- force_mllm=force_mllm,- model_name=model_name,- scheme_tag=f"{index + 1}/{total_schemes} {_short_name(scheme)}",- disk_index=disk_index,- )+ try:+ scores = get_score_for_scheme(+ model,+ tokenizer,+ quant_layer_names,+ fixed_layer_scheme,+ dataset,+ ignore_scale_zp_bits=ignore_scale_zp_bits,+ pbar=_ProgressQueueProxy(progress_queue),+ nsamples=nsamples,+ seqlen=seqlen,+ skip_batches=skip_batches,+ batch_checkpoint=batch_checkpoint,+ need_weight_grad=need_weight_grad,+ enable_torch_compile=enable_torch_compile,+ low_gpu_mem_usage=low_gpu_mem_usage,+ major_device=worker_device,+ batch_size=batch_size,+ offload_context=None,+ processor=processor,+ is_vlm=is_vlm,+ force_mllm=force_mllm,+ model_name=model_name,+ scheme_tag=f"{index + 1}/{total_schemes} {_short_name(scheme)}",+ disk_index=disk_index,+ )+ except RuntimeError as exc:+ if "out of memory" not in str(exc).lower():+ raise+ raise _annotate_worker_oom(index, exc) from excreturn index, scores, _get_worker_memory_report(worker_device)@@ -2256,7 +2737,11 @@ def _gen_layer_config(act_bits = scheme.get("act_bits", 16)if scheme.get("super_group_size"):need_imatrix = True- if bits <= 8 < act_bits:+ # Weight scores must accumulate for every weight-quantized scheme, not+ # only A16 ones: MX options (W4A4/W8A8) quantify weight error solely via+ # the weight score, and skipping it zeroes every layer score, so the+ # bit allocation collapses to the cheapest option.+ if bits <= 8:need_weight_grad = Trueif not auto_scheme.low_gpu_mem_usage:pbar_cnt += nsamples@@ -2403,6 +2888,10 @@ def _gen_layer_config(def _save_per_op_scores(index, scheme, cache_key, cache_path, per_op_scores):if cache_key is None or cache_path is None:return+ try:+ os.remove(_partial_scores_path(cache_path))+ except OSError:+ passif isinstance(scheme, str):scheme_dict = asdict(preset_name_to_scheme(scheme))elif isinstance(scheme, QuantizationScheme):@@ -2469,6 +2958,46 @@ def _gen_layer_config(worker_disk_stream_model = _prefer_disk_stream_scheme_worker(_model_id_for_cache, is_vlm, auto_scheme.low_gpu_mem_usage)+ # Serial scoring runs the full-model forward, which cannot cross GPUs on+ # a manually sharded model. When that is the loaded configuration and a+ # disk-stream worker is possible, lower the parallel floor to 1 so even+ # a single uncached scheme is scored through a worker instead of the+ # broken in-process pass.+ serial_device_safe = _serial_scoring_device_safe(model, visible_cuda_devices=worker_device_pool)+ min_uncached = 2+ if not serial_device_safe and worker_disk_stream_model:+ min_uncached = 1+ logger.info(+ "AutoScheme: serial in-process scoring is unsafe (multi-GPU weights, "+ "hf_device_map, or a streamed model with several visible GPUs); "+ "routing even a single uncached scheme through a disk-stream scoring worker."+ )+ elif not serial_device_safe and uncached_indices:+ raise RuntimeError(+ "AutoScheme serial scoring cannot run on a model whose weights span "+ "multiple GPUs without accelerate dispatch (the full-model forward has "+ "no cross-device activation transfer). Score via disk-stream workers: "+ "point AR_DISK_STREAM_MODEL at a local checkpoint directory and keep "+ "AR_ENABLE_AUTO_SCHEME_PARALLEL enabled, or load the model on a single "+ "GPU / with device_map='auto'."+ )+ if __debug__:+ from collections import Counter++ _param_devs = Counter(str(p_.device) for p_ in model.parameters())+ _hf_map = getattr(model, "hf_device_map", None)+ _hf_devs = sorted({str(v_) for v_ in _hf_map.values()}) if _hf_map else None+ logger.info(+ "AutoScheme serial-safety census: serial_device_safe=%s "+ "param_devices=%s hf_device_map_devices=%s disk_stream_index=%s meta_params=%d "+ "cuda_visible=%s",+ serial_device_safe,+ dict(_param_devs),+ _hf_devs,+ getattr(model, "_disk_stream_index", None) is not None,+ sum(1 for p_ in model.parameters() if p_.device.type == "meta"),+ [str(d_) for d_ in worker_device_pool],+ )# Vision scoring requires a full-model backward and therefore cannot use# the block-wise materialize/free path used by disk streaming.can_parallel = _can_parallel_scheme_scoring(@@ -2480,6 +3009,8 @@ def _gen_layer_config(worker_disk_stream_model,is_vlm,low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage,+ force_mllm=force_mllm,+ min_uncached=min_uncached,)logger.info("AutoScheme scoring mode: parallel_configured=%s, parallel_enabled=%s, disk_stream_enabled=%s",@@ -2519,6 +3050,41 @@ def _gen_layer_config("AutoScheme: if parallel scoring runs out of RAM/VRAM or automatic serial fallback cannot ""recover, set AR_ENABLE_AUTO_SCHEME_PARALLEL=0 and rerun.")+ # free the parent's GPU before spawning workers: non-block params+ # (embed/lm_head/vision) parked on a device by earlier setup are+ # dead weight during parallel scoring -- the parent never runs a+ # forward, and each worker owns its private copy. A shared-GPU+ # worker then gets the card's full budget.+ _moved = 0+ _parked = []+ for _p in model.parameters():+ if _p.device.type == "cuda":+ _parked.append((_p, _p.device))+ _p.data = _p.data.to("cpu")+ _moved += 1+ for _b in model.buffers():+ if _b.device.type == "cuda":+ _parked.append((_b, _b.device))+ _b.data = _b.data.to("cpu")++ def _restore_parked_tensors():+ """Move tensors parked before spawning back to their devices.++ The serial fallback and the post-parallel phases run+ full-model forwards; leaving non-block params on CPU while+ ``model.device`` still reports the GPU crashes the first+ non-block module on a device mismatch (e.g. final norm)."""+ for _tensor, _device in _parked:+ if _tensor.device.type == "cpu" and _device.type != "cpu":+ _tensor.data = _tensor.data.to(_device)+ _parked.clear()++ if _moved:+ logger.info("AutoScheme: parked %d parent-side GPU tensors on CPU before spawning workers", _moved)+ clear_memory()++ _vram_inventory("parent pre-spawn")+spawn_context = multiprocessing.get_context("spawn")with spawn_context.Manager() as manager:progress_queue = manager.Queue()@@ -2545,17 +3111,44 @@ def _gen_layer_config(len(schemes),progress_queue,worker_disk_stream_model,+ scheme_cache_meta[index][1], # cache_path for batch checkpoints)for slot, index in enumerate(uncached_indices)]with spawn_context.Pool(processes=num_workers) as pool:- async_results = pool.map_async(_score_scheme_worker, worker_args)- while not async_results.ready():- _drain_progress_queue(progress_queue, pbar)- memory_monitor.update_cpu()- async_results.wait(timeout=0.1)+ # incremental consumption: persist each scheme's scores to+ # the per-scheme cache as its worker returns, so a partial+ # failure (or serial-fallback crash) never discards+ # completed schemes -- the rerun picks them up from cache+ result_iter = pool.imap_unordered(_score_scheme_worker, worker_args)+ worker_results = []+ failed_workers = 0+ while True:+ try:+ worker_results.append(result_iter.next(timeout=0.5))+ except multiprocessing.TimeoutError:+ _drain_progress_queue(progress_queue, pbar)+ memory_monitor.update_cpu()+ continue+ except StopIteration:+ break+ except Exception as worker_error: # noqa: BLE001+ # one worker's failure must not abort its siblings:+ # keep consuming their results (each already-saved+ # scheme survives in the per-scheme cache)+ failed_workers += 1+ logger.warning(+ "AutoScheme: a scoring worker failed (%s); collecting remaining workers' results.",+ worker_error,+ )+ _drain_progress_queue(progress_queue, pbar)+ continue+ index, scores, _report = worker_results[-1]+ cache_key, cache_path, _ = scheme_cache_meta[index]+ _save_per_op_scores(index, schemes[index], cache_key, cache_path, scores)_drain_progress_queue(progress_queue, pbar)- worker_results = async_results.get()+ if not worker_results:+ raise RuntimeError("all parallel scoring workers failed") from Noneparallel_results = {index: scores for index, scores, _ in worker_results}_merge_worker_memory_reports(memory_monitor, [report for _, _, report in worker_results])@@ -2599,8 +3192,17 @@ def _gen_layer_config(if not check_bf16_scheme(scheme):pbar.update(pbar_cnt // effective_scheme_num if effective_scheme_num > 0 else 1)else:- per_op_scores = parallel_results[index]- _save_per_op_scores(index, scheme, cache_key, cache_path, per_op_scores)+ if index in parallel_results:+ per_op_scores = parallel_results[index]+ _save_per_op_scores(index, scheme, cache_key, cache_path, per_op_scores)+ else:+ # worker died for this scheme; siblings' results are saved.+ # Rerunning resumes/scores only the missing schemes.+ raise RuntimeError(+ f"AutoScheme: scheme {index} ({_scheme_short_name(scheme)}) was lost with its "+ "worker; completed schemes are persisted in the per-scheme cache -- rerun to "+ "score only the failed schemes."+ )total_loss = _record_scheme_scores(index, per_op_scores)logger.info("AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.6f)",@@ -2610,8 +3212,19 @@ def _gen_layer_config()parallel_done = Truelogger.info("AutoScheme: parallel scoring completed.")+ _restore_parked_tensors()post_scoring_started = time.perf_counter()except Exception as parallel_error: # noqa: BLE001+ _restore_parked_tensors()+ if _parallel_scoring_must_raise(parallel_error):+ logger.error(+ "AutoScheme: keeping the parallel scoring failure as a hard error "+ "(AR_AUTO_SCHEME_NO_SERIAL_FALLBACK is set, or a scheme was lost with its worker); "+ "completed schemes and batches are persisted in the per-scheme cache -- "+ "rerun to score only the failed parts, e.g. with a smaller "+ "AR_AUTO_SCHEME_BATCH_SIZE / AR_AUTO_SCHEME_NSAMPLES."+ )+ raiselogger.warning("AutoScheme: parallel scoring failed, falling back to serial: %s. ""If fallback cannot recover (for example after RAM/VRAM exhaustion), rerun with "diff --git a/auto_round/envs.py b/auto_round/envs.pyindex 2401c8ce2..5eaf8351c 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -27,6 +27,7 @@ if TYPE_CHECKING:AR_AUTO_SCHEME_NSAMPLES: Optional[int] = NoneAR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = NoneAR_AUTO_SCHEME_CACHE: Optional[str] = None+ AR_AUTO_SCHEME_NO_SERIAL_FALLBACK: bool = FalseAR_ENABLE_AUTO_SCHEME_PARALLEL: bool = TrueAR_NVFP4_E5M3_CACHE_HP_WEIGHT: bool = FalseAR_DISK_STREAM_MODEL: bool = False@@ -104,6 +105,8 @@ environment_variables: dict[str, Callable[[], Any]] = {# Stores persistent AutoScheme scoring results independently from AR_WORK_SPACE,# whose contents are temporary working data and may be cleaned after a run."AR_AUTO_SCHEME_CACHE": lambda: os.getenv("AR_AUTO_SCHEME_CACHE", None),+ "AR_AUTO_SCHEME_NO_SERIAL_FALLBACK": lambda: os.getenv("AR_AUTO_SCHEME_NO_SERIAL_FALLBACK", "0").lower()+ in ("1", "true", "yes"),# Enables AutoScheme to score schemes in parallel. Enabled by default;# set it to 0 when workers could exhaust host RAM or device memory."AR_ENABLE_AUTO_SCHEME_PARALLEL": lambda: os.getenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "1").lower()diff --git a/auto_round/utils/disk_stream_util.py b/auto_round/utils/disk_stream_util.pyindex 5115b300f..3da7864e6 100644--- a/auto_round/utils/disk_stream_util.py+++ b/auto_round/utils/disk_stream_util.py@@ -12,6 +12,7 @@ from __future__ import annotationsimport jsonimport logging+import refrom pathlib import Pathfrom typing import Dict@@ -76,6 +77,65 @@ class SafetensorsIndex:return [n for n in self.weight_map if n == prefix or n.startswith(dotted)]+# Model-side module names sometimes differ from the checkpoint-side tensor+# names. Transformers already maintains the authoritative per-family renames+# (``transformers/conversion_mapping.py``, checkpoint -> model direction);+# the disk-stream materializers know only model-side names, so those entries+# are inverted here. A rename is applied only when the model-side name is+# absent from the index AND the renamed candidate exists in it, so checkpoints+# whose names already match the model are untouched.+from functools import lru_cache+++@lru_cache(maxsize=None)+def _reverse_renamings_for(model_type):+ """Invert the checkpoint-conversion WeightRenaming entries for one family."""+ if not model_type:+ return ()+ try:+ from transformers.conversion_mapping import WeightRenaming, get_checkpoint_conversion_mapping+ except ImportError: # pragma: no cover - transformers is a hard dep, guarded for safety+ return ()+ mapping = get_checkpoint_conversion_mapping(model_type)+ if not mapping:+ return ()+ renamings = []+ for entry in mapping:+ if not isinstance(entry, WeightRenaming):+ # Other converter types (fusions, prefix moves) are not invertible+ # by simple regex substitution and are skipped.+ continue+ for target in entry.target_patterns:+ for source in entry.source_patterns:+ renamings.append((target, source))+ return tuple(renamings)+++@lru_cache(maxsize=None)+def _index_model_type(index):+ config_path = Path(index.checkpoint_dir) / "config.json"+ try:+ with open(config_path) as f:+ return json.load(f).get("model_type")+ except OSError:+ return None+++def _resolve_checkpoint_name(index, full_name: str):+ """Map a model-side parameter name to its checkpoint-side tensor name."""+ if index.has_tensor(full_name):+ return full_name+ for target_pattern, source_replacement in _reverse_renamings_for(_index_model_type(index)):+ # ``source_replacement`` arrives regex-escaped (``mlp\.router\.gate``);+ # as a ``re.sub`` replacement those backslashes would land in the string+ # verbatim, so unescape it to the plain checkpoint-side spelling.+ replacement = re.sub(r"\\(.)", r"\1", source_replacement)+ candidate = re.sub(target_pattern, lambda _m: replacement, full_name)+ if candidate != full_name and index.has_tensor(candidate):+ return candidate+ return None++def materialize_module(module: nn.Module, module_name: str, index: SafetensorsIndex, device: str) -> None:"""Populate `module`'s (currently meta) parameters/buffers with real data readdirectly from the checkpoint, onto `device`. `module_name` is `module`'s dotted@@ -125,14 +185,15 @@ def materialize_module(module: nn.Module, module_name: str, index: SafetensorsInif str(tensor.device) != "meta":continue # already materialized (e.g. shared/tied weights)full_name = f"{module_name}.{name}".replace(".orig_layer.", ".")- if not index.has_tensor(full_name):+ resolved_name = _resolve_checkpoint_name(index, full_name)+ if resolved_name is None:sliced = _fused_lookup(full_name)if sliced is not None:fused_targets.append((name, sliced))continuelogger.warning("No checkpoint tensor found for %s, leaving on meta", full_name)continue- targets.append((name, full_name, tensor.dtype))+ targets.append((name, resolved_name, tensor.dtype))for name, value in fused_targets:set_module_tensor_to_device(module, name, device, value=value, dtype=value.dtype)@@ -208,6 +269,14 @@ def build_meta_model(model_name: str, trust_remote_code: bool = True):model = model_cls(config)else:model = AutoModelForCausalLM.from_config(config, trust_remote_code=trust_remote_code)+ # Constructing from config does not tie weights (that happens in+ # from_pretrained), so tied params such as lm_head.weight -- absent from+ # the checkpoint by design -- stay separate meta tensors and later crash+ # with "Cannot copy out of meta tensor" on the first device move. Tie+ # here: the shared tensor materializes once from the checkpoint side it+ # is stored under and both names become real together.+ if getattr(model.config, "tie_word_embeddings", False):+ model.tie_weights()tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=trust_remote_code)index = SafetensorsIndex(model_name)return model, tokenizer, index@@ -232,10 +301,11 @@ def materialize_non_block_params(if str(tensor.device) != "meta" or _in_block(name):continuefull_name = name.replace(".orig_layer.", ".")- if not index.has_tensor(full_name):+ resolved_name = _resolve_checkpoint_name(index, full_name)+ if resolved_name is None:logger.warning("No checkpoint tensor found for %s, leaving on meta", full_name)continue- targets.append((name, full_name))+ targets.append((name, resolved_name))if not targets:return@@ -245,6 +315,12 @@ def materialize_non_block_params(# is required so the checkpoint's real dtype wins over whatever the# meta skeleton happened to declare.set_module_tensor_to_device(model, name, device, value=values[full_name], dtype=values[full_name].dtype)+ # set_module_tensor_to_device replaces parameter objects, which breaks+ # weight tying: a tied lm_head still references the old (meta) parameter+ # while the embedding points at the new real one. Re-tie so every tied+ # name follows its checkpoint-backed source tensor.+ if getattr(getattr(model, "config", None), "tie_word_embeddings", False):+ model.tie_weights()class stream_block_forward:diff --git a/auto_round/utils/offload.py b/auto_round/utils/offload.pyindex bc3d72f35..4f5ee3481 100644--- a/auto_round/utils/offload.py+++ b/auto_round/utils/offload.py@@ -243,6 +243,26 @@ def load_block_from_model_files(model_dir: str, block_name: str, block: torch.nnblock: The ``nn.Module`` to load weights into."""model_dir = _resolve_model_dir(model_dir)++ def _has_meta(module: torch.nn.Module) -> bool:+ return any(tensor.device.type == "meta" for tensor in list(module.parameters()) + list(module.buffers()))++ if _has_meta(block):+ # A meta-skeleton block (disk-streamed model): resolve each parameter's+ # checkpoint-side name (checkpoint-conversion renames, fused expert+ # layouts) instead of raw prefix matching, which silently skips tensors+ # whose checkpoint spelling differs (e.g. the router stored as+ # ``mlp.router.gate`` behind a ``mlp.gate`` module) and leaves them on+ # meta until a downstream ``.to(device)`` crashes.+ try:+ from auto_round.utils.disk_stream_util import SafetensorsIndex, materialize_module++ materialize_module(block, block_name, SafetensorsIndex(model_dir), device="cpu")+ if not _has_meta(block):+ return+ except FileNotFoundError:+ pass # non-safetensors checkpoint: fall through to prefix loading+weight_map = _build_weight_map(model_dir)prefix = block_name + "."diff --git a/docs/environments.md b/docs/environments.mdindex 6d16b3130..84a6fcdef 100644--- a/docs/environments.md+++ b/docs/environments.md@@ -161,6 +161,16 @@ export AR_AUTO_SCHEME_BATCH_SIZE=1export AR_AUTO_SCHEME_SEQLEN=1024```+### AR_AUTO_SCHEME_NO_SERIAL_FALLBACK+- **Description**: Turn a parallel-scoring failure into a hard error instead of falling back to serial scoring. Useful when the serial pass is known to be unable to run (or would take workers-count times longer): completed schemes and batches are persisted in the per-scheme cache, so a rerun scores only the failed parts.+- **Default**: unset -> parallel scoring failure falls back to serial+- **Valid Values**: `1`, `true`, `yes`+- **Usage**: Set this to fail fast on parallel scoring errors++```bash+export AR_AUTO_SCHEME_NO_SERIAL_FALLBACK=1+```+### AR_AUTO_SCHEME_CACHE- **Description**: Stores persistent per-scheme AutoScheme scoring JSON files. This directory is independent of `AR_WORK_SPACE`, which is reserved for temporary working data.- **Default**: `~/.cache/auto_round`@@ -181,6 +191,16 @@ export AR_AUTO_SCHEME_CACHE=/path/to/auto_scheme_cacheexport AR_ENABLE_AUTO_SCHEME_PARALLEL=0```+### AR_SCHEME_MEM_INVENTORY+- **Description**: When enabled, AutoScheme streaming scoring prints a live CUDA-tensor census (`[mem-inv]` lines) at block boundaries -- tensors grouped by (shape, dtype), largest first -- which makes unreleased module weights or retained autograd graphs visible as they accumulate. Independently of this variable, a richer census (live tensors, the retaining containers, and the failing op's traceback) is always attached to scoring-worker CUDA OOM errors.+- **Default**: `False`+- **Valid Values**: `"1"`, `"true"`, `"yes"` (case-insensitive) for enabling; any other value for disabling+- **Usage**: Enable when investigating VRAM growth during AutoScheme scoring++```bash+export AR_SCHEME_MEM_INVENTORY=1+```+### AR_NVFP4_E5M3_CACHE_HP_WEIGHT- **Description**: Controls whether `NVFP4E5M3QuantLinear` caches a dequantized high-precision weight after the first forward pass, instead of dequantizing the packed FP4 weight on every call.- **Default**: `False` (equivalent to `"0"`)diff --git a/docs/environments_CN.md b/docs/environments_CN.mdindex 8648b4d76..a9c1f4aea 100644--- a/docs/environments_CN.md+++ b/docs/environments_CN.md@@ -161,6 +161,16 @@ export AR_AUTO_SCHEME_BATCH_SIZE=1export AR_AUTO_SCHEME_SEQLEN=1024```+### AR_AUTO_SCHEME_NO_SERIAL_FALLBACK+- **描述**:将并行评分失败变为硬错误,而不是回退到串行评分。当串行路径已知无法运行(或耗时会是 worker 数量的倍数)时非常有用:已完成的方案和批次会持久化在单方案缓存中,重新运行时只补打失败的部分。+- **默认值**:未设置时,并行评分失败会回退到串行评分+- **有效值**:`1`、`true`、`yes`(不区分大小写)+- **用法**:设置该变量以在并行评分出错时快速失败++```bash+export AR_AUTO_SCHEME_NO_SERIAL_FALLBACK=1+```+### AR_AUTO_SCHEME_CACHE- **描述**:存放可持久复用的 AutoScheme 单方案评分 JSON 文件。该目录独立于用于临时工作数据的 `AR_WORK_SPACE`。- **默认值**:`~/.cache/auto_round`@@ -181,6 +191,16 @@ export AR_AUTO_SCHEME_CACHE=/path/to/auto_scheme_cacheexport AR_ENABLE_AUTO_SCHEME_PARALLEL=0```+### AR_SCHEME_MEM_INVENTORY+- **描述**:启用后,AutoScheme 流式评分会在 block 边界打印当前 CUDA tensor 清单(`[mem-inv]` 行)——按 (shape, dtype) 分组、按大小降序——便于发现未释放的模块权重或被保留的计算图。与该变量无关,打分 worker 的 CUDA OOM 错误始终会附带更详细的清单(活动 tensor、持有它们的容器以及出错算子的 traceback)。+- **默认值**:`False`+- **有效值**:`"1"`、`"true"`、`"yes"`(不区分大小写)表示启用;其他值表示禁用+- **用法**:排查 AutoScheme 打分期间显存增长问题时启用++```bash+export AR_SCHEME_MEM_INVENTORY=1+```+### AR_NVFP4_E5M3_CACHE_HP_WEIGHT- **描述**:控制 `NVFP4E5M3QuantLinear` 是否在首次前向后缓存解量化得到的高精度权重,而不是每次调用都从打包的 FP4 权重重新解量化。- **默认值**:`False`(等价于 `"0"`)diff --git a/test/unit/test_cpu/schemes/test_auto_scheme.py b/test/unit/test_cpu/schemes/test_auto_scheme.pyindex 43b90815d..652affa4a 100644--- a/test/unit/test_cpu/schemes/test_auto_scheme.py+++ b/test/unit/test_cpu/schemes/test_auto_scheme.py@@ -3,8 +3,18 @@ import osimport shutilimport pytest+import torchfrom auto_round import AutoRound, AutoScheme+from auto_round.auto_scheme.delta_loss import (+ AutoSchemeWrapperLinear,+ _annotate_worker_oom,+ _clear_wrapper_score_caches,+ _parallel_scoring_must_raise,+ _replay_retain_graph,+ _tensor_referrer_snippet,+ _vram_inventory_text,+)from auto_round.auto_scheme.utils import _build_layer_config_header_rows, _short_summary_name@@ -457,6 +467,21 @@ class TestAutoScheme:f"cache may have stored merged group scores instead of individual scores")+ def test_mxfp_options_produce_mixed_layer_config(self, tiny_opt_model_path):+ """MX schemes score weight error via the weight-score path, so their+ layer scores must be nonzero and the allocator must mix MXFP4/MXFP8+ (a zeroed score set collapses the config onto the cheapest option)."""+ scheme = AutoScheme(+ avg_bits=7,+ options=("MXFP4", "MXFP8"),+ shared_layers=["q_proj", "k_proj", "v_proj"],+ nsamples=1,+ )+ ar = AutoRound(model=tiny_opt_model_path, scheme=scheme, iters=0, nsamples=1)+ _, config = ar.quantize()+ bits_used = {v["bits"] for v in config.values() if "bits" in v}+ assert bits_used == {4, 8}, f"expected a mixed MXFP4/MXFP8 config, got {sorted(bits_used)}"+def test_different_avg_bits_produces_different_layer_config(self, tiny_opt_model_path):"""Changing avg_bits should change the resulting layer_config."""scheme_low = AutoScheme(@@ -787,12 +812,18 @@ def test_autoscheme_cache_path_is_independent_from_workspace(tmp_path, monkeypatfrom auto_round.auto_scheme.delta_loss import _autoscheme_cache_pathmonkeypatch.setenv("HOME", str(tmp_path / "home"))+ # Windows expanduser prefers USERPROFILE over HOME; patch both so the+ # default-cache assertion holds on every platform (no-op on Linux).+ monkeypatch.setenv("USERPROFILE", str(tmp_path / "home"))monkeypatch.setenv("AR_WORK_SPACE", str(tmp_path / "workspace"))monkeypatch.delenv("AR_AUTO_SCHEME_CACHE", raising=False)cache_path = _autoscheme_cache_path("cache_key", 1)- assert cache_path == str(tmp_path / "home" / ".cache" / "auto_round" / "scheme_01_cache_key.json")+ # normpath: expanduser keeps the forward slashes of "~/.cache" on Windows+ assert os.path.normpath(cache_path) == os.path.normpath(+ str(tmp_path / "home" / ".cache" / "auto_round" / "scheme_01_cache_key.json")+ )def test_autoscheme_cache_path_can_be_overridden(tmp_path, monkeypatch):@@ -884,17 +915,26 @@ def test_parallel_scheme_scoring_supports_disk_stream_model():assert _can_parallel_scheme_scoring(True, "local-model", 1, 2, False, True, False)-def test_parallel_scheme_scoring_rejects_disk_stream_vlm():+def test_parallel_scheme_scoring_allows_text_only_disk_stream_vlm():+ """c34538d2: text-only scoring of a VLM language tower streams blocks like+ a text model; only vision scoring (force_mllm) is excluded from disk+ streaming."""from auto_round.auto_scheme.delta_loss import _can_parallel_scheme_scoring- assert not _can_parallel_scheme_scoring(True, "local-model", 1, 2, False, True, True)+ # is_vlm alone no longer disqualifies disk-stream parallel scoring+ assert _can_parallel_scheme_scoring(True, "local-model", 1, 2, False, True, True)+ # force_mllm (vision scoring needs a full-model backward) still does+ assert not _can_parallel_scheme_scoring(True, "local-model", 1, 2, False, True, False, force_mllm=True)@pytest.mark.parametrize("model_id,is_vlm,low_gpu_mem_usage,expected",[+ # c34538d2: is_vlm no longer disqualifies disk-stream workers -- the+ # language tower streams blocks like a text model; multimodal archs+ # are covered via materialize_non_block_params.("local-model", False, True, True),- ("local-model", True, True, False),+ ("local-model", True, True, True),("local-model", False, False, False),(None, False, True, False),],@@ -935,7 +975,15 @@ def test_disk_stream_scheme_worker_builds_meta_model(monkeypatch):from auto_round.auto_scheme import delta_lossfrom auto_round.utils import disk_stream_util- expected = (object(), object(), object())+ class FakeMetaModel:+ """Minimal module stand-in: the loader structurally unfuses MoE modules+ (named_modules walk) before weights are streamed."""++ def named_modules(self, memo=None):+ return iter(())++ model, tokenizer, disk_index = FakeMetaModel(), object(), object()+ expected = (model, tokenizer, disk_index)monkeypatch.setattr(disk_stream_util, "build_meta_model", lambda model_name: expected)assert delta_loss._load_disk_stream_scheme_worker_model("local-model") == expected@@ -946,7 +994,11 @@ def test_disk_stream_scheme_worker_applies_model_replacements(monkeypatch):from auto_round.auto_scheme import delta_lossfrom auto_round.utils import disk_stream_util- model = object()+ class FakeMetaModel:+ def named_modules(self, memo=None):+ return iter(())++ model = FakeMetaModel()updated_model = object()handled_model = object()tokenizer = object()@@ -1053,3 +1105,386 @@ def test_replacement_wrapper_without_tuning_device_uses_major_device():move_module_to_tuning_device(wrapper, major_device="cpu")assert wrapper.orig_layer.weight.device.type == "cpu"+++def test_env_ar_auto_scheme_no_serial_fallback(monkeypatch):+ import auto_round.envs as envs++ monkeypatch.delenv("AR_AUTO_SCHEME_NO_SERIAL_FALLBACK", raising=False)+ assert envs.AR_AUTO_SCHEME_NO_SERIAL_FALLBACK is False+ monkeypatch.setenv("AR_AUTO_SCHEME_NO_SERIAL_FALLBACK", "1")+ assert envs.AR_AUTO_SCHEME_NO_SERIAL_FALLBACK is True+++def test_parallel_error_must_raise_respects_env_and_lost_worker(monkeypatch):+ import auto_round.envs as envs++ monkeypatch.delenv("AR_AUTO_SCHEME_NO_SERIAL_FALLBACK", raising=False)+ # plain failure, env off -> serial fallback proceeds+ assert not _parallel_scoring_must_raise(RuntimeError("CUDA out of memory"))+ # scheme lost with its worker always raises+ assert _parallel_scoring_must_raise(RuntimeError("scheme 2 (W4A16) was lost with its worker"))+ # env on -> any failure raises+ monkeypatch.setenv("AR_AUTO_SCHEME_NO_SERIAL_FALLBACK", "1")+ assert _parallel_scoring_must_raise(RuntimeError("CUDA out of memory"))+++class TestScoreAnchorWeightScoring:+ """The scoring wrapper quantizes each layer once and routes the score hook+ through a scalar anchor, so block backwards never materialize weight-shaped+ gradients or re-run the quant search."""++ def _make_wrapper(self):+ """Minimal scoring wrapper: a Linear with the attributes _qdq_weight reads."""+ layer = torch.nn.Linear(8, 8, bias=False)+ layer.bits, layer.group_size, layer.sym = 4, 8, True+ layer.scale_dtype = None+ layer.data_type = "int"+ wrapper = AutoSchemeWrapperLinear.__new__(AutoSchemeWrapperLinear)+ torch.nn.Module.__init__(wrapper)+ wrapper.orig_layer = layer+ wrapper.device = "cpu"+ wrapper.data_type = "int"+ wrapper.q_scale_thresh = None+ wrapper.grad_mode = True+ wrapper.need_weight_grad = True+ wrapper.enable_act_quant = False+ wrapper.minmax_scale_bound = (1e-4, 1e3)+ wrapper.weight_min = None+ wrapper.weight_max = None+ from auto_round.wrapper import WrapperLinear++ wrapper.super_qdq_func = WrapperLinear._qdq_weight.__get__(wrapper)+ wrapper.act_score = 0.0+ wrapper.avg_act_score = 0.0+ wrapper.act_cnt = 0.0+ wrapper.weight_score = 0.0+ wrapper.mix_score = 0.0+ wrapper.max_act_value = 0+ wrapper._score_qdq_cpu = None++ def _fake_quant(weight, **kwargs):+ scale = weight.abs().amax().clamp(min=1e-4) / 7.0+ return ((weight / scale).round().clamp(-7, 7) * scale), scale, None++ wrapper.weight_quant_func = _fake_quant+ layer.weight.requires_grad = True+ return wrapper, layer++ def test_qdq_weight_quantizes_once_and_reuses_cache(self):+ wrapper, layer = self._make_wrapper()+ calls = {"n": 0}+ real_quant = wrapper.weight_quant_func++ def counting_quant(weight, **kwargs):+ calls["n"] += 1+ return real_quant(weight, **kwargs)++ wrapper.weight_quant_func = counting_quant+ args = (torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))+ first, _, _ = wrapper._qdq_weight(*args)+ second, _, _ = wrapper._qdq_weight(*args)+ assert calls["n"] == 1+ assert torch.equal(first, second)++ def test_score_anchor_accumulates_expected_weight_score(self):+ torch.manual_seed(0)+ wrapper, layer = self._make_wrapper()+ with torch.no_grad():+ layer.weight.copy_(torch.randn(8, 8) * 0.5)+ x = torch.randn(4, 8)+ qdq_w, _, _ = wrapper._qdq_weight(torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))+ assert qdq_w.requires_grad+ torch.nn.functional.linear(x, qdq_w).sum().backward()+ # dL/d(qdq_w)[j, k] = sum_i x[i, k], so the expected score is+ # sum_{j,k} |x.sum(0)[k]| * |w_diff[j, k]|+ w_diff = layer.weight - wrapper._score_qdq_cpu+ expected = (w_diff.abs().sum(dim=0) * x.sum(dim=0).abs()).sum().item()+ assert wrapper.weight_score > 0+ assert abs(wrapper.weight_score - expected) < 1e-5++ def test_orig_weight_gets_no_grad(self):+ wrapper, layer = self._make_wrapper()+ qdq_w, _, _ = wrapper._qdq_weight(torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))+ torch.nn.functional.linear(torch.randn(2, 8), qdq_w).sum().backward()+ assert layer.weight.grad is None++ def test_capture_mode_returns_fresh_tensor_without_cache(self):+ wrapper, layer = self._make_wrapper()+ wrapper.grad_mode = False+ out, _, _ = wrapper._qdq_weight(torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))+ assert not out.requires_grad+ assert wrapper._score_qdq_cpu is None # one-shot phases must not grow CPU caches++ def test_clear_wrapper_score_caches_drops_block_cache(self):+ import torch.nn as nn++ wrapper, layer = self._make_wrapper()+ block = nn.Sequential(wrapper)+ wrapper._qdq_weight(torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))+ assert wrapper._score_qdq_cpu is not None+ _clear_wrapper_score_caches(block)+ assert wrapper._score_qdq_cpu is None+++class TestOomCensus:+ def test_vram_inventory_text_cpu_only_returns_graceful_string(self):+ text = _vram_inventory_text(top_k=4)+ assert isinstance(text, str)+ assert len(text) > 0++ def test_annotate_worker_oom_embeds_census_and_original(self, monkeypatch):+ monkeypatch.setattr("auto_round.auto_scheme.delta_loss._vram_inventory_text", lambda top_k=12: "CENSUS-LINE")+ out = _annotate_worker_oom(2, RuntimeError("CUDA out of memory. Tried to allocate 12.00 MiB"))+ assert "CENSUS-LINE" in str(out)+ assert "Tried to allocate 12.00 MiB" in str(out)++ def test_non_oom_runtime_errors_are_not_annotated(self, monkeypatch):+ called = {"n": 0}++ def boom(top_k=12):+ called["n"] += 1+ return "CENSUS-LINE"++ monkeypatch.setattr("auto_round.auto_scheme.delta_loss._vram_inventory_text", boom)+ exc = RuntimeError("something else entirely")+ try:+ if "out of memory" not in str(exc).lower():+ raise _annotate_worker_oom(0, exc) if False else exc+ except RuntimeError as caught:+ assert caught is exc+ assert called["n"] == 0+++class TestVramReferrerTrace:+ def test_referrer_snippet_names_container_types(self):+ import torch++ holder = {"box": [torch.zeros(2, 3)]}++ def _find():+ import gc++ for obj in gc.get_objects():+ if torch.is_tensor(obj) and tuple(obj.shape) == (2, 3):+ return obj+ return None++ t = _find()+ assert t is not None+ snippet = _tensor_referrer_snippet(t, max_depth=3)+ assert isinstance(snippet, str)+ assert "list" in snippet or "dict" in snippet+++class TestReferrerAttributeNaming:+ def test_module_referrer_names_the_holding_attribute(self):+ import torch.nn as nn++ class Holder(nn.Module):+ def __init__(self):+ super().__init__()+ self.cache_box = []++ t = torch.zeros(2, 2)+ holder = Holder()+ holder.cache_box.append(t)+ snippet = _tensor_referrer_snippet(t, max_depth=3)+ assert "Holder" in snippet+ assert "cache_box" in snippet+++class TestReplayRetainGraph:+ def test_int_data_type_frees_graph(self):+ import torch.nn as nn++ block = nn.Sequential(nn.Linear(4, 4))+ block[0].data_type = "int"+ assert _replay_retain_graph(block) is False++ def test_mx_family_data_type_retains_graph(self):+ import torch.nn as nn++ block = nn.Sequential(nn.Linear(4, 4))+ block[0].data_type = "mxfp4"+ assert _replay_retain_graph(block) is True++ def test_grads_flow_with_freed_graph(self):+ lin = torch.nn.Linear(4, 4)+ x = torch.randn(2, 4, requires_grad=True)+ out = lin(x).sum()+ torch.autograd.backward(out, retain_graph=False)+ assert x.grad is not None+ assert x.grad.abs().sum() > 0+++class TestScoreLinearRecompute:+ """The scoring forward runs the linear through a custom Function that saves+ only the input activation: the weight is re-transferred from the CPU cache+ in backward, so the block graph never retains weight-shaped copies."""++ def _make_wrapper(self):+ layer = torch.nn.Linear(8, 6, bias=False)+ layer.bits, layer.group_size, layer.sym = 4, 8, True+ layer.scale_dtype = None+ layer.data_type = "int"+ wrapper = AutoSchemeWrapperLinear.__new__(AutoSchemeWrapperLinear)+ torch.nn.Module.__init__(wrapper)+ wrapper.orig_layer = layer+ wrapper.device = "cpu"+ wrapper.output_device = "cpu"+ wrapper.data_type = "int"+ wrapper.q_scale_thresh = None+ wrapper.grad_mode = True+ wrapper.need_weight_grad = True+ wrapper.enable_act_quant = False+ wrapper.enable_norm_bias_tuning = False+ wrapper.minmax_scale_bound = (1e-4, 1e3)+ wrapper.weight_min = None+ wrapper.weight_max = None+ from auto_round.wrapper import WrapperLinear++ wrapper.super_qdq_func = WrapperLinear._qdq_weight.__get__(wrapper)+ wrapper.act_qdq_func = WrapperLinear._qdq_act.__get__(wrapper)+ wrapper.act_score = 0.0+ wrapper.avg_act_score = 0.0+ wrapper.act_cnt = 0.0+ wrapper.weight_score = 0.0+ wrapper.mix_score = 0.0+ wrapper.max_act_value = 0+ wrapper._score_qdq_cpu = None+ wrapper._custom_score_forward = True+ wrapper.orig_forward = torch.nn.functional.linear+ wrapper.value = torch.tensor(0.0)+ wrapper.min_scale = torch.tensor(1.0)+ wrapper.max_scale = torch.tensor(1.0)++ def _fake_quant(weight, **kwargs):+ scale = weight.abs().amax().clamp(min=1e-4) / 7.0+ return (weight / scale).round().clamp(-7, 7) * scale, scale, None++ wrapper.weight_quant_func = _fake_quant+ layer.weight.requires_grad = True+ return wrapper, layer++ def test_forward_matches_reference_linear(self):+ torch.manual_seed(0)+ wrapper, layer = self._make_wrapper()+ with torch.no_grad():+ layer.weight.copy_(torch.randn(6, 8) * 0.5)+ x = torch.randn(4, 8, requires_grad=True)+ out = wrapper.forward(x)+ ref_w, _, _ = wrapper._qdq_weight(torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0))+ ref = torch.nn.functional.linear(x, ref_w.detach())+ assert torch.equal(out, ref)++ def test_input_grads_match_reference(self):+ torch.manual_seed(0)+ wrapper, layer = self._make_wrapper()+ x = torch.randn(4, 8, requires_grad=True)+ out = wrapper.forward(x)+ grad_out = torch.randn_like(out)+ out.backward(grad_out)+ ref_w = wrapper._score_qdq_cpu+ ref_x = x.detach().clone().requires_grad_(True)+ ref = torch.nn.functional.linear(ref_x, ref_w)+ ref.backward(grad_out)+ assert torch.allclose(x.grad, ref_x.grad, atol=1e-6)++ def test_weight_score_accumulated_in_backward(self):+ torch.manual_seed(0)+ wrapper, layer = self._make_wrapper()+ with torch.no_grad():+ layer.weight.copy_(torch.randn(6, 8) * 0.5)+ x = torch.randn(4, 8, requires_grad=True)+ out = wrapper.forward(x)+ grad_out = torch.randn_like(out)+ out.backward(grad_out)+ qdq_w = wrapper._score_qdq_cpu+ grad_w = grad_out.t() @ x.detach()+ expected = (grad_w.abs() * (layer.weight - qdq_w).abs()).sum().item()+ assert wrapper.weight_score > 0+ assert abs(wrapper.weight_score - expected) / expected < 1e-5++ def test_weight_refetched_in_backward_and_no_weight_grad(self):+ wrapper, layer = self._make_wrapper()+ x = torch.randn(4, 8, requires_grad=True)+ wrapper.forward(x).sum().backward()+ assert layer.weight.grad is None+ assert wrapper.weight_score > 0++ def test_capture_mode_uses_base_forward(self):+ torch.manual_seed(1)+ wrapper, layer = self._make_wrapper()+ wrapper.grad_mode = False+ x = torch.randn(4, 8)+ out = wrapper.forward(x)+ assert not out.requires_grad+ assert wrapper._score_qdq_cpu is None+++def test_parallel_scheme_scoring_min_uncached_overrides_single_scheme():+ """A single uncached scheme can still be scored through a worker when the+ serial full-model pass cannot run (weights spanning several GPUs)."""+ from auto_round.auto_scheme.delta_loss import _can_parallel_scheme_scoring++ # default keeps the >=2 heuristic: one scheme stays serial+ assert not _can_parallel_scheme_scoring(True, "local-model", 1, 1, False, True, False)+ # an explicit lower floor admits the single-scheme worker run+ assert _can_parallel_scheme_scoring(True, "local-model", 1, 1, False, True, False, min_uncached=1)+++def test_serial_scoring_device_safe():+ """Serial full-model scoring is device-safe only on single-GPU/CPU models;+ any multi-GPU weight span is unsafe regardless of leftover hook markers+ (partial dispatch / manual placement does not shuttle activations)."""+ import torch+ import torch.nn as nn++ from auto_round.auto_scheme.delta_loss import _serial_scoring_device_safe, _weights_span_multiple_gpus++ class Tiny(nn.Module):+ def __init__(self):+ super().__init__()+ self.fc1 = nn.Linear(4, 4)++ assert _serial_scoring_device_safe(Tiny()) is True++ # a meta-skeleton (disk-stream) model has all params on meta - the live+ # scan sees nothing; the placement plan in hf_device_map is what the+ # serial streaming path materializes blocks onto+ class MetaSkeleton(Tiny):+ hf_device_map = {"": "cuda:0", "model.layers.1": "cuda:3"}++ assert _serial_scoring_device_safe(MetaSkeleton()) is False++ class SingleDeviceMap(Tiny):+ hf_device_map = {"": "cuda:1"}++ assert _serial_scoring_device_safe(SingleDeviceMap()) is True++ # a streamed (meta-skeleton) model with several visible CUDA devices is+ # serial-unsafe: the in-process streaming pass is only validated with a+ # single visible GPU; workers pin one GPU each+ class StreamedSkeleton(Tiny):+ _disk_stream_index = object()++ assert _serial_scoring_device_safe(StreamedSkeleton(), visible_cuda_devices=["cuda:0", "cuda:1"]) is False+ assert _serial_scoring_device_safe(StreamedSkeleton(), visible_cuda_devices=["cuda:0"]) is True+ assert _serial_scoring_device_safe(Tiny(), visible_cuda_devices=["cuda:0", "cuda:1"]) is True++ assert _weights_span_multiple_gpus([torch.device("cuda:0"), torch.device("cuda:0")]) is False+ assert _weights_span_multiple_gpus([torch.device("cuda:0"), torch.device("cuda:3")]) is True+ assert _weights_span_multiple_gpus([torch.device("cpu"), torch.device("cpu")]) is False++ # a multi-GPU span is unsafe even when an _hf_hook marker is present:+ # the marker alone does not guarantee activation transfer under the+ # scoring wrappers+ class FakeHooked(nn.Module):+ def __init__(self):+ super().__init__()+ self._hf_hook = object()++ assert (+ _weights_span_multiple_gpus([torch.device("cuda:0"), torch.device("cuda:1")]) is True+ ) # the unsafe case _serial_scoring_device_safe keys ondiff --git a/test/unit/test_cpu/schemes/test_auto_scheme_disk_stream.py b/test/unit/test_cpu/schemes/test_auto_scheme_disk_stream.pyindex 4205bde95..27d67b6da 100644--- a/test/unit/test_cpu/schemes/test_auto_scheme_disk_stream.py+++ b/test/unit/test_cpu/schemes/test_auto_scheme_disk_stream.py@@ -99,3 +99,30 @@ class TestDiskStreamUtilRoundTrip:for _, tensor in list(block.named_parameters()):assert str(tensor.device) == "meta"assert total_resident_bytes(block) == 0+++class TestDiskStreamWorkerModelPrep:+ def test_disk_stream_build_unfuses_moe_modules(self, monkeypatch):+ """The disk-stream worker build must structurally unfuse fused MoE experts+ (the regular load pipeline does this via custom replacements; the+ meta-skeleton build skips them, leaving per-expert quant layers+ unresolvable)."""+ import auto_round.auto_scheme.delta_loss as dl++ seen = {}+ sentinel_model, sentinel_tokenizer, sentinel_index = object(), object(), object()++ def _fake_build_meta_model(model_name):+ return sentinel_model, sentinel_tokenizer, sentinel_index++ def _fake_handle_moe(model):+ seen["model"] = model+ return []++ monkeypatch.setattr("auto_round.utils.disk_stream_util.build_meta_model", _fake_build_meta_model)+ monkeypatch.setattr("auto_round.modeling.fused_moe.replace_modules._handle_moe_modules", _fake_handle_moe)++ model, _tokenizer, _index = dl._load_disk_stream_scheme_worker_model("dummy-model")++ assert model is sentinel_model+ assert seen["model"] is sentinel_modeldiff --git a/test/unit/test_cpu/utils/test_disk_stream_util.py b/test/unit/test_cpu/utils/test_disk_stream_util.pyindex bfc9949f8..bc20165b3 100644--- a/test/unit/test_cpu/utils/test_disk_stream_util.py+++ b/test/unit/test_cpu/utils/test_disk_stream_util.py@@ -14,6 +14,7 @@"""Unit tests for auto_round.utils.disk_stream_util (AR_DISK_STREAM_MODEL primitives)."""+import pytestimport torchimport torch.nn as nnfrom accelerate import init_empty_weights@@ -120,3 +121,171 @@ class TestBuildMetaModel:# Decoder blocks must still be untouched (meta).for name, tensor in model.model.decoder.layers[0].named_parameters():assert str(tensor.device) == "meta", f"{name} was unexpectedly materialized"++ def test_tied_lm_head_shares_the_embedding_tensor(self, tiny_opt_model_path):+ """A tied lm_head.weight has no checkpoint tensor of its own; the meta+ skeleton must tie it to embed_tokens.weight so materializing the+ embedding makes both real (otherwise it stays a separate meta tensor+ and the first device move crashes)."""+ model, _tokenizer, index = build_meta_model(tiny_opt_model_path)+ if not getattr(model.config, "tie_word_embeddings", False):+ pytest.skip("fixture model does not tie word embeddings")+ assert model.lm_head.weight is model.model.decoder.embed_tokens.weight++ block_names = ["model.decoder.layers.0", "model.decoder.layers.1"]+ materialize_non_block_params(model, block_names, index, device="cpu")+ assert str(model.lm_head.weight.device) != "meta", "tied lm_head stayed on meta"+++class TestCheckpointNameAliases:+ """Model-side names that differ from checkpoint-side tensor names must+ still materialize, resolving through transformers' checkpoint-conversion+ renames (the router stored as ``mlp.router.gate`` behind a ``mlp.gate``+ module, ``shared_mlp`` tensors behind ``shared_experts``)."""++ def _make_index(self, tensors, tmp_path):+ import json++ from auto_round.utils.disk_stream_util import SafetensorsIndex++ checkpoint_dir = tmp_path / "ckpt"+ checkpoint_dir.mkdir(exist_ok=True)+ (checkpoint_dir / "config.json").write_text(json.dumps({"model_type": "hy_v3"}))++ class _AliasIndex(SafetensorsIndex):+ def __init__(self):+ self.checkpoint_dir = checkpoint_dir+ self.weight_map = {name: "dummy" for name in tensors}+ self._tensors = tensors++ def has_tensor(self, name):+ return name in self._tensors++ def read_tensors(self, names, device="cpu"):+ return {n: self._tensors[n].clone() for n in names}++ return _AliasIndex()++ def _make_block(self):+ import torch.nn as nn++ class _Shared(nn.Module):+ def __init__(self):+ super().__init__()+ self.gate_proj = nn.Linear(4, 4, bias=False)+ self.up_proj = nn.Linear(4, 4, bias=False)++ class _Mlp(nn.Module):+ def __init__(self):+ super().__init__()+ self.gate = nn.Linear(4, 4, bias=False)+ self.shared_experts = _Shared()+ self.up_proj = nn.Linear(4, 4, bias=False)++ with init_empty_weights():+ return _Mlp()++ def test_aliased_names_materialize_from_checkpoint_names(self, tmp_path):+ with torch.no_grad():+ tensors = {+ "model.layers.1.mlp.router.gate.weight": torch.randn(4, 4, dtype=torch.float16),+ "model.layers.1.mlp.shared_mlp.gate_proj.weight": torch.randn(4, 4, dtype=torch.float16),+ "model.layers.1.mlp.shared_mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float16),+ }+ block = self._make_block()+ materialize_module(block, "model.layers.1.mlp", self._make_index(tensors, tmp_path), device="cpu")+ assert block.gate.weight.device.type != "meta"+ assert block.shared_experts.gate_proj.weight.device.type != "meta"+ assert block.shared_experts.up_proj.weight.device.type != "meta"++ def test_exact_names_still_resolve_without_alias(self, tmp_path):+ with torch.no_grad():+ tensors = {"model.layers.1.mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float16)}+ block = self._make_block()+ materialize_module(block, "model.layers.1.mlp", self._make_index(tensors, tmp_path), device="cpu")+ assert block.up_proj.weight.device.type != "meta"++ def test_non_block_params_use_aliases_too(self, tmp_path):+ import torch.nn as nn++ class _Top(nn.Module):+ def __init__(self):+ super().__init__()+ self.mlp = nn.Module()+ self.mlp.gate = nn.Linear(4, 4, bias=False)++ with torch.no_grad():+ tensors = {"mlp.router.gate.weight": torch.randn(4, 4, dtype=torch.float16)}+ with init_empty_weights():+ top = _Top()+ materialize_non_block_params(top, ["unrelated.block"], self._make_index(tensors, tmp_path), device="cpu")+ assert top.mlp.gate.weight.device.type != "meta"+++class TestOffloadLoadBlockAliases:+ """The offloader's block loader must resolve checkpoint-conversion renames+ for meta-skeleton blocks (disk-streamed models), matching what the scoring+ streams do -- otherwise aliased tensors (router, shared experts) silently+ stay on meta and crash the tuning loop's device placement."""++ def _write_checkpoint(self, tmp_path, tensors):+ import json++ from safetensors.torch import save_file++ model_dir = tmp_path / "offload_ckpt"+ model_dir.mkdir(exist_ok=True)+ (model_dir / "config.json").write_text(json.dumps({"model_type": "hy_v3"}))+ save_file(tensors, str(model_dir / "model.safetensors"))+ weight_map = {name: "model.safetensors" for name in tensors}+ (model_dir / "model.safetensors.index.json").write_text(json.dumps({"weight_map": weight_map}))+ return str(model_dir)++ def _make_block(self):+ import torch.nn as nn++ class _Shared(nn.Module):+ def __init__(self):+ super().__init__()+ self.gate_proj = nn.Linear(4, 4, bias=False)+ self.up_proj = nn.Linear(4, 4, bias=False)++ class _Mlp(nn.Module):+ def __init__(self):+ super().__init__()+ self.gate = nn.Linear(4, 4, bias=False)+ self.shared_experts = _Shared()+ self.up_proj = nn.Linear(4, 4, bias=False)++ with init_empty_weights():+ return _Mlp()++ def test_aliased_meta_block_materializes(self, tmp_path):+ from auto_round.utils.offload import load_block_from_model_files++ with torch.no_grad():+ tensors = {+ "model.layers.1.mlp.router.gate.weight": torch.randn(4, 4, dtype=torch.float16),+ "model.layers.1.mlp.shared_mlp.gate_proj.weight": torch.randn(4, 4, dtype=torch.float16),+ "model.layers.1.mlp.shared_mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float16),+ "model.layers.1.mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float16),+ }+ model_dir = self._write_checkpoint(tmp_path, tensors)+ block = self._make_block()+ load_block_from_model_files(model_dir, "model.layers.1.mlp", block)+ assert block.gate.weight.device.type != "meta"+ assert block.shared_experts.gate_proj.weight.device.type != "meta"+ assert block.shared_experts.up_proj.weight.device.type != "meta"+ assert block.up_proj.weight.device.type != "meta"+ assert torch.equal(block.gate.weight, tensors["model.layers.1.mlp.router.gate.weight"])++ def test_real_block_still_uses_prefix_loading(self, tmp_path):+ from auto_round.utils.offload import load_block_from_model_files++ with torch.no_grad():+ tensors = {"model.layers.1.mlp.up_proj.weight": torch.randn(4, 4, dtype=torch.float32)}+ model_dir = self._write_checkpoint(tmp_path, tensors)+ block = self._make_block()+ load_block_from_model_files(model_dir, "model.layers.1.mlp", block)+ assert block.up_proj.weight.device.type != "meta"+ assert torch.equal(block.up_proj.weight, tensors["model.layers.1.mlp.up_proj.weight"])