PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat: data-parallel block tuning via --data_parallel (#2351)
avtc6c824f741be3
diff --git a/auto_round/algorithms/block_runner.py b/auto_round/algorithms/block_runner.pyindex 358303600..5c889809f 100644--- a/auto_round/algorithms/block_runner.py+++ b/auto_round/algorithms/block_runner.py@@ -28,6 +28,7 @@ from typing import TYPE_CHECKING, Anyimport torchfrom auto_round.compressors.utils import block_forward+from auto_round.logger import loggerfrom auto_round.utils.device_manager import device_managerif TYPE_CHECKING:@@ -87,6 +88,28 @@ register_block_output("GlmMoeDsaDecoderLayer", ["hidden_states", "prev_topk_indi# TODO wenhuach better follow heng's imp to decouple llm/diffusion+def _cat_device_safe(tensors: list, dim: int) -> torch.Tensor:+ """Concatenate per-sample tensors that may live on different devices.++ Distributed calibration pools keep each sample on its owning DDP device;+ batch selection that crosses owners moves the minority to the first+ piece's device (same-device selections -- the common DDP case -- stay+ copy-free).+ """+ if not tensors:+ raise ValueError("_cat_device_safe: empty selection")+ dev = tensors[0].device+ if any(t.device != dev for t in tensors):+ logger.debug(+ "_cat_device_safe: moved %d/%d pieces to %s",+ sum(1 for t in tensors if t.device != dev),+ len(tensors),+ dev,+ )+ tensors = [t.to(dev) for t in tensors]+ return torch.cat(tensors, dim=dim)++class BlockForwardRunner:"""Stateless block-forward execution engine shared across quantizer & compressor.@@ -191,6 +214,15 @@ class BlockForwardRunner:if indices is not None, this func returns tensor, otherwise listNormalized output tensor on ``cache_device`` (or ``self.cache_device``)."""+ if cache_device is None:+ from auto_round.algorithms.parallel.spmd import current_rank++ rank = current_rank()+ if rank is not None:+ # a data-parallel rank keeps its outputs on its own device: the+ # loss is staged there, and parking them on the shared cache+ # device would copy every rank's output out and back each step+ cache_device = rank.deviceout_device = cache_device if cache_device is not None else self.cache_device# Per-chunk calibration-data placement (--calibration_data_device): list mode (indices is# None -- the cat at the end of indices-mode requires same-device outputs)@@ -305,32 +337,58 @@ class BlockForwardRunner:# ── Private ──────────────────────────────────────────────────────────────+ def _forward_device(self, block) -> torch.device:+ """Staging device for a forward pass through ``block``.++ Normally the runner's global device. The one exception is a block+ that sits WHOLE on a single accelerator device other than the runner+ device -- a DDP mirror replica: staging its inputs to the global+ (primary) device would put the hidden states on a different GPU than+ the replica's weights. Blocks that span several accelerator devices+ (upstream sharded placement, routed by their align hooks) and+ mixed CPU/GPU blocks (CPU-pinned tables) keep the global device, so+ serial behavior is unchanged.+ """+ try:+ devs = {pp.device for pp in block.parameters()}+ except Exception as e: # pragma: no cover - exotic modules+ logger.warning("block device sniff failed (%s); assuming the runner device", e)+ return self.device+ acc = {d for d in devs if d.type in ("cuda", "xpu", "hpu")}+ if len(acc) == 1 and devs <= (acc | {torch.device("cpu")}):+ (d,) = acc+ if d != self.device:+ return d+ return self.device+def _forward_one_batch(self, block, batch_inputs, batch_others) -> Any:"""Forward one already-selected batch through the block (raw output)."""+ # Stage on the block's actual device (see _forward_device). The shared+ # kwargs follow the hidden states so masks/position ids reach the same+ # GPU as the weights; for the serial path both are already there and+ # the moves are no-ops.+ from auto_round.utils.model import to_device++ fwd_device = self._forward_device(block)if isinstance(batch_inputs, dict):batch_inputs = dict(batch_inputs)batch_others = dict(batch_others)hidden_states = batch_inputs.pop("hidden_states")batch_others.update(batch_inputs)- return self.block_forward(- block,- hidden_states,- batch_others,- self.amp,- self.amp_dtype,- self.device,- None,- )else:- return self.block_forward(- block,- batch_inputs,- batch_others,- self.amp,- self.amp_dtype,- self.device,- None,- )+ hidden_states = batch_inputs+ if torch.is_tensor(hidden_states) and hidden_states.device != fwd_device:+ hidden_states = hidden_states.to(fwd_device)+ batch_others = to_device(batch_others, fwd_device)+ return self.block_forward(+ block,+ hidden_states,+ batch_others,+ self.amp,+ self.amp_dtype,+ fwd_device,+ None,+ )def _count_samples(self, inputs: Any) -> int:if isinstance(inputs, dict):@@ -400,6 +458,11 @@ class BlockForwardRunner:"""Select a subset of inputs by indices."""batch_dim = self.batch_dimshared_cache_keys = self.shared_cache_keys+ # Host ints, resolved once. ``indices`` rides on the inputs' device, so a+ # per-element ``int(i)`` read syncs an accelerator once per element;+ # every host-side consumer below draws from this list instead. Device+ # consumers (index_select on the inputs) keep the original tensor.+ idx_list = indices.tolist() if isinstance(indices, torch.Tensor) else list(indices)if isinstance(inputs, dict):selected_inputs = {}@@ -408,13 +471,13 @@ class BlockForwardRunner:if isinstance(val, list) and len(val) == 1:selected_inputs[key] = val[0]elif isinstance(val, list) and len(val) > 1:- idx = int(indices[0]) if len(indices) == 1 else 0+ idx = idx_list[0] if len(idx_list) == 1 else 0selected_inputs[key] = val[idx] if idx < len(val) else val[0]else:selected_inputs[key] = valelse:if isinstance(val, list):- batch_vals = self._gather_same_device([val[i] for i in indices], self.device)+ batch_vals = self._gather_same_device([val[i] for i in idx_list], self.device)selected_inputs[key] = torch.cat(batch_vals, dim=batch_dim)elif isinstance(val, torch.Tensor):# relocate indices like the input_others branch below:@@ -426,35 +489,105 @@ class BlockForwardRunner:selected_inputs[key] = valelse:if isinstance(inputs, list):- batch_vals = self._gather_same_device([inputs[i] for i in indices], self.device)+ batch_vals = self._gather_same_device([inputs[i] for i in idx_list], self.device)selected_inputs = torch.cat(batch_vals, dim=batch_dim)else:selected_inputs = torch.index_select(inputs, batch_dim, indices)selected_others = {"positional_inputs": input_others.get("positional_inputs")}+ def _take_rows(t, rows):+ """Rows of ``t`` on the batch dim. A broadcast (size-1) batch dim+ passes through unsliced, as does a tensor too short for ``rows``+ (with a warning). The range is checked on the host: on CUDA an+ out-of-range index_select is a device-side assert, not an exception."""+ rows = rows.cpu() # host-side bounds check (no-op for host rows)+ size = t.shape[batch_dim] if t.dim() > batch_dim else 0+ if size == 1:+ return t+ if not len(rows) or int(rows.max()) >= size:+ logger.warning_once(+ "kwarg row slicing fell back to the unsliced tensor (batch size %d, rows up to %d); "+ "sub-batch draws may see the wrong batch's entry",+ size,+ int(rows.max()) if len(rows) else -1,+ )+ return t+ return t.index_select(batch_dim, rows.to(device=t.device))++ def _slice_shared_entry(entry, rows):+ """Within-batch row slice of one shared-cache entry (batch pick already done)."""+ if isinstance(entry, torch.Tensor):+ return _take_rows(entry, rows)+ if isinstance(entry, tuple) and entry and all(torch.is_tensor(t) for t in entry):+ return tuple(_take_rows(t, rows) for t in entry)+ return entry+for key, val in input_others.items():if "positional_inputs" in key:continueif key in shared_cache_keys:if isinstance(val, list) and len(val) == 1:selected_others[key] = val[0]+ elif isinstance(val, list) and len(val) > 1 and inputs is not None and len(indices) >= 1:+ # Two cached-list layouts exist: per-SAMPLE (len == pool size)+ # and per-BATCH (len == pool / collection batch; entries are+ # whole collection-batch tensors/tuples). The collection batch+ # comes from the cache itself: a data-parallel rank tunes with+ # its share of the batch, so its own batch_size is smaller.+ _n = self._count_samples(inputs)+ _per_entry = -(-_n // len(val)) if _n > len(val) else 1+ if len(val) == _n and len(indices) > 1:+ # per-sample list: pick the drawn samples' entries+ _picks = [val[i] if i < len(val) else val[0] for i in idx_list]+ if len(_picks) == 1:+ selected_others[key] = _picks[0]+ elif all(isinstance(pp, tuple) and pp and all(torch.is_tensor(t) for t in pp) for pp in _picks):+ selected_others[key] = tuple(+ _cat_device_safe([pp[slot] for pp in _picks], dim=batch_dim)+ for slot in range(len(_picks[0]))+ )+ elif all(torch.is_tensor(pp) for pp in _picks):+ selected_others[key] = _cat_device_safe(_picks, dim=batch_dim)+ else:+ selected_others[key] = val[0]+ elif len(indices) < _per_entry:+ # draw smaller than the collection batch (a rank's share):+ # the owning batch's entry, sliced to the drawn rows+ _b = idx_list[0] // _per_entry+ _entry = val[_b] if 0 <= _b < len(val) else val[0]+ _rows = torch.tensor([i % _per_entry for i in idx_list], dtype=torch.long)+ selected_others[key] = _slice_shared_entry(_entry, _rows)+ else:+ idx = idx_list[0] if len(idx_list) == 1 else 0+ selected_others[key] = val[idx] if idx < len(val) else val[0]elif isinstance(val, list) and len(val) > 1:- idx = int(indices[0]) if len(indices) == 1 else 0+ idx = idx_list[0] if len(idx_list) == 1 else 0selected_others[key] = val[idx] if idx < len(val) else val[0]else:selected_others[key] = valelif isinstance(val, list):- batch_vals = self._gather_same_device([val[i] for i in indices], self.device)+ batch_vals = self._gather_same_device([val[i] for i in idx_list], self.device)if len(batch_vals) == 1:selected_others[key] = batch_vals[0]else:- selected_others[key] = torch.cat(batch_vals, dim=batch_dim)+ selected_others[key] = _cat_device_safe(batch_vals, dim=batch_dim)elif isinstance(val, torch.Tensor):# ``batch_indices`` are created on CPU by the sampler. XPU# (and other accelerator backends) require index tensors on# the same device as the indexed value.selected_others[key] = torch.index_select(val, batch_dim, indices.to(device=val.device))+ elif isinstance(val, tuple) and val and all(torch.is_tensor(t) for t in val):+ # tuple-of-tensors kwargs (transformers v5 passes rope as+ # ``position_embeddings=(cos, sin)`` with a per-sample batch+ # dim). Slice each element like the tensor branch; elements+ # that are shared across samples (e.g. broadcast [1, S, D]+ # tables) fail the select and pass through unsliced, preserving their+ # broadcast behavior. Without this, shard/batch forwards+ # smaller than the cached batch crash in apply_rotary_pos_emb+ # (hidden batch N vs cos batch full).+ _idx = torch.tensor(idx_list, dtype=torch.long)+ selected_others[key] = tuple(_take_rows(t, _idx) for t in val)elif isinstance(val, (str, bool, type(None))):selected_others[key] = valelse:diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.pyindex 98f850100..7d5001dd4 100644--- a/auto_round/algorithms/composer.py+++ b/auto_round/algorithms/composer.py@@ -42,6 +42,11 @@ from auto_round.algorithms.config_resolver import (resolve_shared_config_values,split_quantization_configs,)+from auto_round.algorithms.parallel.data_parallel import (+ ENGINE_CALIBRATION_STATS,+ calibration_merge_stats,+ parallel_state,+)from auto_round.algorithms.utils import _has_nvfp4_layerfrom auto_round.logger import loggerfrom auto_round.utils import clear_memory@@ -360,22 +365,30 @@ class AlgorithmComposer:return register_act_max_hooks(block, getattr(self.block_quantizer.config, "act_data_type", None))- def _get_fp_act_hooks(self, block: torch.nn.Module) -> list:- """Register FP-input act_max + quantizer forward hooks."""+ def _get_fp_act_hooks(self, block: torch.nn.Module) -> tuple[list, dict[str, str] | None]:+ """Register FP-input act_max + quantizer forward hooks.++ Returns ``(handles, merge_stats)``: the shard-merge map of the+ statistics these hooks write (``None``: the pass stays serial, see+ ``calibration_merge_stats``).+ """if not self.need_quanted_input():# If having q_input, act_max will be collected in q_input forward hook,# no need to collect in fp_input forward hookhandles = self._register_act_max_hooks(block)else:handles = []- handles.extend(self.block_quantizer.register_fp_input_forward_hooks(block))- return handles+ quantizer_hooks = self.block_quantizer.register_fp_input_forward_hooks(block)+ handles.extend(quantizer_hooks)+ return handles, calibration_merge_stats(self.block_quantizer, quantizer_hooks)- def _get_q_act_hooks(self, block: torch.nn.Module) -> list:- """Register Q-input act_max + quantizer forward hooks."""+ def _get_q_act_hooks(self, block: torch.nn.Module) -> tuple[list, dict[str, str] | None]:+ """Register Q-input act_max + quantizer forward hooks; returns+ ``(handles, merge_stats)`` like ``_get_fp_act_hooks``."""handles = self._register_act_max_hooks(block)- handles.extend(self.block_quantizer.register_qinput_forward_hooks(block))- return handles+ quantizer_hooks = self.block_quantizer.register_qinput_forward_hooks(block)+ handles.extend(quantizer_hooks)+ return handles, calibration_merge_stats(self.block_quantizer, quantizer_hooks)def _attach_act_max_for_outside_layer(self, layer: torch.nn.Module, fp_inputs, q_inputs) -> None:"""Compute and attach act_max for an outside-block layer from cached inputs.@@ -432,6 +445,62 @@ class AlgorithmComposer:def compress_embedding_layer(self):return self.block_quantizer.quantize_embedding_layer()+ def _collection_context(self, block, fp_inputs):+ """TuneParallelContext for the collection phase (P3): mirror devices+ for sharding the no-grad collection forwards, with the same engagement+ resolver as the tune so the gates can never diverge."""+ from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext++ return TuneParallelContext.for_collection(self, block, fp_inputs)++ def _collect_forward_timed(self, *args, **kwargs):+ """_collect_forward with wall accumulation into self.last_collect_wall."""+ import time as _ctime++ _t0 = _ctime.perf_counter()+ try:+ return self._collect_forward(*args, **kwargs)+ finally:+ self.last_collect_wall = getattr(self, "last_collect_wall", 0.0) + (_ctime.perf_counter() - _t0)++ def _collect_forward(+ self,+ block,+ inputs,+ input_others,+ out_dev=None,+ allow_shard: bool = True,+ hook_pass: bool = False,+ merge_stats=ENGINE_CALIBRATION_STATS,+ ):+ """Collection forward: sharded across DDP mirrors when eligible.++ ``out_dev`` optionally overrides the runner's output cache device.+ ``allow_shard=False`` forces the serial path. ``merge_stats`` is the+ shard-merge map of the statistics the pass's hooks write (from+ ``_get_fp_act_hooks`` / ``_get_q_act_hooks``); ``None`` keeps the+ pass serial (undeclared quantizer statistics).++ ``hook_pass=True`` may cap the concurrent shards (env-driven; default+ no cap): forward hooks+ force dynamo graph breaks, leaving the compiled runner as+ python-bound eager sections that GIL-convoy under many threads.+ """+ if self._coll_ctx is None:+ from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext++ self._coll_ctx = TuneParallelContext()+ return self._coll_ctx.collect_forward(+ self.block_forward,+ block,+ inputs,+ input_others,+ out_dev,+ allow_shard=allow_shard,+ hook_pass=hook_pass,+ merge_stats=merge_stats,+ )+# ── Per-block pipeline orchestration ─────────────────────────────────────def compress_block(@@ -470,8 +539,15 @@ class AlgorithmComposer:``enable_quanted_input`` is ``False``).- *reference_output*: FP reference output collected before optimization."""+ self.last_collect_wall = 0.0block_forward_fn = self.block_forward+ self._coll_ctx = self._collection_context(block, fp_inputs)+ # distributed calibration pool: each DDP device owns its sample+ # shard (matching the tune shards), so shard-local reads stay+ # device-local; serial consumers use device-safe cats+ self._coll_ctx.distribute_pools(fp_inputs, q_inputs)+# ── Step 0: Layer-wise rotation (before any reference/calibration) ────# Each rotation member rotates this block's weights and installs online# hooks so all downstream calibration and reference collection operate on@@ -537,9 +613,18 @@ class AlgorithmComposer:# ── Step 3: Quantizer calibration (act_max, imatrix, etc.) ─────────────if fp_inputs is not None:with torch.no_grad():- quant_hooks = self._get_fp_act_hooks(block)+ quant_hooks, merge_stats = self._get_fp_act_hooks(block)if reference_output is None or reference_invalidated:- reference_output = block_forward_fn(block, fp_inputs, input_others)+ reference_output = self._collect_forward_timed(+ block,+ fp_inputs,+ input_others,+ # the declared statistics (act_max, the quantizer's+ # calibration_stats) are folded from the mirrors, so+ # hook-carrying passes may shard+ hook_pass=bool(quant_hooks),+ merge_stats=merge_stats,+ )reference_next_input = getattr(block_forward_fn, "last_output_dict", None) or reference_outputelse:# No forward happened in this step, so last_output_dict may@@ -549,9 +634,15 @@ class AlgorithmComposer:h.remove()if self.block_quantizer.enable_quanted_input:- q_hooks = self._get_q_act_hooks(block)+ q_hooks, q_merge_stats = self._get_q_act_hooks(block)if q_hooks:- block_forward_fn(block, q_inputs if q_inputs is not None else fp_inputs, input_others)+ self._collect_forward_timed(+ block,+ q_inputs if q_inputs is not None else fp_inputs,+ input_others,+ hook_pass=bool(q_hooks),+ merge_stats=q_merge_stats,+ )for h in q_hooks:h.remove()@@ -588,6 +679,32 @@ class AlgorithmComposer:# lanes without a separate q-input pool (first block; quanted-input# disabled) still get the pre-tuning cache clear the base code hadclear_memory()+ # ── Step 3.6: mirror-pool handoff ──────────────────────────────────────+ # The collection passes may have left a per-block mirror pool alive+ # (built once, reused across the collection forwards). At iters=0 the+ # RTN/OptRTN quantize runs on the home block, the mirrors are synced+ # to it afterwards and the cascade forwards run on them; at iters>0+ # the tune ranks tune the mirrors (spmd.quantize_block_on_ranks).+ from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer+ from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer++ # pool readers: the RTN family (its mirrors are synced to the+ # quantized home block for the sharded cascade) and the SignRound+ # family (its tune ranks run on the mirrors). NOTE: this ensure+ # is only the SAFETY NET for when no collection pass was+ # shard-eligible -- ref-collect sharding builds the pool lazily for+ # EVERY lane and is untouched by this gate.+ _pool_consumer = isinstance(self.block_quantizer, (RTNQuantizer, SignRoundQuantizer))+ if _pool_consumer and getattr(self._coll_ctx, "devices", None) and len(self._coll_ctx.devices) >= 2:+ self._coll_ctx.ensure_pool(block)+ _pool = getattr(self._coll_ctx, "pool", None)+ if _pool is not None:+ # the tune ranks take the resident mirrors (no second weight copy);+ # the RTN family leaves them to _settle_block_pool. A pool nobody+ # consumed is released before the post-quantize steps below.+ _state = parallel_state(self.block_quantizer, create=True)+ _state.pool = _pool+ _state.pool_used = False# ── Step 4: quantize_block ──────────────────────────────────────────────# When quantized input is available from the previous block, use it;# otherwise fall back to the FP input.@@ -600,36 +717,89 @@ class AlgorithmComposer:output = block_forward_fn(wrapped_block, effective_input, input_others)warm_start_outputs.append(getattr(block_forward_fn, "last_output_dict", None) or output)- self._run_tuning(- block,- lambda: self.block_quantizer.quantize_block(+ def tune_block():+ # data-parallel tune: the unchanged quantize_block runs once per+ # rank (one thread per device); otherwise the serial call+ from auto_round.algorithms.parallel.spmd import quantize_block_on_ranks++ engaged, rank0_params = quantize_block_on_ranks(+ self.block_quantizer,+ block,+ effective_input,+ input_others,+ reference_output,+ q_inputs,+ block_ctx,+ input_ids=input_ids,+ )+ if engaged:+ return rank0_params+ return self.block_quantizer.quantize_block(block, effective_input, input_others, reference_output, q_inputs, block_ctx, input_ids=input_ids- ),- collect_warm_start_output,- )+ )- # Quantizer-agnostic block finalization (e.g. NVFP MoE act_max), kept- # out of the per-algorithm tuning loop.- self.block_quantizer.finalize_block(block, block_ctx)+ try:+ self._run_tuning(block, tune_block, collect_warm_start_output)- # ── Step 5: post_quantize_block ─────────────────────────────────────────- for pre in self.preprocessors:- pre.post_quantize_block(block_ctx)-- # ── Step 6: Collect quantized-block outputs for the next block ──────────- if warm_start_outputs:- # Warm-up has restored FP weights by now; use the output collected- # while the wrappers still held the best quantization parameters.- new_q_input = warm_start_outputs[0]- elif self.block_quantizer.enable_quanted_input:- with torch.no_grad():- new_q_input = block_forward_fn(block, effective_input, input_others)- new_q_input = getattr(block_forward_fn, "last_output_dict", None) or new_q_input- else:- new_q_input = None+ # Quantizer-agnostic block finalization (e.g. NVFP MoE act_max), kept+ # out of the per-algorithm tuning loop.+ self.block_quantizer.finalize_block(block, block_ctx)++ # the pool is block-scoped and belongs to the shared tune state:+ # settle it (sync the mirrors for the RTN family, keep or release+ # it) before the post-quantize steps and the Step-6 cascade+ _pool = self._settle_block_pool(block, _pool)++ # ── Step 5: post_quantize_block ─────────────────────────────────────+ for pre in self.preprocessors:+ pre.post_quantize_block(block_ctx)++ # ── Step 6: Collect quantized-block outputs for the next block ──────+ if warm_start_outputs:+ # Warm-up has restored FP weights by now; use the output collected+ # while the wrappers still held the best quantization parameters.+ new_q_input = warm_start_outputs[0]+ elif self.block_quantizer.enable_quanted_input:+ with torch.no_grad():+ new_q_input = self._collect_forward_timed(block, effective_input, input_others)+ new_q_input = getattr(block_forward_fn, "last_output_dict", None) or new_q_input+ else:+ new_q_input = None+ finally:+ # the pool is block-scoped: drop it even on failures so the next+ # block's pool starts clean+ _state = parallel_state(self.block_quantizer)+ if _state is not None:+ _state.pool = None+ self._coll_ctx.release_pool()return new_q_input, reference_next_input+ def _settle_block_pool(self, block, pool):+ """Pool disposition after quantize_block; returns the pool kept for the Step-6 cascade.++ - zero-shot RTN/OptRTN quantized the HOME block through main's search+ lane (which spreads its searches over idle devices itself) -> sync+ the quantized home state onto the mirrors and keep the pool, so the+ cascade still runs sharded;+ - the tune ranks took the mirrors (they end the block in their own+ tuned state while home was unwrapped with the BEST params) -> drop+ the pool; the cascade runs on the canonical home block;+ - unconsumed (declined lane / vanished plan) -> release.+ """+ if pool is None:+ return None+ from auto_round.algorithms.parallel.mirror_sync import sync_mirrors_to_home+ from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer++ if isinstance(self.block_quantizer, RTNQuantizer):+ sync_mirrors_to_home(self.block_quantizer, block)+ _state = parallel_state(self.block_quantizer)+ if _state is None or _state.pool is None or not _state.pool_used:+ self._coll_ctx.release_pool()+ return None+ return pool+def compress_layer_outside_block(self,layer: torch.nn.Module,diff --git a/auto_round/algorithms/parallel/__init__.py b/auto_round/algorithms/parallel/__init__.pynew file mode 100644index 000000000..efef15e6b--- /dev/null+++ b/auto_round/algorithms/parallel/__init__.py@@ -0,0 +1,16 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Engine-level single-process data-parallel machinery (policy, plans,+mirroring, collectives, sharded execution). Algorithm code consumes it+through ``TuneParallelContext``; it owns no algorithm specifics."""diff --git a/auto_round/algorithms/parallel/data_parallel.py b/auto_round/algorithms/parallel/data_parallel.pynew file mode 100644index 000000000..e7e0cb7f4--- /dev/null+++ b/auto_round/algorithms/parallel/data_parallel.py@@ -0,0 +1,1197 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Single-process data parallelism for the SignRound tuning loop.++Runs one block's iteration loop on the home GPU plus W-1 mirror replicas+(persistent deep copies of the wrapped block). Per iteration the global+calibration batch is sharded across replicas; each replica computes its+forward/loss/backward on its own device; gradients are exchanged with a+halving-doubling all-reduce over flat fp32 buffers so every replica ends+with the same averaged gradient; the deterministic SignSGD step is then+applied locally on every replica (identical inputs -> identical update),+which keeps mirrors in sync without any parameter broadcast.++World size 1 (default) executes none of this code.+"""++from __future__ import annotations++import copy+import threading+from collections.abc import Mapping, Sequence+from dataclasses import dataclass, field+from types import MappingProxyType++import torch++from auto_round.logger import logger+++@dataclass+class DDPPlan:+ """Resolved data-parallel plan for one block."""++ world: int+ devices: list[torch.device]+ shard_size: int+ notes: list[str] = field(default_factory=list)++ @property+ def enabled(self) -> bool:+ return self.world > 1+++def _parse_device_token(tok: str, dev_type: str = "cuda") -> torch.device:+ """Parse an explicit-device token: ``3`` -> ``<dev_type>:3``, ``cuda:3``/``xpu:3`` as-is."""+ tok = tok.strip()+ if tok.isdigit():+ return torch.device(dev_type, int(tok))+ return torch.device(tok)+++_SUPPORTED_ACCEL_TYPES = ("cuda", "xpu", "hpu")+++def _accel_device_count(dev_type: str) -> int:+ """Device count for an accelerator backend, 0 when the backend is absent.++ ``getattr(torch, ...)`` can raise during backend module init on builds+ compiled without the backend, so the whole probe is guarded.+ """+ try:+ mod = getattr(torch, dev_type, None)+ fn = getattr(mod, "device_count", None)+ if callable(fn):+ return int(fn())+ except Exception: # pragma: no cover - backend-specific probe failure+ pass+ return 0+++def _accel_free_bytes_map(dev_type: str):+ """Per-device free-memory map for the backend, or None when unavailable.++ cuda/xpu expose ``mem_get_info``; other backends (and missing APIs) return+ None so the plan resolves without VRAM filtering (explicit devices then+ decide the fleet).+ """+ try:+ mod = getattr(torch, dev_type, None)+ if mod is not None and getattr(mod, "is_available", lambda: False)():+ mem = getattr(mod, "mem_get_info", None)+ if callable(mem):+ return {torch.device(dev_type, i): mem(i)[0] for i in range(_accel_device_count(dev_type))}+ except Exception as e: # pragma: no cover - backend-specific probe failure+ logger.warning(+ "[tune-ddp] free-memory probe failed for %s (%s); resolving the plan without VRAM filtering",+ dev_type,+ e,+ )+ return None+++@dataclass(frozen=True)+class ParallelPolicy:+ """Resolved ``--data_parallel`` policy, set once at the entry.++ ``world`` is the requested DDP world (1 = off/serial); ``source`` records+ where it came from ("off" | "auto" | "explicit"); ``collect_forward_cap``+ mirrors AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES (0 = uncapped);+ ``batch_mode`` carries ``--parallel_batch`` ("split" = the global batch+ shards across replicas, serial-equivalent updates; "per_replica" = every+ replica draws the full --batch_size from its own pool shard, torchrun-like+ effective batch = batch_size x world). The policy is set on the+ CompressContext once at the entry; the engine reads it from there.+ """++ world: int = 1+ source: str = "off"+ collect_forward_cap: int = 0+ batch_mode: str = "split"++ @property+ def enabled(self) -> bool:+ return self.world > 1+++def _quantizer_policy(quantizer) -> ParallelPolicy:+ """The run's ParallelPolicy as visible to a quantizer (default: off)."""+ ctx = getattr(quantizer, "compress_context", None)+ policy = getattr(ctx, "parallel_policy", None)+ return policy if isinstance(policy, ParallelPolicy) else ParallelPolicy()+++class ParallelTuneState:+ """Engine-owned parallel state for one quantizer run.++ The quantizer hosts it as ``parallel_state`` (a mount point only -- every+ field is engine state): the cached engagement plan, the block-scoped+ collection mirror pool handed over by the composer, and the flag that a+ lane consumed that pool.+ """++ __slots__ = ("plan", "pool", "pool_used")++ def __init__(self) -> None:+ self.plan: DDPPlan | None = None+ self.pool: MirrorPool | None = None+ self.pool_used: bool = False+++def parallel_state(quantizer, create: bool = False) -> ParallelTuneState | None:+ """The quantizer's engine state holder (created on demand)."""+ st = getattr(quantizer, "parallel_state", None)+ if st is None and create:+ st = ParallelTuneState()+ quantizer.parallel_state = st+ return st+++def resolve_parallel_world(parallel: str, n_visible: int, iters: int = 0) -> int:+ """Resolve the ``--data_parallel`` argument into a DDP world size.++ Policy entry point (called by the CLI): ``off`` -> 1, ``auto`` -> all+ visible devices, ``N`` -> N. The power-of-two rule applies only when the+ tune loop runs (``iters > 0``): the gradient exchange is a recursive+ halving-doubling reduction over the replicas and needs a power-of-two+ rank count. At ``iters == 0`` the loop is empty, every phase that runs+ (sharded collection, searches) is world-count-agnostic, so any device+ count works and ``auto`` takes all visible devices. A requested world is+ a requirement: an invalid value, or more devices requested than are+ visible, raises instead of silently shrinking the request.+ """+ parallel = str(parallel or "off").strip().lower()+ if parallel == "off":+ return 1+ if n_visible < 2:+ raise RuntimeError("--data_parallel auto needs at least 2 visible CUDA devices")+ if parallel == "auto":+ if iters > 0:+ return 1 << (max(n_visible, 1).bit_length() - 1) # largest power of two <= visible+ return n_visible # no exchange at iters=0: every visible device works+ if parallel.isdigit():+ world = int(parallel)+ if world < 2:+ raise RuntimeError(f"--data_parallel world must be at least 2, got {world}")+ if world > n_visible:+ raise RuntimeError(+ f"--data_parallel {world} requires at least {world} visible CUDA devices, got {n_visible}"+ )+ if world & (world - 1) and iters > 0:+ raise RuntimeError(+ f"--data_parallel world must be a power of two when iters > 0 (the gradient "+ f"exchange is a halving-doubling reduction), got {world}"+ )+ return world+ raise RuntimeError(f"--data_parallel accepts off|auto|N, got {parallel!r}")+++def log_zero_shot_parallel_noop_(policy) -> None:+ """ONE INFO line when an explicit ``--data_parallel N`` lands on+ the zero-shot RTN path.++ Plain RTN without calibration (``need_calib == False``) runs+ ``Orchestrator._quantize_zero_shot`` -- no calibration forward, so the+ parallel engine never engages and the request is silently ignored. Emit+ exactly one INFO line for an EXPLICIT request (``auto``/``off`` add no+ user expectation and stay silent). No refusal: weights-only RTN has+ nothing to shard.+ """+ if isinstance(policy, ParallelPolicy) and getattr(policy, "source", "") == "explicit" and policy.world > 1:+ logger.info(+ "[tune-ddp] --data_parallel %d has no effect on this run: "+ "zero-shot RTN needs no calibration forward (weights only); quantizing serially",+ policy.world,+ )+++def check_parallel_device_map_compat_(world: int, device_map, iters: int) -> None:+ """Stop at policy resolution (before model load) when a run cannot parallel-tune.++ A multi-device ``--device_map`` with a live tune loop (``iters > 0``)+ places blocks across several accelerator devices (pipeline-style+ stages); the data-parallel lane mirrors whole blocks on single devices,+ so without this check the run loads the model and fails only at block 0+ in ``resolve_tune_ddp_plan_`` ("block spans N devices; parallel tuning+ requires single-device block placement"). Detect the same condition+ from the device map alone and stop with the same message, before the+ model loads. The runtime per-block check stays as the second line of+ defense (API users, and any map the resolver observes differently).+ Device-map semantics are untouched: a later PP+DP design keeps+ device_map as the per-group pipeline map.+ """+ if world < 2 or iters <= 0 or device_map is None:+ return+ from auto_round.utils.device import parse_available_devices++ try:+ devs = [d for d in parse_available_devices(device_map) if not str(d).startswith("cpu")]+ except Exception: # pragma: no cover - unparsable map: the runtime check decides+ return+ if len(devs) > 1:+ dev_type = str(devs[0]).split(":")[0]+ dev_type = dev_type.upper() if ":" in str(devs[0]) else "CUDA"+ raise RuntimeError(+ f"block spans {len(devs)} {dev_type} devices; parallel tuning requires "+ "single-device block placement (run with no --device_map or a single-device "+ "--device_map; multi-device/auto maps may shard a block across GPUs -- or run "+ "with --data_parallel off)"+ )+++def resolve_ddp_plan(+ world: int,+ home: torch.device,+ batch_size: int,+ visible_devices: Sequence[int] | None = None,+ vram_free_bytes: int | None = None,+ mirror_footprint_bytes: int | None = None,+ margin_bytes: int = 2 * 1024**3,+ pow2_floor: bool = True,+) -> DDPPlan:+ """Pick mirror devices for ``world``-way data parallelism of one block.++ Rules (each adjustment is recorded in ``notes``):+ - world <= 1 or unsupported home -> disabled+ - batch smaller than the world -> demote to the largest world that fits+ the batch: with ``pow2_floor`` (the iters>0 contract) the largest power+ of two that fits; with it off (iters=0, no gradient exchange) the exact+ batch size; a batch of 1 runs serial (logged; a tiny batch is a+ legitimate configuration)+ - home + next devices in ascending visible order+ - visible devices fewer than the requested world -> reported (the+ engagement layer raises)+ - per-mirror VRAM check: advisory only -- a WARNING names devices whose+ free memory is below the estimated mirror footprint; the requested+ world stays intact (a wrong estimate failing the run loudly is+ preferable to silently shrinking it)+ """+ notes: list[str] = []+ if world is None or world <= 1:+ return DDPPlan(1, [home], batch_size, notes)+ if home.type not in _SUPPORTED_ACCEL_TYPES:+ notes.append(f"home device {home} is not a supported accelerator")+ return DDPPlan(1, [home], batch_size, notes)+ world = int(world)+ if batch_size < world:+ # sum-reduced losses make uneven shards exact, so the batch may+ # split unevenly -- with a live tune loop the exchange world itself+ # must stay a power of two (pow2_floor); at iters=0 there is no+ # exchange and the demotion is exact. A replica needs at least one+ # sample; below two usable replicas the lane runs serial (logged; a+ # tiny batch is a legitimate configuration)+ if pow2_floor:+ _w = world+ while _w > 1 and batch_size < _w:+ _w //= 2+ else:+ _w = min(world, max(batch_size, 1))+ if _w < 2:+ logger.info("[tune-ddp] batch %d smaller than any usable world; running serial", batch_size)+ return DDPPlan(1, [home], batch_size, notes)+ if _w < world:+ notes.append(f"batch {batch_size} smaller than world {world}; world set to {_w}")+ logger.info("[tune-ddp] %s", notes[-1])+ world = _w++ if visible_devices:+ order = [torch.device(home.type, i) for i in sorted(visible_devices)]+ else:+ order = [torch.device(home.type, i) for i in range(_accel_device_count(home.type))]++ rotated = [home] + [d for d in order if d != home]+ if len(rotated) < world:+ # the requested world is a requirement: fewer visible devices than+ # requested is reported to the caller (the engagement layer raises);+ # the world stays as asked+ notes.append(f"visible {home.type} devices ({len(rotated)}) fewer than the requested world {world}")+ return DDPPlan(1, [home], batch_size, notes)+ devices = rotated[:world]+ if vram_free_bytes is not None and mirror_footprint_bytes is not None:+ for dev in devices:+ if dev == home:+ continue # the home already holds the block; this check prices mirrors+ free = vram_free_bytes.get(dev, 0) if hasattr(vram_free_bytes, "get") else vram_free_bytes+ if free < mirror_footprint_bytes + margin_bytes:+ notes.append(+ f"low free VRAM on {dev}: free {free / 2**30:.1f}GiB < estimated mirror footprint "+ f"{(mirror_footprint_bytes + margin_bytes) / 2**30:.1f}GiB (estimate only; device kept)"+ )+ logger.warning("[tune-ddp] %s", notes[-1])+ return DDPPlan(world, devices, batch_size // world, notes)+++def _move_tensor(t: torch.Tensor, device: torch.device) -> torch.Tensor:+ """Move a tensor to ``device`` (seam for tests on single-device hosts)."""+ return t.to(device)+++def resolve_tune_ddp_plan_(quantizer, block, fp_inputs, fp_outputs, home, world=None, log: bool = True):+ """Resolve the engaged DDP plan for one block -- shared by quantizer + composer.++ Single source of truth for engagement so the collection pass (composer)+ and the tune (quantizer) resolve identically: both check the same eligibility+ (policy world, accelerator home, no LFQ, list pools, not+ on the torchrun lane), price the same mirror+ footprint, apply the same device selection and the same+ power-of-two gate. The resolved plan is cached on the quantizer's parallel+ state (engine-owned);+ the tune-side caller re-resolves post-wrap (exact wrapper pricing, fresh+ VRAM), so pass ``log=False`` from the pre-wrap collection call to keep the+ engagement/decline lines one-per-block. ``fp_outputs`` may be None when the+ reference outputs are not collected yet (composer entry) -- the tune-side+ caller re-checks the outputs are a list before engaging.++ Returns the DDPPlan (world=1 => not engaged).+ """+ from auto_round.utils.distributed import is_distributed++ _st = parallel_state(quantizer)+ cached = _st.plan if _st is not None else None+ if cached is not None:+ return cached+ if world is None:+ world = _quantizer_policy(quantizer).world+ # the pow2 rule is iters-conditional: the exchange collectives need a+ # power-of-two rank count, and they run only when the tune loop does+ _iters = int(getattr(quantizer, "iters", 0) or 0)+ home = torch.device(home) if not isinstance(home, torch.device) else home+ if home.type in _SUPPORTED_ACCEL_TYPES and home.index is None:+ try:+ _mod = getattr(torch, home.type, None)+ _cur = getattr(_mod, "current_device", None)+ home = torch.device(home.type, int(_cur()) if callable(_cur) else 0)+ except Exception: # backend module init can assert on builds without it+ home = torch.device(home.type, 0)+ decline = []+ # NOTE: engagement itself is not iters-gated -- the DDP world shards+ # the sharded no-grad collection passes at iters=0 exactly as at+ # iters>0 (the tune loop has zero iterations there); only the pow2+ # rule is iters-conditional (see _iters below)+ if world > 1 and home.type not in _SUPPORTED_ACCEL_TYPES:+ decline.append(f"home device {home} is not a supported accelerator ({'/'.join(_SUPPORTED_ACCEL_TYPES)})")+ # eligibility constraints are DECLARED by the quantizer+ # (BaseQuantizer.parallel_constraints); the resolver consumes the+ # declaration+ _constraints_fn = getattr(quantizer, "parallel_constraints", None)+ _constraints = _constraints_fn() if callable(_constraints_fn) else {}+ _lfq_active = bool(_constraints.get("enable_lfq"))+ _home_span_devs = {p.device for p in block.parameters() if p.device.type == home.type}+ eligible = (+ world > 1+ and home.type in _SUPPORTED_ACCEL_TYPES+ and not _lfq_active+ and isinstance(fp_inputs, list)+ and (fp_outputs is None or isinstance(fp_outputs, list))+ and not is_distributed()+ and len(_home_span_devs) <= 1+ )+ if world > 1 and _lfq_active:+ decline.append("enable_lfq")+ if world > 1 and not isinstance(fp_inputs, list):+ decline.append("non-list calibration inputs (diffusion-style pools)")+ if world > 1 and isinstance(fp_inputs, list) and fp_outputs is not None and not isinstance(fp_outputs, list):+ decline.append("non-list reference outputs (diffusion-style pools)")+ # Parallel tuning tunes whole-block mirrors: every replica, including the+ # home copy, must sit on ONE device for the tune. A block whose weights+ # span several devices of the home's accelerator type (multi-device auto+ # placement) is a different topology (pipeline-style stages), so fail+ # visibly: the placement itself must be single-device when parallel+ # tuning runs. CPU-resident tensors are legal alongside the+ # home device (pinned subtrees such as ngram embedding tables stay on CPU+ # by design).+ if world > 1 and len(_home_span_devs) > 1:+ decline.append(+ f"block spans {len(_home_span_devs)} {home.type.upper()} devices; parallel tuning requires "+ "single-device block placement (run with no --device_map or a single-device "+ "--device_map; multi-device/auto maps may shard a block across GPUs -- or run "+ "with --data_parallel off)"+ )+ if world > 1 and is_distributed():+ decline.append("multi-process torchrun lane is active (--data_parallel is single-process)")++ plan = DDPPlan(1, [home], len(fp_inputs) if isinstance(fp_inputs, list) else 0)+ if eligible:+ nsamples = len(fp_inputs)+ batch_size = getattr(getattr(quantizer, "calibration_context", None), "batch_size", nsamples)+ global_batch_size = min(nsamples, batch_size * getattr(quantizer, "gradient_accumulate_steps", 1))+ mirror_bytes = sum(pp.numel() * pp.element_size() for pp in block.parameters()) + sum(+ pp.numel() * pp.element_size()+ for _m in block.modules()+ if hasattr(_m, "orig_layer")+ for pp in _m.params.values()+ )+ # charge the per-forward activation working set on top of the mirror:+ # the existing estimator walks the block's real shapes at the+ # per-forward batch (the serial micro-batch size -- replicas chunk+ # their shards to it), so an activation-heavy block cannot silently+ # pass a weights-only budget+ try:+ from auto_round.utils.device import estimate_tuning_block_mem++ _per_fwd = min(int(batch_size), max(1, global_batch_size // max(1, world)))+ (+ _layer_mem,+ _act_mem,+ _io_mem,+ _add_mem,+ _pd_act,+ _pd_exp,+ ) = estimate_tuning_block_mem(block, fp_inputs, _per_fwd)+ # ``_act_mem`` already sums the per-layer ``output_memory``+ # (grad-doubled) with the MoE ratio applied -- the dict totals+ # are already included. The two per-device maps+ # (``_pd_act``/``_pd_exp``) split the same activation charge by+ # weight home for multi-GPU placement planning; a mirror is+ # single-device by contract, so the aggregate charge is the+ # right price here.+ _act_bytes = int((_act_mem + _io_mem + _add_mem) * 2**30)+ mirror_bytes += _act_bytes+ except Exception as e: # pragma: no cover - estimator is best-effort+ # a failed estimate is a degraded state: the budget silently+ # drops back to weights-only pricing (an activation-heavy block+ # can then pass a plan it does not fit)+ logger.warning("[tune-ddp] activation pricing skipped (estimator failed: %s)", e)+ free = _accel_free_bytes_map(home.type)+ _n_devs = _accel_device_count(home.type)+ plan = resolve_ddp_plan(+ world,+ home,+ global_batch_size,+ visible_devices=list(range(_n_devs)) if _n_devs else None,+ vram_free_bytes=free,+ mirror_footprint_bytes=mirror_bytes,+ pow2_floor=_iters > 0,+ )+ if plan.world > 1:+ # F5: the final peak-VRAM summary only reports devices the+ # memory monitor samples, and its callers pass the device-map+ # list -- replicas engaged outside the map would stay invisible.+ # Register the plan devices once the lane engages (idempotent,+ # per block; best-effort: the monitor must never gate tuning)+ try:+ from auto_round.utils.device import memory_monitor++ memory_monitor.register_extra_devices(plan.devices)+ except Exception: # pragma: no cover - diagnostics must not gate tuning+ pass+ if log:+ global _ENGAGED_LOGGED_SIG+ sig = (plan.world, tuple(str(d) for d in plan.devices), plan.shard_size)+ _changed = sig != _ENGAGED_LOGGED_SIG+ else:+ _changed = False+ if plan.enabled and _iters > 0 and plan.world & (plan.world - 1) != 0:+ decline.append(+ f"resolved world {plan.world} is not a power of two (the gradient exchange needs "+ "one when the tune loop runs)"+ )+ plan = DDPPlan(1, [home], nsamples)+ elif plan.enabled and log and _changed:+ _mode = _quantizer_policy(quantizer).batch_mode+ if _mode == "per_replica":+ # each replica draws the full requested batch (clamped to its+ # pool shard): the effective global batch scales with the world+ _per_draw = max(1, min(global_batch_size, nsamples // max(plan.world, 1)))+ _eff_batch = _per_draw * plan.world+ else:+ _eff_batch = global_batch_size+ logger.info(+ "[tune-ddp] engaged: world=%d, batch_size per device=%d, batch=%s, "+ "effective global batch=%d, devices=%s",+ plan.world,+ plan.shard_size,+ _mode,+ _eff_batch,+ [str(d) for d in plan.devices],+ )+ _ENGAGED_LOGGED_SIG = sig+ if plan.enabled and plan.world > 1:+ # parallel lanes multiply the static shape specializations of the+ # shared compiled quant functions (per-layer v shapes x shard /+ # micro-batch chunk shapes across the replicas' searches), so the+ # dynamo cache limit must scale with the engaged world or the lane+ # silently degrades to eager mid-block once the limit is hit.+ # Monotone raise-only, so the+ # per-block resolve is idempotent and a serial run keeps the+ # plain env default.+ try:+ from auto_round import envs as _cache_envs+ from auto_round.utils.device import _bump_dynamo_cache_limit++ _bump_dynamo_cache_limit(int(_cache_envs.AR_DYNAMO_CACHE_SIZE_LIMIT) * plan.world)+ except Exception: # pragma: no cover - best effort; the tune continues+ pass+ if world > 1 and not plan.enabled:+ reasons = decline + [n for n in plan.notes if n]+ if reasons:+ # A requested parallel world is a hard requirement: continuing+ # on the serial path would silently invalidate the requested+ # configuration and any measurement against it. `auto`+ # resolves to the largest power of two fitting the visible devices+ # at entry and the VRAM check is advisory, so reaching this raise+ # means parallel tuning is genuinely ineligible for this run+ # (an engagement gate or fewer visible devices than the world).+ raise RuntimeError(+ f"parallel tuning with world={world} is ineligible: "+ f"{'; '.join(reasons)}. Fix the blocking condition listed above, or run "+ "with --data_parallel off to tune serial deliberately."+ )+ parallel_state(quantizer, create=True).plan = plan+ return plan+++def _relocate_params(module: torch.nn.Module, device: torch.device) -> None:+ """Move ALL state that ``nn.Module.to()`` cannot see onto ``device``.++ Wrapper modules keep three kinds of non-registered state that a mirrored+ block would otherwise leave on the home device:+ - tunable tensors in the plain ``params`` dict (``v``/``min_scale``/...)+ -- recreated as fresh leaf Parameters (grad state resets, correct for a+ fresh mirror);+ - plain tensor attributes (``weight_min``/``weight_max`` anchors, cached+ imatrix slices, ...) -- moved in place;+ - device-typed attributes (``device``/``output_device``/``tuning_device``)+ -- repointed so wrapper forward staging targets the mirror.+ """+ for _n, m in module.named_modules():+ params = getattr(m, "params", None)+ if isinstance(params, dict):+ for key, val in params.items():+ if isinstance(val, torch.nn.Parameter):+ # move IN PLACE: the dict entry and the registered+ # _parameters entry are the SAME object by construction+ # (wrapper _init_params). Replacing the object here broke+ # that aliasing -- optimizers collected the dict's dead+ # clone while the forward/backward used the registered+ # Parameter, so mirror grads stayed None, sync_grads+ # silently skipped the allreduce, and the home stepped on+ # its own shard's gradient only (quality regressed,+ # monotonically in world size).+ val.data = _move_tensor(val.detach(), device)+ for key, val in list(m.__dict__.items()):+ if isinstance(val, torch.Tensor) and val.device != device and val.device.type != "meta":+ m.__dict__[key] = _move_tensor(val, device)+ elif isinstance(val, torch.device):+ m.__dict__[key] = device+ elif (+ key in ("tuning_device", "device", "output_device")+ and isinstance(val, str)+ and _parse_device_token(val).type == device.type+ ):+ # wrapper staging targets (plain strings -- invisible to the+ # torch.device branch above); same accelerator family only+ m.__dict__[key] = str(device)+++def _param_grad_buffers(params_by_device: list[list[torch.nn.Parameter]]) -> list[torch.Tensor | None]:+ """Flatten each replica's gradients into one contiguous fp32 buffer."""+ bufs: list[torch.Tensor | None] = []+ for params in params_by_device:+ parts = [p.grad.detach().reshape(-1) for p in params if p.grad is not None]+ if not parts:+ bufs.append(None)+ continue+ buf = torch.cat(parts) if len(parts) > 1 else parts[0].clone()+ bufs.append(buf.to(torch.float32) if buf.dtype != torch.float32 else buf)+ return bufs+++def _write_back_grads(buf: torch.Tensor | None, params: list[torch.nn.Parameter]) -> None:+ """Scatter an averaged flat buffer back into ``param.grad`` tensors."""+ if buf is None:+ return+ offset = 0+ for p in params:+ if p.grad is None:+ continue+ n = p.grad.numel()+ p.grad.copy_(buf[offset : offset + n].view_as(p.grad))+ offset += n+++def _transport_segment(seg: torch.Tensor, dev: torch.device) -> torch.Tensor:+ """Move a peer's segment onto ``dev`` (fp32 wire, lossless).++ The wire hop uses non_blocking=True: the allreduce cost is+ payload-independent across dtypes (host-blocking-bound), and cross-device+ copy_ is stream-ordered on both endpoints, so dropping the host block+ lets the exchange chain execute back-to-back on the GPUs without races.+ """+ return seg.to(dev, non_blocking=True)+++def halving_doubling_allreduce(buffers: list[torch.Tensor], scale: float = 1.0) -> None:+ """In-place all-reduce across device-resident buffers (single process).++ Chunked recursive halving-doubling: the flat space is split into W+ chunks; the reduce-scatter phase leaves rank r owning reduced chunk r+ (each step exchanges half of the working block with the partner rank);+ the all-gather phase re-exchanges owned chunks until every rank holds+ the fully reduced buffer. Per-rank traffic is 2*(W-1)/W*bytes, same as a+ ring. Cross-device ``to()`` copies use P2P when available. ``scale`` is+ applied at the end (pass ``1/world`` to average). Requires a+ power-of-two world (the resolver guarantees it).+ """+ world = len(buffers)+ if world < 2:+ if buffers:+ buffers[0].mul_(scale)+ return+ if world & (world - 1):+ raise ValueError(f"halving_doubling_allreduce needs a power-of-two world, got {world}")++ numel = buffers[0].numel()+ chunk = (numel + world - 1) // world++ _reduce_scatter_halving(buffers)+ _allgather_doubling(buffers)++ for buf in buffers:+ buf.mul_(scale)+++def _reduce_scatter_halving(buffers: list[torch.Tensor]) -> None:+ """Recursive-halving reduce-scatter across device-resident buffers.++ The flat space is split into W chunks; the working block per rank halves+ each step (each rank keeps one half, adding the partner's copy of+ it). Rank r ends up owning reduced chunk r; the other+ chunks hold partial garbage. Accumulation stays fp32 on the receiver.+ Requires a power-of-two world.+ """+ world = len(buffers)+ numel = buffers[0].numel()+ chunk = (numel + world - 1) // world+ length = world # chunks in each rank's working block+ while length > 1:+ half = length // 2+ for rank in range(world):+ base = rank - (rank % length) # aligned working-block start+ mid, hi = base + half, base + length+ if rank % length < half:+ partner = rank + half # keep [base, mid)+ seg = buffers[partner][chunk * base : chunk * mid]+ seg = _transport_segment(seg, buffers[rank].device)+ buffers[rank][chunk * base : chunk * mid].add_(seg)+ else:+ partner = rank - half # keep [mid, hi)+ seg = buffers[partner][chunk * mid : chunk * hi]+ seg = _transport_segment(seg, buffers[rank].device)+ buffers[rank][chunk * mid : chunk * hi].add_(seg)+ length = half+++def _allgather_doubling(buffers: list[torch.Tensor]) -> None:+ """Recursive-doubling all-gather of per-rank owned chunks.++ Mirrors the reduce-scatter block structure: the working block per rank+ doubles each step; ranks exchange the chunks the partner owns (copies,+ no adds). Every rank ends holding every chunk -- bitwise identical+ (same-device fp32 copies for gradient buffers, plain int8 copies for+ the sign buffers).+ """+ world = len(buffers)+ numel = buffers[0].numel()+ chunk = (numel + world - 1) // world+ length = 2+ while length <= world:+ half = length // 2+ for rank in range(world):+ base = rank - (rank % length)+ mid, hi = base + half, base + length+ if rank % length < half:+ partner = rank + half+ src = buffers[partner][chunk * mid : chunk * hi]+ dst = buffers[rank][chunk * mid : chunk * hi]+ dst.copy_(_transport_segment(src, dst.device))+ else:+ partner = rank - half+ src = buffers[partner][chunk * base : chunk * mid]+ dst = buffers[rank][chunk * base : chunk * mid]+ dst.copy_(_transport_segment(src, dst.device))+ length *= 2+++_ENGAGED_LOGGED_SIG = None # module-level init for the global read in resolve_tune_ddp_plan_+++# Calibration statistics folded from the mirror copies into the home after a+# sharded collection pass. The engine owns act_max (the composer's act_max+# hooks); a block quantizer declares what its own hooks write in+# ``BaseQuantizer.calibration_stats``, and a pass merges the union (see+# ``calibration_merge_stats``). Associative reductions reproduce the serial+# totals up to fp32 summation order.+ENGINE_CALIBRATION_STATS = MappingProxyType({"act_max": "max"}) # element-wise running max+STAT_REDUCTIONS = ("sum", "max")+++def calibration_merge_stats(quantizer, quantizer_hooks) -> dict[str, str] | None:+ """Shard-merge map for a calibration pass carrying ``quantizer_hooks``.++ The engine's act_max plus the quantizer's declared ``calibration_stats``.+ ``None`` when the quantizer registered hooks without a declaration: the+ engine cannot fold statistics it does not know, so the pass must stay+ serial (a sharded pass would keep only the home shard's samples).+ """+ declared = getattr(quantizer, "calibration_stats", None)+ if quantizer_hooks and declared is None:+ return None+ stats = dict(ENGINE_CALIBRATION_STATS)+ for attr, how in (declared or {}).items():+ if how not in STAT_REDUCTIONS:+ raise ValueError(+ f"{type(quantizer).__name__}.calibration_stats[{attr!r}] = {how!r}: "+ f"supported shard reductions are {STAT_REDUCTIONS}"+ )+ stats[attr] = how+ return stats+++def _merge_mirror_stats(home: torch.nn.Module, mirrors: list[torch.nn.Module], stats) -> None:+ """Fold hook-written statistics from mirror copies back into the home.++ Each mirror forwarded a disjoint sample shard; its hooks accumulated+ into mirror-local module attrs which die with the mirror. ``stats`` maps+ each attribute to its reduction ("sum" | "max"); both are associative, so+ the merged home totals equal the serial pass's totals up to fp32+ summation order (~1e-7 relative) -- far below any quantization-relevant+ scale. Python-scalar statistics (a row count) fold with the same+ semantics, so they always co-travel with their tensor sibling.++ Each mirror value is folded exactly once and then removed from the+ mirror: pool mirrors live across all collection passes of a block, so a+ later pass's merge would otherwise add the same shard values to the home+ again (a hook pass followed by any other sharded pass doubled the+ mirrors' share).+ """+ home_mods = dict(home.named_modules())+ for mirror in mirrors:+ if mirror is None:+ continue+ for name, m_mod in mirror.named_modules():+ h_mod = home_mods.get(name)+ if h_mod is None:+ continue+ for attr, how in stats.items():+ m_val = getattr(m_mod, attr, None)+ if m_val is None:+ continue+ h_val = getattr(h_mod, attr, None)+ if not torch.is_tensor(m_val):+ # scalar stats (imatrix_cnt is a python int): fold with the+ # same copy-or-sum semantics so they always co-travel with+ # their tensor sibling+ if not isinstance(m_val, (int, float)) or isinstance(m_val, bool):+ continue+ if h_val is None:+ setattr(h_mod, attr, m_val)+ else:+ setattr(h_mod, attr, h_val + m_val)+ elif h_val is None:+ setattr(h_mod, attr, m_val.to(h_mod.weight.device if hasattr(h_mod, "weight") else m_val.device))+ else:+ m_val = m_val.to(h_val.device, h_val.dtype)+ if how == "sum":+ setattr(h_mod, attr, h_val + m_val)+ else: # max+ setattr(h_mod, attr, torch.max(h_val, m_val))+ delattr(m_mod, attr) # folded: a later pass must not add it again+++_MODULE_INTERNALS = frozenset(vars(torch.nn.Module()))+++def _stat_signature(val):+ """Change signature of a module attribute a hook may accumulate into:+ tensors by identity + in-place version, containers holding tensors by+ identity + length; other kinds (config scalars, strings, bookkeeping+ such as hook-handle lists) are not tracked."""+ if torch.is_tensor(val):+ return (id(val), val._version)+ if isinstance(val, (list, dict)):+ items = val.values() if isinstance(val, dict) else val+ if any(torch.is_tensor(v) for v in items):+ return (id(val), len(val))+ return None+++def _mirror_attr_snapshot(mirrors) -> list[dict]:+ return [+ {+ (name, attr): _stat_signature(val)+ for name, mod in mirror.named_modules()+ for attr, val in vars(mod).items()+ if attr not in _MODULE_INTERNALS+ }+ for mirror in mirrors+ ]+++def _warn_undeclared_mirror_writes(mirrors, snapshots, merge_stats) -> None:+ """Safety net for the ``calibration_stats`` contract.++ A tensor/container attribute that changed on a mirror module during a+ sharded pass but is not in the merge map dies with the mirror, so the+ home keeps only its own shard's values. Warn once per attribute name.+ """+ for mirror, snap in zip(mirrors, snapshots):+ for name, mod in mirror.named_modules():+ for attr, val in vars(mod).items():+ if attr in _MODULE_INTERNALS or attr in merge_stats:+ continue+ sig = _stat_signature(val)+ if sig is not None and snap.get((name, attr)) != sig:+ logger.warning_once(+ "[tune-ddp] '%s' changed on mirror modules during a sharded collection pass but is not a "+ "declared calibration statistic: if a calibration hook accumulates it, the mirrors' share "+ "is dropped (the home keeps only its own shard). Declare its reduction in the block "+ "quantizer's calibration_stats, or run with --data_parallel off.",+ attr,+ )+++def distribute_pool(pool: list[torch.Tensor], devices: list[torch.device]) -> None:+ """Scatter a per-sample calibration pool across ``devices`` (in place).++ Device r owns its contiguous ceil/floor slice -- the same boundaries the+ DDP tune shards and the sharded collection use (contiguous_shard_bounds;+ a remainder lands on an earlier replica) -- so shard-local reads+ (ref_r build, _select_batch for shard r, per-replica collections) never+ cross devices. Pieces already on their target device are untouched+ (idempotent; a pool produced by the previous block's sharded collection+ on the same group costs nothing). Pools smaller than the world are left+ alone (serial consumers handle them via move-on-demand cats).+ """+ n = len(pool)+ world = len(devices)+ if world < 2 or n < world:+ return+ bounds = contiguous_shard_bounds(n, world)+ for r, dev in enumerate(devices):+ dev = torch.device(dev)+ for i in range(bounds[r], bounds[r + 1]):+ if pool[i].device != dev:+ pool[i] = pool[i].to(dev)+++def contiguous_shard_bounds(n: int, world: int) -> list[int]:+ """Contiguous ceil/floor split bounds over ``n`` items for ``world`` shards.++ The first ``n % world`` shards take one extra item (remainder on the+ EARLIER shards). This exact idiom is a cross-component invariant: pool+ placement, collection shards, tune shards and warm-up windows must all+ slice identically or pools and shards silently misalign -- every site+ must call this helper for the bounds.+ """+ sizes = [n // world + (1 if r < n % world else 0) for r in range(world)]+ bounds = [0]+ for sz in sizes:+ bounds.append(bounds[-1] + sz)+ return bounds+++class MirrorPool:+ """Persistent-per-block ephemeral mirrors, reused across collection phases.++ Builds the same per-device replicas ``sharded_nograd_forward`` builds+ per call (home device keeps the original block; every other device gets a+ relocated deepcopy), but ONCE per block: the collection forwards reuse+ the pool; after a zero-shot RTN/OptRTN quantize the mirrors are synced to+ the quantized home block so the cascade reuses them; the tune ranks tune+ the mirrors (spmd). ``release()`` drops the references; the home block+ stays canonical throughout.+ """++ def __init__(self, block, devices):+ import copy as _copy++ home_dev = _block_device(block)+ self.block = block+ self.devices = list(devices)+ self.reps: list[torch.nn.Module] = []+ self._owned: list[torch.nn.Module] = [] # deepcopies this pool must free+ for dev in self.devices:+ if dev == home_dev:+ self.reps.append(block)+ else:+ m = _copy.deepcopy(block).to(dev)+ _relocate_params(m, dev)+ self.reps.append(m)+ self._owned.append(m)++ @property+ def world(self) -> int:+ return len(self.reps)++ def mirror_layer(self, rep_idx: int, name: str) -> torch.nn.Module:+ for n, m in self.reps[rep_idx].named_modules():+ if n == name:+ return m+ raise KeyError(name)++ def sync_hooks_from(self, block) -> None:+ """Mirror the home block's CURRENT forward hooks onto every replica.++ A fresh-deepcopy pass inherits the hook registry at copy time; a+ persistent pool must re-sync instead, or it would carry hooks removed+ from the home block (stale stats writes) and miss hooks registered+ after the pool was built (missing shard statistics). Hook callables+ are shared by identity -- the same behavior deepcopy provides.+ """+ home_hooks = {}+ for n, m in block.named_modules():+ fwd = getattr(m, "_forward_hooks", None)+ pre = getattr(m, "_forward_pre_hooks", None)+ if fwd or pre:+ fwd_wk = getattr(m, "_forward_hooks_with_kwargs", None)+ pre_wk = getattr(m, "_forward_pre_hooks_with_kwargs", None)+ home_hooks[n] = (dict(fwd or {}), dict(pre or {}), dict(fwd_wk or {}), dict(pre_wk or {}))+ for rep in self.reps:+ if rep is block:+ continue+ for _n, m in rep.named_modules():+ m._forward_hooks.clear()+ m._forward_pre_hooks.clear()+ if hasattr(m, "_forward_hooks_with_kwargs"):+ m._forward_hooks_with_kwargs.clear()+ if hasattr(m, "_forward_pre_hooks_with_kwargs"):+ m._forward_pre_hooks_with_kwargs.clear()+ for n, (fwd, pre, fwd_wk, pre_wk) in home_hooks.items():+ try:+ m = rep.get_submodule(n)+ except AttributeError:+ continue+ for k, h in fwd.items():+ m.register_forward_hook(h, with_kwargs=bool(fwd_wk.get(k, False)))+ for k, h in pre.items():+ m.register_forward_pre_hook(h, with_kwargs=bool(pre_wk.get(k, False)))++ def release(self) -> None:+ self._owned.clear()+ self.reps = []+++def sharded_nograd_forward(+ runner,+ block,+ inputs,+ input_others: dict,+ out_device: torch.device,+ devices: list[torch.device],+ sample_count: int | None = None,+ merge_stats: Mapping[str, str] | None = None,+ max_devices: int = 0,+ pool: MirrorPool | None = None,+):+ """Parallelize a no-grad collection forward across ``devices``.++ ``merge_stats`` (attr -> "sum" | "max") folds the hook-written statistics+ from the mirrors into the home after the pass; ``None`` merges nothing.++ ``max_devices > 0`` truncates the shard set to the first K devices+ (shards grow accordingly): forward hooks force dynamo graph breaks,+ leaving the compiled runner as python-bound eager sections that+ GIL-convoy beyond a handful of threads -- hook-carrying passes are+ capped by the caller while hookless passes shard wide.++ The collection passes (reference outputs, quantized-output cascade) are+ plain forwards over the whole sample pool on one GPU while the mirrors+ idle. Here the pool is split into equal disjoint shards; an ephemeral+ copy of the block on each device forwards its shard in a parallel thread;+ per-shard outputs are parked on ``out_device`` and concatenated in order.+ Bit-identical to the serial pass: rows are sample-independent and the+ module copies carry identical weights. Falls back to the serial runner+ call when the pool is not divisible or fewer than two devices are given.+ Mirrors are dropped (freed) afterwards. Returned pieces stay on the+ device that computed them (distributed pool); consumers either read+ shard-locally or use device-safe cats.+ """+ world = len(devices)+ n = sample_count if sample_count is not None else (len(inputs) if isinstance(inputs, list) else 0)+ if max_devices and 0 < max_devices < world:+ logger.info(+ "[tune-ddp] sharded collect: capping concurrent shards at %d of %d device(s) (hook pass)",+ max_devices,+ world,+ )+ devices = list(devices)[:max_devices]+ world = max_devices+ # ceil/floor contiguous split (same tolerance the search seam applies):+ # any sample count shards -- a remainder lands on an earlier replica and+ # the wall is the largest slice; nothing is dropped or run serially+ world = max(1, min(world, n))+ if len(devices) < 2 or world < 2:+ return runner(block, inputs, input_others, cache_device=out_device)+ devices = list(devices)[:world]+ bounds = contiguous_shard_bounds(n, world)+ shards = [list(range(bounds[r], bounds[r + 1])) for r in range(world)]+ # the input pool typically arrives pooled on the primary device; without+ # re-placement every shard pulls its pieces cross-device from that ONE+ # source, which dominates the shard-forward time. Distribute first+ # (idempotent, same layout the tune engagement+ # enforces later); serial consumers afterwards re-gather to home.+ # Tensor pools only: the helper's contract also admits opaque pools.+ if (+ isinstance(inputs, list)+ and inputs+ and all(isinstance(t, torch.Tensor) for t in inputs)+ and (sample_count is None or sample_count == len(inputs))+ ):+ distribute_pool(inputs, devices)+ import time as _time++ _t_mirrors = _time.perf_counter()+ mirrors: list[torch.nn.Module] = [] # the actual deepcopies used this pass+ if pool is not None and pool.world >= len(devices):+ # capped passes REUSE the full-width pool: take the first K replicas+ # and leave the rest idle (a fresh per-pass deepcopy of the block,+ # plus a second full-width rebuild once the pool was withheld, would+ # cost far more). The unused replicas idle;+ # their stats stay stale for this pass and the home fold below only+ # merges the mirrors that actually ran.+ pool.sync_hooks_from(block)+ reps = pool.reps[: len(devices)]+ mirrors = [m for m in reps if m is not block]+ pool_owned = True+ else:+ home_dev = _block_device(block)+ reps: list[torch.nn.Module] = []+ for dev in devices:+ if dev == home_dev:+ reps.append(block)+ else:+ m = copy.deepcopy(block).to(dev)+ _relocate_params(m, dev)+ reps.append(m)+ mirrors.append(m)+ pool_owned = False+ _t_setup_ms = (_time.perf_counter() - _t_mirrors) * 1000+ parts: list = [None] * world+ fwd_walls = [0.0] * world # per-thread stores at distinct indices: race-free+ evs: list = [None] * world++ def _run(r):+ rep = reps[r]+ dev_r = _block_device(rep)+ t_r = _time.perf_counter()+ ev = None+ if dev_r.type == "cuda":+ ev = (torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True))+ ev[0].record()+ # outputs stay ON the replica that computed them: the per-sample list+ # returned below is a distributed pool (device r owns shard r's rows)+ if dev_r.type == "cuda":+ with torch.cuda.device(dev_r):+ parts[r] = runner(rep, inputs, input_others, shards[r], cache_device=dev_r)+ else:+ parts[r] = runner(rep, inputs, input_others, shards[r], cache_device=dev_r)+ if ev is not None:+ ev[1].record()+ evs[r] = ev+ fwd_walls[r] = (_time.perf_counter() - t_r) * 1000++ # spawn path: collection passes run twice per block, so the per-call+ # thread spawn cost is irrelevant here+ _attr_snaps = _mirror_attr_snapshot(mirrors) if merge_stats is not None else None+ run_threaded_spawn([lambda r=r: _run(r) for r in range(world)])+ # per-shard GPU times: sync the participating devices (their outputs are+ # consumed right after anyway -- the first downstream read syncs too)+ fwd_gpu = [0.0] * world+ for r, dev in enumerate(devices):+ if evs[r] is not None:+ try:+ torch.cuda.synchronize(dev)+ fwd_gpu[r] = evs[r][0].elapsed_time(evs[r][1])+ except RuntimeError as e: # a broken pair stays non-fatal to the pass+ logger.warning_once("[perf] sharded-collect GPU wall unavailable for a pair (%s)", e)+ fwd_gpu[r] = float("nan")+ _t_split = _time.perf_counter()+ # Return the SERIAL structure: a per-sample list ([1, S, H] pieces from+ # split_outputs) -- downstream consumers cat per-sample refs along dim 0+ # (tune loss) and iterate the list for the+ # cascade inputs, so a tensor here changes every per-sample shape.+ pieces: list[torch.Tensor] = []+ split_outputs = getattr(runner, "split_outputs", None)+ for part in parts:+ for piece in split_outputs(part) if split_outputs else torch.split(part, 1, dim=0):+ # clone: torch.split/split_outputs return VIEWS into the batched+ # shard base -- a view-holding pool pins the whole [n, S, H] base+ # on its device for as long as any piece lives, and that base is+ # large at tuning batch sizes. Standalone pieces free the base at+ # pass end.+ pieces.append(piece.clone())+ # threaded shard calls each stamped a PARTIAL last_output_dict (their own+ # shard's rows); the serial text path leaves it unset so callers fall back+ # to the returned list -- clear the residue to keep that contract.+ runner.last_output_dict = None+ _t_merge = _time.perf_counter()+ if merge_stats is not None:+ _merge_mirror_stats(block, mirrors, merge_stats)+ _warn_undeclared_mirror_writes(mirrors, _attr_snaps, merge_stats)+ if not pool_owned:+ mirrors.clear() # drop mirror refs; the caching allocator reclaims them+ reps.clear() # pool-owned reps stay with the pool (later passes reuse them)+ # steady-state breakdown for the collection passes: setup = ephemeral+ # mirror deepcopy/relocate (per pass!), fwd = threaded shard forwards+ # (wall includes enqueue, gpu is the device-side chain incl. the+ # per-thread output parking), split/merge = host assembly. Logged for+ # EVERY pass so first-pass warm-up is separable from steady state.+ _walls = sorted(fwd_walls)+ _gpus = sorted(fwd_gpu)+ logger.debug(+ "[tune-ddp] sharded collect breakdown: world=%d n=%d setup=%.0fms "+ "fwd_wall[min/med/max]=%.0f/%.0f/%.0fms fwd_gpu[min/med/max]=%.0f/%.0f/%.0fms "+ "merge=%.0fms",+ world,+ n,+ _t_setup_ms,+ _walls[0],+ _walls[len(_walls) // 2],+ _walls[-1],+ _gpus[0],+ _gpus[len(_gpus) // 2],+ _gpus[-1],+ (_time.perf_counter() - _t_merge) * 1000,+ )+ return pieces+++def run_threaded_spawn(fns: Sequence) -> None:+ """Run one callable per item in parallel SPAWNED threads (legacy path).++ Exceptions propagate: the first failure (by thread index) is re-raised+ in the joining thread -- a swallowed worker failure would otherwise+ leave missing shard losses/grads and corrupt the step silently.+ """+ errors: dict = {}++ def _guarded(idx, fn):+ try:+ fn()+ except BaseException as exc: # noqa: BLE001 - re-raised below+ errors[idx] = exc++ threads = [threading.Thread(target=_guarded, args=(i, fn)) for i, fn in enumerate(fns)]+ for t in threads:+ t.start()+ for t in threads:+ t.join()+ if errors:+ raise errors[min(errors)]+++def _block_device(block) -> torch.device:+ p = next(block.parameters(), None)+ return p.device if p is not None else torch.device("cpu")diff --git a/auto_round/algorithms/parallel/mirror_sync.py b/auto_round/algorithms/parallel/mirror_sync.pynew file mode 100644index 000000000..3db729f9d--- /dev/null+++ b/auto_round/algorithms/parallel/mirror_sync.py@@ -0,0 +1,154 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Resident mirror-pool synchronization for the data-parallel engine.++The collection passes leave a per-block MirrorPool (one resident block copy+per replica device). This module keeps those mirrors consistent with the HOME+block at phase boundaries: device barriers, home search statistics pushed to+the mirrors before the tune ranks wrap them, and the quantized home state pushed to+the mirrors after a zero-shot RTN/OptRTN quantize so the Step-6 cascade can+still run sharded on them."""++import torch++from auto_round.algorithms.parallel.data_parallel import parallel_state+from auto_round.logger import logger+from auto_round.utils import check_to_quantized+++def sync_mirrors_to_home(quantizer, block) -> bool:+ """Make the block's resident mirrors exactly-home after a home-only quantize.++ Zero-shot RTN/OptRTN quantize the HOME block through main's own search+ lane (which already spreads the searches over idle devices). The+ collection mirrors still serve the sharded Step-6 cascade, so push the+ quantized home state (weight + result attrs) onto every replica, release+ the home search stats and mark the pool consumed. Returns False (nothing+ done) when no pool aligned with this block and plan is resident.+ """+ from auto_round.algorithms.parallel.data_parallel import MirrorPool++ _st = parallel_state(quantizer)+ plan = getattr(_st, "plan", None) if _st is not None else None+ pool = getattr(_st, "pool", None) if _st is not None else None+ if not (+ plan is not None+ and isinstance(pool, MirrorPool)+ and pool.world == plan.world+ and pool.block is block+ and list(pool.devices) == list(plan.devices)+ ):+ return False+ targets = [(n, m) for n, m in block.named_modules() if hasattr(m, "global_name") and check_to_quantized(m)]+ _t_bar = _sync_pool_devices(pool.devices)+ _sync_quantized_to_mirrors(pool, block, targets)+ for _n, m in targets:+ for _attr in ("imatrix", "imatrix_cnt"):+ if hasattr(m, _attr):+ delattr(m, _attr)+ _st.pool_used = True+ _t_bar += _sync_pool_devices(pool.devices)+ logger.debug("[mirror-sync] quantized home state synced to %d mirrors (bar=%.2fs)", pool.world - 1, _t_bar)+ return True+++_POOL_SEARCH_STATS = ("imatrix", "imatrix_cnt", "act_max", "weight_global_scale")+++def _sync_pool_devices(devices) -> float:+ """Synchronize every distinct CUDA pool device; returns elapsed seconds.++ Called at phase boundaries: cross-device copies issued in one+ phase (stats distribution, result placement, mirror sync) complete before+ the next phase touches the same tensors, and a CUDA fault surfaces at+ the boundary of the phase that enqueued it instead of at an unrelated+ later call.+ """+ import time as _ptime++ _t0 = _ptime.perf_counter()+ for d in dict.fromkeys(devices):+ if d.type == "cuda":+ torch.cuda.synchronize(d)+ return _ptime.perf_counter() - _t0+++def _distribute_search_stats(pool, block, targets):+ """Copy the home layers' collected stats onto every mirror.++ The pool mirrors collected their own sample-shard statistics during the+ collection passes; the merged totals live on the home layers (folded by+ ``_merge_mirror_stats``). Tune ranks wrapping the mirrors must see the FULL+ stats, so each mirror layer is overwritten with the home totals first.+ """+ for name, m in targets:+ stats = {a: getattr(m, a) for a in _POOL_SEARCH_STATS if hasattr(m, a)}+ if not stats:+ continue+ for r in range(pool.world):+ rep = pool.reps[r]+ if rep is block:+ continue+ lm = pool.mirror_layer(r, name)+ dev = next(rep.parameters(), None)+ dev = dev.device if dev is not None else torch.device("cpu")+ for a, v in stats.items():+ setattr(lm, a, v.to(dev) if isinstance(v, torch.Tensor) else v)+++_POOL_RESULT_ATTRS = ("scale", "zp", "q_scale_thresh", "data_type")+++def _sync_quantized_to_mirrors(pool, block, targets):+ """Propagate every target's quantized weight + result attrs home -> mirrors.++ The quantize ran on the HOME block only, so without this sync each mirror+ would still hold the FP block -- and the cascade shards SAMPLES across+ replicas. One weight copy per (target, non-home replica) makes every+ mirror carry the exact home state (the cascade contract). Weight-only iters=0+ contract: activation-quantization WrapperWALayer targets are outside this+ lane (structural mirror/home parity is not guaranteed for them).+ """+ # per-MIRROR threads: each replica's copies are independent, so a serial+ # loop pays the sum of all mirrors' copies while threading collapses the+ # wall to the slowest single mirror+ from auto_round.algorithms.parallel.data_parallel import run_threaded_spawn++ def _sync_mirror(r):+ rep = pool.reps[r]+ if rep is block:+ return+ _p0 = next(rep.parameters(), None)+ dev = _p0.device if _p0 is not None else torch.device("cpu")++ def _copy_targets():+ for name, _m in targets:+ home_layer = block.get_submodule(name)+ w = home_layer.weight.data+ attrs = {a: getattr(home_layer, a) for a in _POOL_RESULT_ATTRS if hasattr(home_layer, a)}+ lm = pool.mirror_layer(r, name)+ lm.weight.data.copy_(w.to(dev, non_blocking=True))+ for a, v in attrs.items():+ setattr(lm, a, v.to(dev) if isinstance(v, torch.Tensor) else v)++ if dev.type == "cuda":+ with torch.cuda.device(dev):+ try:+ _copy_targets()+ finally:+ torch.cuda.synchronize(dev)+ else:+ _copy_targets()++ run_threaded_spawn([lambda r=r: _sync_mirror(r) for r in range(pool.world)])diff --git a/auto_round/algorithms/parallel/spmd.py b/auto_round/algorithms/parallel/spmd.pynew file mode 100644index 000000000..926e6a040--- /dev/null+++ b/auto_round/algorithms/parallel/spmd.py@@ -0,0 +1,432 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Thread ranks: a DDP-style wrapper for the per-block tuning loop.++The engine runs the algorithm's UNCHANGED ``quantize_block`` once per rank,+one thread per device, like ``torch.nn.parallel.DistributedDataParallel``+runs an unchanged training loop once per process:++- module: each rank tunes its own replica of the block (rank 0 = the home+ block, the others = resident mirrors);+- dataloader: ``make_index_sampler`` hands each rank a sampler over its+ contiguous shard of the (device-distributed) calibration pool;+- gradients: ``sync_gradients()`` from ``setup_ddp_if_needed_`` averages+ the gradients across the ranks (DDP semantics);+- loss: ``all_reduce_loss`` gives every rank the same global loss, so+ best-iteration selection and early stopping agree on every rank.++The algorithm uses only those three hooks, the same ones the torchrun path+provides. Rank 0's result is the block result; the mirrors are discarded.++The first block of each shape runs each rank's first phase (wrap, optimizer+build and first forward/backward, up to its first collective) serially in+rank order: first-time ``torch.compile`` / triton autotuning is not+thread-safe. Later blocks of the same shape on the same devices find their+kernels compiled and run the ranks concurrently from the start.+"""++from __future__ import annotations++import copy+import dataclasses+import logging+import random+import threading+from collections.abc import Callable+from typing import Any++import torch++from auto_round.algorithms.parallel.data_parallel import (+ MirrorPool,+ _param_grad_buffers,+ _quantizer_policy,+ _write_back_grads,+ contiguous_shard_bounds,+ distribute_pool,+ halving_doubling_allreduce,+ parallel_state,+ resolve_tune_ddp_plan_,+)+from auto_round.algorithms.parallel.mirror_sync import (+ _POOL_SEARCH_STATS,+ _distribute_search_stats,+ _sync_pool_devices,+)+from auto_round.logger import logger++_tls = threading.local()+# (block shape, devices) signatures whose rank kernels are compiled already+_WARM_SIGNATURES: set = set()+_WARM_LOCK = threading.Lock()+++def block_signature(block: torch.nn.Module, devices) -> tuple:+ """What decides whether a block's rank kernels are compiled already."""+ shapes = tuple((n, tuple(p.shape), p.dtype) for n, p in block.named_parameters())+ return (type(block).__name__, shapes, tuple(str(d) for d in devices))+++class RankAborted(RuntimeError):+ """Raised in a rank whose peer failed (the group was aborted)."""+++class RankGroup:+ """In-process collective group for one block's thread ranks."""++ def __init__(self, devices: list[torch.device], serial_first_phase: bool = True) -> None:+ self.devices = [torch.device(d) for d in devices]+ self.world = len(self.devices)+ self._barrier = threading.Barrier(self.world)+ self._slots: list[Any] = [None] * self.world+ self._result: Any = None+ self._turn = threading.Condition()+ # without the serial first phase every rank starts at once+ self.concurrent = not serial_first_phase+ self._next_turn = 0 if serial_first_phase else self.world+ self._released = [False] * self.world+ self._broken = False+ # one shuffle seed per rank, drawn in rank order from the run's RNG+ self.seeds = [random.getrandbits(63) for _ in range(self.world)]++ # ── first-phase turnstile ────────────────────────────────────────────+ def wait_turn(self, rank: int) -> None:+ with self._turn:+ self._turn.wait_for(lambda: self._broken or self._next_turn >= rank)+ if self._broken:+ raise RankAborted("rank group aborted before this rank started")++ def release_turn(self, rank: int) -> None:+ with self._turn:+ if not self._released[rank]:+ self._released[rank] = True+ self._next_turn = max(self._next_turn, rank + 1)+ self._turn.notify_all()++ def abort(self) -> None:+ with self._turn:+ self._broken = True+ self._turn.notify_all()+ self._barrier.abort()++ # ── collectives ──────────────────────────────────────────────────────+ def _wait(self) -> None:+ try:+ self._barrier.wait()+ except threading.BrokenBarrierError as e:+ raise RankAborted("a peer rank failed") from e++ def exchange(self, rank: int, value: Any, reduce_fn: Callable[[list], Any]) -> Any:+ """Post ``value``; rank 0 reduces every rank's value; all get the result."""+ self.release_turn(rank)+ self._slots[rank] = value+ self._wait()+ if rank == 0:+ try:+ self._result = reduce_fn(list(self._slots))+ except BaseException:+ self.abort()+ raise+ self._wait()+ return self._result++ def all_reduce_grads(self, rank: int, module: torch.nn.Module) -> None:+ """Average the gradients of ``module``'s parameters across the ranks."""+ params = [p for p in module.parameters() if p.grad is not None]++ def _reduce(per_rank):+ counts = {len(ps) for ps in per_rank}+ if len(counts) != 1:+ raise RuntimeError(f"ranks hold different gradient sets: {[len(ps) for ps in per_rank]}")+ bufs = _param_grad_buffers(per_rank)+ if any(b is None for b in bufs):+ return+ halving_doubling_allreduce(bufs, scale=1.0 / self.world)+ for buf, ps in zip(bufs, per_rank):+ _write_back_grads(buf, ps)+ return++ self.exchange(rank, params, _reduce)++ def all_reduce_mean(self, rank: int, value: float) -> float:+ """Mean of a python scalar across the ranks (identical on every rank)."""+ return self.exchange(rank, float(value), lambda vals: sum(vals) / len(vals))++ def all_reduce_weighted_mean(self, rank: int, value: float, weight: float) -> float:+ """sum(value * weight) / sum(weight) across the ranks (identical on every rank)."""++ def _reduce(pairs):+ total = sum(w for _v, w in pairs)+ return sum(v * w for v, w in pairs) / total if total else sum(v for v, _w in pairs) / len(pairs)++ return self.exchange(rank, (float(value), float(weight)), _reduce)+++class RankContext:+ """What one rank thread knows about itself."""++ def __init__(self, group: RankGroup, rank: int) -> None:+ self.group = group+ self.rank = rank++ @property+ def world(self) -> int:+ return self.group.world++ @property+ def device(self) -> torch.device:+ return self.group.devices[self.rank]++ def index_sampler(self, nsamples: int, batch_size: int):+ """Sampler over this rank's contiguous shard of the calibration pool."""+ from auto_round.compressors.utils import IndexSampler++ bounds = contiguous_shard_bounds(nsamples, self.world)+ lo, hi = bounds[self.rank], bounds[self.rank + 1]+ return IndexSampler(+ hi - lo,+ min(batch_size, hi - lo),+ indices=list(range(lo, hi)),+ rng=random.Random(self.group.seeds[self.rank]),+ )++ def sync_gradients(self, module: torch.nn.Module) -> None:+ self.group.all_reduce_grads(self.rank, module)++ def all_reduce_mean(self, value: float) -> float:+ return self.group.all_reduce_mean(self.rank, value)++ def all_reduce_weighted_mean(self, value: float, weight: float) -> float:+ return self.group.all_reduce_weighted_mean(self.rank, value, weight)+++def run_rank_sharded_wrap_searches(rank: RankContext, wrappers: list) -> bool:+ """Shard a block's deferred wrap-time searches across the ranks.++ Each rank searches the wrappers it owns (index % world) on its own+ replica, the searched init scales are exchanged, and every rank+ finalizes all its wrappers with them. Returns False (the caller runs+ every search locally) when the ranks are in their serial first phase:+ a collective there would release the compile turnstile early.+ """+ if not rank.group.concurrent:+ return False+ from auto_round.algorithms.quantization import search_dispatch++ world, me = rank.world, rank.rank+ own = [w for i, w in enumerate(wrappers) if i % world == me]+ summary = search_dispatch.SearchSummary()+ if own and not search_dispatch.run_batched_wrap_search(own, summary=summary):+ for w in own:+ with summary.timed(rank.device) as st:+ w._run_deferred_search_now()+ st["singletons"] += 1+ st["modules"] += 1+ summary.finish()+ mine = {i: w.init_scale for i, w in enumerate(wrappers) if i % world == me}++ def _merge(per_rank):+ merged = {}+ for part, _summary in per_rank:+ merged.update(part)+ return merged, [s for _part, s in per_rank]++ # the block's summary line: every rank's share, logged once+ scales, summaries = rank.group.exchange(me, (mine, summary), _merge)+ if me == 0:+ search_dispatch.SearchSummary.merged(summaries).log("[batched-search] wrap search:")+ for i, w in enumerate(wrappers):+ if i % world != me:+ val = scales[i]+ w.finalize_batched_search(val.to(rank.device) if torch.is_tensor(val) else val)+ return True+++def current_rank() -> RankContext | None:+ """The calling thread's rank, or None outside a thread-rank run."""+ return getattr(_tls, "ctx", None)+++class _NonZeroRankFilter(logging.Filter):+ """Rank threads other than rank 0 log warnings and errors only."""++ def filter(self, record: logging.LogRecord) -> bool:+ ctx = current_rank()+ return ctx is None or ctx.rank == 0 or record.levelno >= logging.WARNING+++logger.addFilter(_NonZeroRankFilter())+++def rank_quantizer(quantizer, world: int, split_batch: bool):+ """Shallow per-rank copy of the quantizer.++ The copy shares the model and contexts (read-only during the tune) and+ gets a private calibration context whose batch size is the rank's share+ of the global batch (``split``) or the full batch (``per_replica``).+ The loss is staged on the rank's own device.+ """+ rq = copy.copy(quantizer)+ run_ctx = quantizer._BaseAlgorithm__run_ctx+ cal = copy.copy(run_ctx.calibration_context)+ if split_batch:+ cal.batch_size = max(1, int(cal.batch_size) // world)+ rq._BaseAlgorithm__run_ctx = dataclasses.replace(run_ctx, calibration_context=cal)+ rq._loss_device = None+ rq._card_0_in_high_risk = False+ return rq+++def _decline(reason: str) -> None:+ logger.info("[tune-ranks] declining: %s; serial path", reason)+++def quantize_block_on_ranks(+ quantizer, block, fp_inputs, input_others, fp_outputs, q_inputs, block_ctx, input_ids=None+) -> tuple[bool, Any]:+ """Run ``quantizer.quantize_block`` once per data-parallel rank.++ Returns ``(engaged, rank-0 result)``; ``engaged=False`` means the caller+ runs the serial call. Engagement needs a tuning quantizer (iters > 0),+ an engaged DDP plan, list pools whose size the world divides, and a+ per-rank batch that fits its shard.+ """+ from auto_round.utils.device_manager import device_manager++ if int(getattr(quantizer, "iters", 0) or 0) <= 0:+ return False, None+ enable_q = q_inputs is not None and getattr(quantizer, "enable_quanted_input", True)+ active_inputs = q_inputs if enable_q else fp_inputs+ if not isinstance(active_inputs, list) or not isinstance(fp_outputs, list):+ return False, None++ _st = parallel_state(quantizer, create=True)+ # an engaged plan cached by this block's collection pass is reused:+ # re-resolving would price a second set of mirrors next to the resident+ # pool and warn about VRAM the pool itself occupies. A declined or+ # missing plan resolves here, where its reasons are logged.+ plan = _st.plan+ if plan is None or not plan.enabled:+ _st.plan = None+ plan = resolve_tune_ddp_plan_(quantizer, block, active_inputs, fp_outputs, device_manager.device)+ _st.plan = None # block-scoped: the next block resolves its own+ if not plan.enabled:+ return False, None+ world = plan.world+ nsamples = len(active_inputs)+ policy = _quantizer_policy(quantizer)+ split = str(getattr(policy, "batch_mode", "split") or "split") != "per_replica"+ batch_size = int(quantizer.calibration_context.batch_size)+ acc = int(getattr(quantizer, "gradient_accumulate_steps", 1) or 1)+ if nsamples % world:+ _decline(f"{nsamples} samples do not split evenly over {world} ranks")+ return False, None+ if split and batch_size % world:+ _decline(f"batch size {batch_size} does not split evenly over {world} ranks")+ return False, None+ local_batch = (batch_size // world if split else batch_size) * acc+ if local_batch > nsamples // world:+ _decline(f"per-rank batch {local_batch} exceeds the per-rank shard {nsamples // world}")+ return False, None++ devices = list(plan.devices)+ # data: each device owns its contiguous shard; the rank samplers read+ # only their own shard (make_index_sampler)+ distribute_pool(active_inputs, devices)+ distribute_pool(fp_outputs, devices)++ # module: rank 0 tunes the home block, the others tune mirrors (the+ # collection pool's resident mirrors when aligned, else fresh copies)+ pool = getattr(_st, "pool", None)+ if not (isinstance(pool, MirrorPool) and pool.block is block and list(pool.devices) == devices):+ pool = MirrorPool(block, devices)+ else:+ targets = [(n, m) for n, m in block.named_modules() if any(hasattr(m, a) for a in _POOL_SEARCH_STATS)]+ if targets:+ _distribute_search_stats(pool, block, targets)+ pool.sync_hooks_from(block)+ _sync_pool_devices(devices)+ # the mirrors end the block in their own tuned state: nothing reuses them+ _st.pool = None++ signature = block_signature(block, devices)+ with _WARM_LOCK:+ warm = signature in _WARM_SIGNATURES+ group = RankGroup(devices, serial_first_phase=not warm)+ rank_qs = [rank_quantizer(quantizer, world, split) for _ in range(world)]+ logger.info(+ "[tune-ranks] engaged: world=%d, batch per rank=%d (%s), first phase %s, devices=%s",+ world,+ rank_qs[0].calibration_context.batch_size,+ "split" if split else "per_replica",+ "concurrent" if warm else "serial (compile)",+ [str(d) for d in devices],+ )++ def _rank_fn(r):+ return lambda: rank_qs[r].quantize_block(+ pool.reps[r], fp_inputs, input_others, fp_outputs, q_inputs, block_ctx, input_ids=input_ids+ )++ try:+ results = run_ranks([_rank_fn(r) for r in range(world)], devices, group)+ with _WARM_LOCK:+ _WARM_SIGNATURES.add(signature)+ finally:+ pool.release()+ _st.plan = None+ return True, results[0]+++def run_ranks(rank_fns: list[Callable[[], Any]], devices: list[torch.device], group: RankGroup) -> list[Any]:+ """Run one callable per rank; rank 0 runs in the calling thread.++ Every rank sees its own device as ``device_manager.device`` and as the+ current CUDA device. A failure in any rank aborts the group (peers raise+ RankAborted at their next collective) and the first real error is+ re-raised here.+ """+ from auto_round.algorithms.parallel.tune_parallel import _device_scope+ from auto_round.utils.device_manager import device_manager++ world = len(rank_fns)+ results: list[Any] = [None] * world+ errors: list[BaseException | None] = [None] * world++ def _run(r: int) -> None:+ _tls.ctx = RankContext(group, r)+ try:+ with device_manager.thread_device_scope(devices[r]), _device_scope(devices[r]):+ group.wait_turn(r)+ results[r] = rank_fns[r]()+ except BaseException as e: # noqa: BLE001 - re-raised by the caller below+ errors[r] = e+ group.abort()+ finally:+ group.release_turn(r)+ _tls.ctx = None++ threads = [threading.Thread(target=_run, args=(r,), name=f"ar-rank-{r}", daemon=True) for r in range(1, world)]+ for t in threads:+ t.start()+ _run(0)+ for t in threads:+ t.join()+ real = [e for e in errors if e is not None and not isinstance(e, RankAborted)]+ if real:+ raise real[0]+ first = next((e for e in errors if e is not None), None)+ if first is not None:+ raise first+ return resultsdiff --git a/auto_round/algorithms/parallel/tune_parallel.py b/auto_round/algorithms/parallel/tune_parallel.pynew file mode 100644index 000000000..d4dd983df--- /dev/null+++ b/auto_round/algorithms/parallel/tune_parallel.py@@ -0,0 +1,213 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Composer-owned context for the sharded calibration (collection) forwards.++``TuneParallelContext.for_collection`` resolves the block's devices; the+no-grad collection forwards then shard across the block's mirror pool+(:func:`data_parallel.sharded_nograd_forward`). ``devices is None`` keeps the+serial single-GPU collection. The tune itself runs on thread ranks+(:mod:`spmd`).+"""++import time as _ptime++import torch++from auto_round.algorithms.parallel.data_parallel import (+ ENGINE_CALIBRATION_STATS,+ _quantizer_policy,+ distribute_pool,+ resolve_tune_ddp_plan_,+ sharded_nograd_forward,+)+from auto_round.logger import logger+++class _null_scope:+ """``with``-scope that is a no-op (CPU devices cannot enter torch.cuda.device)."""++ def __enter__(self):+ return None++ def __exit__(self, *exc):+ return False+++def _device_scope(dev: torch.device):+ """``torch.cuda.device(dev)`` for cuda devices, no-op otherwise."""+ if dev.type == "cuda":+ return torch.cuda.device(dev)+ return _null_scope()+++class TuneParallelContext:+ """Collection-phase parallel context; ``devices is None`` means serial."""++ def __init__(self) -> None:+ self.devices: list[torch.device] | None = None+ # collect-forward concurrency cap (from the run's ParallelPolicy;+ # 0 = uncapped) -- see collect_forward(hook_pass=True)+ self.collect_forward_cap: int = 0+ # per-block mirror pool (None until the first sharded collection+ # pass; reused by the collection forwards, the iters=0 cascade and+ # the tune ranks)+ self.pool = None++ @classmethod+ def for_collection(cls, composer, block, fp_inputs) -> "TuneParallelContext":+ """Mirror devices for sharding the no-grad collection forwards.++ Uses the SAME engagement resolver as the tune+ (:func:`data_parallel.resolve_tune_ddp_plan_`, cached on the quantizer)+ so the collection gates stay identical to what the tune later engages+ -- plus two collection-specific gates: the runner must use the+ plain tensor output layout, and the block class must not be a+ multi-output registry entry (sharding clears ``last_output_dict``,+ which those blocks need to feed the next block). ``devices is None``+ keeps the serial single-GPU collection.+ """+ ctx = cls()+ if not isinstance(fp_inputs, list) or not fp_inputs:+ return ctx+ runner_cfg = getattr(composer.block_forward, "output_config", None)+ if runner_cfg and list(runner_cfg) != ["hidden_states"]:+ return ctx # multi-output layouts need the serial last_output_dict path+ from auto_round.algorithms.block_runner import _BLOCK_OUTPUT_REGISTRY++ if len(_BLOCK_OUTPUT_REGISTRY.get(type(block).__name__, ["hidden_states"])) != 1:+ return ctx # registry multi-output blocks (e.g. GlmMoeDsa) need serial collection+ try:+ home = next(block.parameters(), torch.empty(0)).device+ except Exception as e: # pragma: no cover - defensive+ logger.warning("[tune-ddp] collection sharding declined: cannot resolve the block home device (%s)", e)+ return ctx+ try:+ plan = resolve_tune_ddp_plan_(composer.block_quantizer, block, fp_inputs, None, home, log=False)+ except RuntimeError:+ # the resolver raises when a requested world is ineligible+ # (requirement semantics) -- let it propagate: silently continuing+ # with serial collection would invalidate the request+ raise+ except Exception as e: # pragma: no cover - resolver reachability on odd hosts+ logger.warning("[tune-ddp] collection sharding declined: resolver unreachable (%s)", e)+ return ctx+ ctx.devices = plan.devices if plan.world > 1 else None+ ctx.collect_forward_cap = _quantizer_policy(composer.block_quantizer).collect_forward_cap+ return ctx++ def distribute_pools(self, fp_inputs, q_inputs) -> None:+ """Distribute the calibration pool so each DDP device owns its sample+ shard (matching the tune shards); shard-local reads stay device-local;+ serial consumers use device-safe cats."""+ if not self.devices:+ return+ if isinstance(fp_inputs, list) and fp_inputs:+ distribute_pool(fp_inputs, self.devices)+ if isinstance(q_inputs, list) and q_inputs:+ distribute_pool(q_inputs, self.devices)++ def collect_forward(+ self,+ block_forward,+ block,+ inputs,+ input_others,+ out_dev=None,+ allow_shard: bool = True,+ hook_pass: bool = False,+ merge_stats=ENGINE_CALIBRATION_STATS,+ ):+ """Collection forward: sharded across DDP mirrors when eligible.++ ``out_dev`` optionally overrides the runner's output cache device.+ ``allow_shard=False`` forces the serial path.++ ``merge_stats`` maps the statistics the pass's hooks write to their+ shard reduction (``calibration_merge_stats``): hook writes land on the+ mirror copies and are folded back into the home after the pass.+ ``None`` means the block quantizer's hooks write statistics it did+ not declare: the pass stays serial, since sharding would silently+ keep only the home shard's samples.++ ``hook_pass=True`` optionally caps the concurrent shards via the+ policy's ``collect_forward_cap`` (from AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES,+ default 0 = no cap): forward+ hooks force dynamo graph breaks, leaving the compiled runner as+ python-bound eager sections that can GIL-convoy under many threads.+ """+ if self.devices is None or not allow_shard:+ return block_forward(block, inputs, input_others, cache_device=out_dev)+ if merge_stats is None:+ logger.warning_once(+ "[tune-ddp] the block quantizer registers calibration hooks without declaring calibration_stats; "+ "those calibration passes run serially (declare each statistic's shard reduction to shard them)"+ )+ return block_forward(block, inputs, input_others, cache_device=out_dev)+ # The pool is ALWAYS built at the full device width and reused by+ # every sharding-eligible pass; a hook pass capped below the world+ # only limits WHICH pool replicas run -- sharded_nograd_forward slices+ # the replica subset in place.+ world = len(self.devices)+ cap = int(self.collect_forward_cap or 0)+ n = len(inputs) if isinstance(inputs, (list, tuple)) else 0+ pool = None+ if world >= 2 and isinstance(block, torch.nn.Module):+ # ceil/floor split: any n >= world shards (a remainder lands on+ # an earlier replica); n < world runs below the pool's width and+ # stays serial+ if n >= world:+ if self.pool is None:+ from auto_round.algorithms.parallel.data_parallel import MirrorPool++ _t0 = _ptime.perf_counter()+ self.pool = MirrorPool(block, self.devices)+ logger.debug(+ "[tune-ddp] pool build (first sharded pass): world=%d build=%.2fs "+ "(per-device deepcopy + relocate)",+ world,+ _ptime.perf_counter() - _t0,+ )+ pool = self.pool+ return sharded_nograd_forward(+ block_forward,+ block,+ inputs,+ input_others,+ out_dev,+ self.devices,+ merge_stats=merge_stats,+ max_devices=cap if hook_pass else 0,+ pool=pool,+ )++ def ensure_pool(self, block) -> None:+ """Build the per-block mirror pool if absent (block-wide, not tied to+ any single pass's forward eligibility): the searches and the tune+ consume it even when a collection pass fell back serial."""+ if self.pool is None and self.devices and len(self.devices) >= 2:+ from auto_round.algorithms.parallel.data_parallel import MirrorPool++ _t0 = _ptime.perf_counter()+ self.pool = MirrorPool(block, self.devices)+ logger.debug(+ "[tune-ddp] pool build: world=%d build=%.2fs (per-device deepcopy + relocate)",+ len(self.devices),+ _ptime.perf_counter() - _t0,+ )++ def release_pool(self) -> None:+ """Drop the per-block mirror pool (clears the ctx-side references only)."""+ if self.pool is not None:+ self.pool.release()+ self.pool = Nonediff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.pyindex 7832cf2e5..de51df13f 100644--- a/auto_round/algorithms/quantization/base.py+++ b/auto_round/algorithms/quantization/base.py@@ -44,8 +44,18 @@ class BaseQuantizer(BaseAlgorithm):- :meth:`quantize_block` – **must override**: quantize a single block- :meth:`quantize_layer_outside_block` – quantize layers outside blocks- :meth:`finalize_run` – model-level teardown (once after all blocks)+ A quantizer whose calibration hooks write statistics on modules declares+ them in :attr:`calibration_stats`."""+ #: Statistics this quantizer's calibration hooks write on modules, mapped+ #: to their shard reduction ("sum" | "max"). Under --data_parallel+ #: the hook-carrying calibration forwards shard across the block mirrors+ #: and the engine folds these attributes back into the home block.+ #: ``None`` = undeclared: if the quantizer registers calibration hooks,+ #: those passes stay serial. A quantizer without hooks needs no declaration.+ calibration_stats: dict[str, str] | None = None+def __init__(self, config: QuantizationConfig) -> None:super().__init__(config)# Whether to feed quantized-block outputs as inputs to the next block.@@ -142,6 +152,17 @@ class BaseQuantizer(BaseAlgorithm):return is_quantized# ── Abstract quantization interface ───────────────────────────────────────+ def parallel_constraints(self) -> dict:+ """Declare eligibility facts for the single-process data-parallel lane.++ The parallel engine consumes the declaration; algorithm internals stay+ quantizer internals: ``enable_lfq`` reports LFQ layers whose handling is+ single-device. A gradient scaler needs no declaration: every rank keeps+ its own, and the gradients are averaged before ``scaler.step``, so the+ ranks see the same inf/nan checks and keep the same scale.+ """+ return {"enable_lfq": bool(getattr(self, "enable_lfq", False))}+def quantize_block(self,block: "torch.nn.Module",@@ -328,8 +349,14 @@ class BaseQuantizer(BaseAlgorithm):Bulk pulls are a capacity optimization only: when the user explicitlyparked memory on the host (``low_gpu_mem_usage`` or``calibration_data_device="cpu"``) the pool stays where that policy- put it, however much room the target device has.+ put it, however much room the target device has. Under data-parallel+ thread ranks the engine already placed each rank's shard on its device,+ so the pool stays as it is."""+ from auto_round.algorithms.parallel.spmd import current_rank++ if current_rank() is not None:+ return poolcc = self.compress_contextif (getattr(cc, "low_gpu_mem_usage", False)diff --git a/auto_round/algorithms/quantization/rtn/batched_search.py b/auto_round/algorithms/quantization/rtn/batched_search.pyindex 666d3b40c..250423052 100644--- a/auto_round/algorithms/quantization/rtn/batched_search.py+++ b/auto_round/algorithms/quantization/rtn/batched_search.py@@ -32,6 +32,7 @@ from typing import Anyimport torchfrom auto_round.algorithms.quantization.search_dispatch import (+ SearchSummary,_fn_key,_plan_capacity_chunks,_search_fn_stackable,@@ -43,7 +44,6 @@ from auto_round.algorithms.quantization.search_dispatch import ()from auto_round.logger import loggerfrom auto_round.utils import set_module-from auto_round.utils.perf_counters import perf_counters_enabledfrom auto_round.wrapper import WrapperLinear@@ -313,8 +313,9 @@ def run_batched_rtn_search(for worker, _home, chunk in planned:buckets.setdefault(str(worker), []).append(chunk)- # per-worker execution stats for the wrap-grammar summary line- stats: OrderedDict[str, dict] = OrderedDict()+ # per-worker calls and search time for the summary line+ summary = SearchSummary()+ summary.offloaded = sum(1 for worker, home, _chunk in planned if worker != home)def _worker_of(chunk: list[dict[str, Any]]) -> str:for wk, cs in buckets.items():@@ -346,7 +347,7 @@ def run_batched_rtn_search(@torch.no_grad()def _run_chunk(chunk: list[dict[str, Any]], worker: str, threaded: bool = False) -> None:- st = stats.setdefault(str(worker), {"wall": 0.0, "modules": 0, "batches": 0, "singletons": 0})+ st = summary.device(worker)st["modules"] += len(chunk)if len(chunk) > 1:st["batches"] += 1@@ -450,16 +451,9 @@ def run_batched_rtn_search(set_module(model, e["name"], layer)def _run_chunks(wk: str, cs: list[list[dict[str, Any]]]) -> None:- import time as _ptime-- _t0 = _ptime.perf_counter()- try:+ with summary.timed(wk):for c in cs:_run_chunk_guarded(c, _worker_of(c), threaded=True)- finally:- stats.setdefault(str(wk), {"wall": 0.0, "modules": 0, "batches": 0, "singletons": 0})["wall"] += (- _ptime.perf_counter() - _t0- )if len(buckets) > 1:keyed = OrderedDict()@@ -470,31 +464,8 @@ def run_batched_rtn_search(for wk, cs in buckets.items():_run_chunks(str(wk), cs)- if stats and perf_counters_enabled():- # one timing line, same grammar and gating as the wrap-search lane:- # calls are chunks (each chunk is one stacked or per-module search- # call), so the counts cannot read as "one call per module"- from auto_round.utils.pool_placement import _short_device_key-- _n_mod = sum(st["modules"] for st in stats.values())- _n_batch = sum(st["batches"] for st in stats.values())- _n_single = sum(st["singletons"] for st in stats.values())- _n_offloaded = sum(1 for worker, home, _chunk in planned if worker != home)- _walls = sum(st.get("wall", 0.0) for st in stats.values())- per_device = (- "{" + ", ".join(f"'{_short_device_key(dev)}': {st.get('wall', 0.0):.2f}" for dev, st in stats.items()) + "}"- )- logger.info(- "[rtn-batch] %d modules, %d batch + %d singleton search calls"- + (f" ({_n_offloaded} chunk(s) offloaded to idle devices)" if _n_offloaded else "")- + ", %.2fs device-wall (%.2fs summed) walls %s",- _n_mod,- _n_batch,- _n_single,- max((st.get("wall", 0.0) for st in stats.values()), default=0.0),- _walls,- per_device,- )+ if summary.devices:+ summary.finish().log("[rtn-batch]")def _relocate_tensor_kwargs(kwargs: dict[str, Any], device: str) -> dict[str, Any]:diff --git a/auto_round/algorithms/quantization/rtn/quantizer.py b/auto_round/algorithms/quantization/rtn/quantizer.pyindex 6eaf3e3d9..2e0e1275e 100644--- a/auto_round/algorithms/quantization/rtn/quantizer.py+++ b/auto_round/algorithms/quantization/rtn/quantizer.py@@ -83,6 +83,11 @@ class RTNQuantizer(BaseQuantizer):@register_pipeline_member(OptimizedRTNConfig)class OptimizedRTNQuantizer(RTNQuantizer):+ # imatrix_cnt (a python int row count) must co-travel with imatrix: a cold+ # expert can collect zero routed rows on the home shard and receive only+ # the merged tensor, and the normalize step (imatrix /= imatrix_cnt) needs both+ calibration_stats = {"imatrix": "sum", "imatrix_cnt": "sum"}+def __init__(self, config: RTNConfig) -> None:super().__init__(config)diff --git a/auto_round/algorithms/quantization/search_dispatch.py b/auto_round/algorithms/quantization/search_dispatch.pyindex 3aa9c2013..26727a1fd 100644--- a/auto_round/algorithms/quantization/search_dispatch.py+++ b/auto_round/algorithms/quantization/search_dispatch.py@@ -184,6 +184,12 @@ def pick_search_worker_devices("""if cross_device_search_disabled():return [home_device] if home_device is not None else []+ from auto_round.algorithms.parallel.spmd import current_rank++ if current_rank() is not None:+ # a data-parallel rank owns only its device: the "idle" ones are the+ # other ranks' devices, busy with their own searches and tune steps+ return [home_device] if home_device is not None else []home_type = str(home_device).split(":")[0] if home_device else ""if home_type == "hpu" and is_hpu_lazy_mode():return [home_device] if home_device is not None else []@@ -223,6 +229,70 @@ def log_engaged_once(label: str) -> None:logger.info("[batched-search] %s: running weight-local searches with one worker per device", label)+class SearchSummary:+ """Calls and per-device search time of one search phase, logged as one line.++ ``walls`` sums the time each device spent searching -- stacked and+ per-module calls alike; ``device-wall`` is the elapsed time of the phase.+ Summaries of concurrent phases (data-parallel ranks) merge into one line.+ """++ def __init__(self) -> None:+ self._t0 = time.perf_counter()+ self.elapsed = 0.0+ self.offloaded = 0+ self.devices: OrderedDict[str, dict] = OrderedDict()++ def device(self, key) -> dict:+ return self.devices.setdefault(str(key), {"wall": 0.0, "modules": 0, "batches": 0, "singletons": 0})++ @contextlib.contextmanager+ def timed(self, key) -> Iterator[dict]:+ """Charge the enclosed search time to device ``key``; yields its counters."""+ st = self.device(key)+ t0 = time.perf_counter()+ try:+ yield st+ finally:+ st["wall"] += time.perf_counter() - t0++ def finish(self) -> "SearchSummary":+ self.elapsed = time.perf_counter() - self._t0+ return self++ @classmethod+ def merged(cls, parts: "list[SearchSummary]") -> "SearchSummary":+ """One summary for concurrent parts: device counters add up, the elapsed time is the slowest part's."""+ out = cls()+ out.elapsed = max((p.elapsed for p in parts), default=0.0)+ for p in parts:+ out.offloaded += p.offloaded+ for key, st in p.devices.items():+ dst = out.device(key)+ for k, v in st.items():+ dst[k] += v+ return out++ def log(self, label: str) -> None:+ if not perf_counters_enabled():+ return+ from auto_round.utils.pool_placement import _short_device_key++ sts = list(self.devices.values())+ logger.info(+ "%s %d modules, %d batch + %d singleton search calls"+ + (f" ({self.offloaded} chunk(s) offloaded to idle devices)" if self.offloaded else "")+ + ", %.2fs device-wall (%.2fs summed) walls %s",+ label,+ sum(st["modules"] for st in sts),+ sum(st["batches"] for st in sts),+ sum(st["singletons"] for st in sts),+ self.elapsed,+ sum(st["wall"] for st in sts),+ "{" + ", ".join(f"'{_short_device_key(k)}': {st['wall']:.2f}" for k, st in self.devices.items()) + "}",+ )++def batched_search_disabled() -> bool:"""Kill switch for the batched search machinery (stacking + per-device workers)."""return bool(envs.AR_DISABLE_BATCHED_SEARCH)@@ -433,7 +503,9 @@ def _plan_capacity_chunks(entries: list) -> list:return plans-def run_batched_wrap_search(deferred_wrappers: list[WrapperLinear], max_batch: int | None = None) -> bool:+def run_batched_wrap_search(+ deferred_wrappers: list[WrapperLinear], max_batch: int | None = None, summary: "SearchSummary | None" = None+) -> bool:"""Run deferred weight-local wrap searches on stacked same-shape batches.Wrappers stage ``(weight_reshape, data_type, bits, imatrix, q_scale_thresh,@@ -456,11 +528,16 @@ def run_batched_wrap_search(deferred_wrappers: list[WrapperLinear], max_batch: iReturns True when the inputs were consumed; False when batching is disabledby AR_DISABLE_BATCHED_SEARCH (the caller then runs the searches per module).+ A caller-supplied ``summary`` collects the calls and times instead of the+ summary line (a data-parallel rank searches a share of the block)."""if batched_search_disabled():return Falseif not deferred_wrappers:return False+ _log_summary = summary is None+ if summary is None:+ summary = SearchSummary()device_groups = OrderedDict()for w in deferred_wrappers:@@ -511,7 +588,7 @@ def run_batched_wrap_search(deferred_wrappers: list[WrapperLinear], max_batch: ifor worker, home, chunk in planned:buckets.setdefault(worker, []).append((home, chunk))- stats = {wk: {"modules": 0, "batches": 0, "singletons": 0} for wk in buckets}+ summary.offloaded += sum(1 for worker, home, _chunk in planned if worker != home)@torch.no_grad()def _run_one(wrapper: WrapperLinear) -> None:@@ -575,7 +652,9 @@ def run_batched_wrap_search(deferred_wrappers: list[WrapperLinear], max_batch: idump_oom_tensor_census_("wrap search")for w in chunk:w._run_deferred_search_now()- stats[worker_key]["singletons"] += len(chunk)+ st = summary.device(worker_key)+ st["singletons"] += len(chunk)+ st["modules"] += len(chunk)returnif worker_key != home_key:if torch.is_tensor(results):@@ -584,21 +663,23 @@ def run_batched_wrap_search(deferred_wrappers: list[WrapperLinear], max_batch: iresults = [r.to(home_key) if torch.is_tensor(r) else r for r in results]for w, res in zip(chunk, results):w.finalize_batched_search(res)- stats[worker_key]["batches"] += 1- stats[worker_key]["modules"] += len(chunk)+ st = summary.device(worker_key)+ st["batches"] += 1+ st["modules"] += len(chunk)@torch.no_grad()def _execute(worker_key: str, items: list[tuple[str, list[WrapperLinear]]]) -> None:- _t0 = time.perf_counter()- for home_key, chunk in items:- _run_chunk_stacked(home_key, chunk, worker_key)- stats[worker_key]["wall"] = time.perf_counter() - _t0+ with summary.timed(worker_key):+ for home_key, chunk in items:+ _run_chunk_stacked(home_key, chunk, worker_key)# singleton groups and non-stackable searches run per module on the# calling thread, each on its own weight's device- for _dev, w in singletons:- _run_one(w)- _n_single = len(singletons)+ for dev, w in singletons:+ with summary.timed(dev) as st:+ _run_one(w)+ st["singletons"] += 1+ st["modules"] += 1if len(buckets) > 1:pairs = list(buckets.items())@@ -607,27 +688,9 @@ def run_batched_wrap_search(deferred_wrappers: list[WrapperLinear], max_batch: ielif buckets:wk, items = next(iter(buckets.items()))_execute(wk, items)- _t_total = sum(st.get("wall", 0.0) for st in stats.values())- _n_mod = sum(st["modules"] for st in stats.values()) + _n_single- _n_batch = sum(st["batches"] for st in stats.values())- from auto_round.utils.pool_placement import _short_device_key-- per_device = (- "{" + ", ".join(f"'{_short_device_key(dev)}': {st.get('wall', 0.0):.2f}" for dev, st in stats.items()) + "}"- )- _n_offloaded = sum(1 for worker, home, _chunk in planned if worker != home)- if perf_counters_enabled():- logger.info(- "[batched-search] wrap search: %d modules, %d batch + %d singleton search calls"- + (f" ({_n_offloaded} chunk(s) offloaded to idle devices)" if _n_offloaded else "")- + ", %.2fs device-wall (%.2fs summed) walls %s",- _n_mod,- _n_batch,- _n_single,- max(st.get("wall", 0.0) for st in stats.values()) if stats else 0.0,- _t_total,- per_device,- )+ summary.finish()+ if _log_summary:+ summary.log("[batched-search] wrap search:")return True@@ -714,7 +777,8 @@ def run_batched_neuqi_anchors(deferred_wrappers: list) -> bool:for worker, _home, chunk in planned:buckets.setdefault(worker, []).append(chunk)- stats = {wk: {"modules": 0, "batches": 0, "singletons": 0} for wk in buckets}+ summary = SearchSummary()+ summary.offloaded = sum(1 for worker, home, _chunk in planned if worker != home)@torch.no_grad()def _run_anchor_stack(chunk: list, worker_key: str):@@ -741,7 +805,11 @@ def run_batched_neuqi_anchors(deferred_wrappers: list) -> bool:@torch.no_grad()def _execute(worker_key: str, chunk_list: list) -> None:- _t0 = time.perf_counter()+ with summary.timed(worker_key) as st:+ _execute_chunks(worker_key, chunk_list, st)++ @torch.no_grad()+ def _execute_chunks(worker_key: str, chunk_list: list, st: dict) -> None:for chunk in chunk_list:staged = [_staged(w) for w in chunk]homes = [str(s[0].device) for s in staged]@@ -769,18 +837,21 @@ def run_batched_neuqi_anchors(deferred_wrappers: list) -> bool:dump_oom_tensor_census_("neuqi anchor")for w in chunk:w._run_deferred_neuqi_anchor_now()- stats[worker_key]["singletons"] += len(chunk) # counted like the wrap lane's fallbacks+ st["singletons"] += len(chunk) # counted like the wrap lane's fallbacks+ st["modules"] += len(chunk)continuewmin = wmin.reshape(len(chunk), -1)wmax = wmax.reshape(len(chunk), -1)for w, s, mn, mx in zip(chunk, staged, wmin, wmax):w._apply_neuqi_anchor(mn.to(s[0].device), mx.to(s[0].device))- stats[worker_key]["batches"] += 1- stats[worker_key]["modules"] += len(chunk)- stats[worker_key]["wall"] = time.perf_counter() - _t0+ st["batches"] += 1+ st["modules"] += len(chunk)for w in singletons:- w._run_deferred_neuqi_anchor_now()+ with summary.timed(str(_staged(w)[0].device)) as st:+ w._run_deferred_neuqi_anchor_now()+ st["singletons"] += 1+ st["modules"] += 1if len(buckets) > 1:pairs = list(buckets.items())@@ -790,26 +861,5 @@ def run_batched_neuqi_anchors(deferred_wrappers: list) -> bool:wk, items = next(iter(buckets.items()))_execute(wk, items)- _t_total = sum(st.get("wall", 0.0) for st in stats.values())- _n_single = len(singletons) + sum(st["singletons"] for st in stats.values())- _n_mod = sum(st["modules"] for st in stats.values()) + _n_single- _n_batch = sum(st["batches"] for st in stats.values())- _n_offloaded = sum(1 for worker, home, _chunk in planned if worker != home)- from auto_round.utils.pool_placement import _short_device_key-- per_device = (- "{" + ", ".join(f"'{_short_device_key(dev)}': {st.get('wall', 0.0):.2f}" for dev, st in stats.items()) + "}"- )- if perf_counters_enabled():- logger.info(- "[batched-search] neuqi anchors: %d modules, %d batch + %d singleton search calls"- + (f" ({_n_offloaded} chunk(s) offloaded to idle devices)" if _n_offloaded else "")- + ", %.2fs device-wall (%.2fs summed) walls %s",- _n_mod,- _n_batch,- _n_single,- max(st.get("wall", 0.0) for st in stats.values()) if stats else 0.0,- _t_total,- per_device,- )+ summary.finish().log("[batched-search] neuqi anchors:")return Truediff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.pyindex d2a3a8070..e150bbab0 100644--- a/auto_round/algorithms/quantization/sign_round/quantizer.py+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py@@ -95,12 +95,11 @@ from auto_round.utils import ()from auto_round.utils.device import clear_memory_if_reached_thresholdfrom auto_round.utils.device_manager import device_manager-from auto_round.utils.distributed import setup_ddp_if_needed_-from auto_round.wrapper import (- WrapperLinear,- unwrapper_layer,- wrapper_block,-)+from auto_round.utils.distributed import all_reduce_loss, make_index_sampler, setup_ddp_if_needed_+from auto_round.wrapper import WrapperLinear, unwrapper_layer, wrapper_block++if TYPE_CHECKING:+ from auto_round.algorithms.composer import BlockContext@register_pipeline_member(SignRoundConfig)@@ -426,7 +425,10 @@ class SignRoundQuantizer(LFQMixin, BaseQuantizer):best_loss = torch.finfo(torch.float).maxnum_elm = 1mse_reduction = "mean"- if self.gradient_accumulate_steps != 1:+ # a masked loss is a SUM over the valid elements, normalized by the+ # valid-token count below (num_elm) -- also at accumulation == 1, where+ # a mean over all elements divided again by that count normalizes twice+ if self.gradient_accumulate_steps != 1 or valid_token_mask:mse_reduction = "sum"mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)scaler = self._get_scaler() # pylint: disable=assignment-from-none@@ -444,7 +446,7 @@ class SignRoundQuantizer(LFQMixin, BaseQuantizer):num_elm = sum(active_inputs[i.item()].numel() for i in whole_indices)block, sync_gradients = setup_ddp_if_needed_(self, block, device_manager.device_list)- index_sampler = IndexSampler(nsamples, global_batch_size)+ index_sampler = make_index_sampler(nsamples, global_batch_size)block_fwd = self.block_forward# When low_gpu_mem_usage is enabled, active_inputs / fp_outputs are intentionally@@ -652,6 +654,7 @@ class SignRoundQuantizer(LFQMixin, BaseQuantizer):# clear memory to avoid OOM due to memory fragmentationclear_memory_if_reached_threshold(threshold=0.8, device_list=device_manager.device_list)+ total_loss = all_reduce_loss(total_loss, num_elm)if i == 0:init_loss = total_losscurrent_lr = optimizer.param_groups[0]["lr"]diff --git a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py b/auto_round/algorithms/quantization/sign_roundv2/quantizer.pyindex c833a6542..2289e7b16 100644--- a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py+++ b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py@@ -490,6 +490,8 @@ class SignRoundDQWrapperLinear(WrapperLinear):class SignRoundV2Quantizer(SignRoundQuantizer):"""SignRound variant using the open algorithm-extension path in the new architecture."""+ calibration_stats = {"imatrix": "sum"}+def __init__(self, config: SignRoundConfig) -> None:super().__init__(config)self._use_outlier_suppressed_loss = Falsediff --git a/auto_round/cli/main.py b/auto_round/cli/main.pyindex 7984a58bd..96fdc9011 100644--- a/auto_round/cli/main.py+++ b/auto_round/cli/main.py@@ -446,6 +446,45 @@ def tune(args):device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)+ _parallel = str(getattr(args, "data_parallel", "off") or "off").strip().lower()+ _parallel_policy = None+ if _parallel != "off":+ import torch++ from auto_round import envs as _envs+ from auto_round.algorithms.parallel.data_parallel import (+ ParallelPolicy,+ resolve_parallel_world,+ )++ _n = torch.cuda.device_count()+ _iters_cli = int(getattr(args, "iters", 0) or 0)+ _world = resolve_parallel_world(_parallel, _n, iters=_iters_cli)+ # early eligibility: a multi-device --device_map cannot host the+ # data-parallel lane at iters > 0 (blocks shard across GPUs in a+ # pipeline-style placement) -- stop here, before the model loads,+ # with the same message the runtime resolver raises at block 0+ from auto_round.algorithms.parallel.data_parallel import check_parallel_device_map_compat_++ check_parallel_device_map_compat_(_world, args.device_map, _iters_cli)+ _parallel_policy = ParallelPolicy(+ world=_world,+ source="auto" if _parallel == "auto" else "explicit",+ collect_forward_cap=int(getattr(_envs, "AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES", 0) or 0),+ batch_mode=str(getattr(args, "parallel_batch", "split") or "split").strip().lower(),+ )+ if _parallel == "auto" and _world < _n:+ # only `auto` adds information here (the reduction from the+ # visible count); an explicit N either validates silently or+ # raises, and the engagement line reports the resolved plan+ logger.info(+ "[tune-ddp] --data_parallel auto -> world=%d (largest power of two <= %d visible "+ "devices; %d unused)",+ _world,+ _n,+ _n - _world,+ )+if args.enable_torch_compile is False:logger.info("`torch.compile` is explicitly disabled with `--disable_torch_compile`.")@@ -575,6 +614,11 @@ def tune(args):),)+ if _parallel_policy is not None:+ # the resolved --data_parallel policy is set on the compress+ # context; the data-parallel engine reads it from there+ autoround.compress_context.parallel_policy = _parallel_policy+model, folders = autoround.quantize_and_save( # pylint: disable=no-memberargs.output_dir,format=getattr(args, "_api_format", args.format),diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.pyindex 60c29e799..e8ae636f9 100644--- a/auto_round/cli/parser.py+++ b/auto_round/cli/parser.py@@ -154,6 +154,28 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Argurt.add_argument("--device_map", "--device", "--devices", default="0", type=str, help="Device mapping used for quantization.")+ rt.add_argument(+ "--data_parallel",+ default="off",+ type=str,+ help="Single-process data-parallel SignRound tuning: 'off' (default), 'auto' (all visible CUDA "+ "devices at --iters 0, otherwise the largest power of two that fits them), or an explicit "+ "world size N (at least 2, at most the visible device count; a power of two when "+ "--iters > 0). Replicas hold full block mirrors on the plan "+ "devices and draw disjoint calibration shards; free-VRAM estimates are advisory.",+ )+ rt.add_argument(+ "--parallel_batch",+ default="split",+ type=str,+ choices=["split", "per_replica"],+ help="Batch semantics for --data_parallel tuning. 'split' (default): the requested "+ "--batch_size is the GLOBAL per-iteration batch and every replica draws its world-split "+ "share (serial-equivalent updates). 'per_replica': every replica draws the FULL "+ "--batch_size from its own calibration shard each iteration, so the effective global "+ "batch is batch_size x world (torchrun-like); the per-replica draw clamps to the pool shard "+ "when it is smaller.",+ )rt.add_argument("--dataset", default=None, type=str, help="Calibration dataset or local dataset path.")rt.add_argument("--seed", default=42, type=int, help="Random seed for reproducibility.")rt.add_argument(diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.pyindex c87ddb34e..e3fe0dc3b 100644--- a/auto_round/compressors/orchestrator.py+++ b/auto_round/compressors/orchestrator.py@@ -416,6 +416,12 @@ class CompressionOrchestrator(BaseOrchestrator):# folded into the end-of-block phases line as its first item so# the line's total covers the whole block (true-wall comparable)block_name_or_names = n if nblocks == 1 else names+ # tick(key) closes the running span and opens `key`, so every+ # tick sits right BEFORE the region it names. The chain opens+ # here: "prep" (materialize + dtype convert + dispatch_block+ # device placement + BlockContext) costs seconds per block on+ # large MoE models, and the total covers the true per-block wall.+ _perf.tick("prep")# ── Infrastructure: materialize, dtype convert, device placement ──materialize_model_(m)@@ -453,8 +459,9 @@ class CompressionOrchestrator(BaseOrchestrator):# primary cache device when it fits (today's behavior, zero peer# traffic), or sharded across free GPUs when it does not. Placement is# pure memory behavior -- chunk values are bit-identical.- self._attach_pool_placement(m, input_ids, q_input, input_others)_perf.tick("attach")+ self._attach_pool_placement(m, input_ids, q_input, input_others)+ _perf.tick("compress")# ── Run block pipeline (calibration → quantization → collection) ──new_q_input, reference_output = self.alg_composer.compress_block(@@ -465,7 +472,7 @@ class CompressionOrchestrator(BaseOrchestrator):q_inputs=q_input,input_ids=token_ids,)- _perf.tick("compress")+ _perf.tick("swap")# ── Infrastructure: memory management ─────────────────────────────# Mirrors the original q_input-swap + end-of-loop clear_memory semantics:@@ -483,7 +490,7 @@ class CompressionOrchestrator(BaseOrchestrator):clear_memory(input_ids if input_ids is not next_input_ids else None)q_input = new_q_input- _perf.tick("swap")+ _perf.tick("sweep")# Belt-and-braces: no module may carry staged batched-search inputs# (weight reshape + imatrix copies) across the block boundary;@@ -492,10 +499,10 @@ class CompressionOrchestrator(BaseOrchestrator):for _mn, _mm in m.named_modules():if getattr(_mm, "_deferred_search_inputs", None) is not None:_mm._deferred_search_inputs = None- _perf.tick("sweep")+ _perf.tick("hooks")if len(device_manager.device_list) > 1 and not self.model_context.is_diffusion:accelerate.hooks.remove_hook_from_submodules(m)- _perf.tick("hooks")+ _perf.tick("pack")# ── Infrastructure: immediate_pack / shard write ──────────────────# Pack BEFORE draining the block off the GPUs: pack_layer resolves@@ -521,14 +528,14 @@ class CompressionOrchestrator(BaseOrchestrator):if module_name is None:continue_immediate_pack(module_name, _layer_cfg)- _perf.tick("pack")+ _perf.tick("mv")mv_module_from_gpu(m)- _perf.tick("mv")- clear_memory(device_list=device_manager.device_list)_perf.tick("clear")- memory_monitor.log_summary()+ clear_memory(device_list=device_manager.device_list)_perf.tick("monitor")+ memory_monitor.log_summary()+ _perf.tick("write")input_ids = next_input_ids@@ -583,13 +590,13 @@ class CompressionOrchestrator(BaseOrchestrator):_bg_resume.dispatch(lambda _n=n, _q=_resume_q, _i=_resume_i: resume_state.mark_block_done(_n, _q, _i))- _perf.tick("write")if _perf.enabled:_perf.stop("write")# sweep/hooks fold into the printed pack span, monitor into the# printed write span; they were bad-perf-investigation probes that# read 0.00 in every validated run since_parts = [+ f"prep={_perf.get('prep'):.2f}s",f"attach={_perf.get('attach'):.2f}s",f"compress={_perf.get('compress'):.2f}s",f"swap={_perf.get('swap'):.2f}s",@@ -681,6 +688,13 @@ class CompressionOrchestrator(BaseOrchestrator):# full-model materialization and linear CPU RAM growth.logger.info("Zero-shot mode (no calibration data needed): using blockwise quantization.")+ # an explicit --data_parallel N cannot engage here (no+ # calibration forward for the data-parallel lane to shard): one INFO+ # line, no refusal -- weights-only RTN has nothing to parallelize+ from auto_round.algorithms.parallel.data_parallel import log_zero_shot_parallel_noop_++ log_zero_shot_parallel_noop_(getattr(self.compress_context, "parallel_policy", None))+tied_weights_keys = getattr(self.model, "_tied_weights_keys", [])if tied_weights_keys is None:tied_weights_keys = []diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex d94a64d37..78ac1233a 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -185,6 +185,64 @@ def block_forward(input_others[param_name] = valpositional_inputs = ()+ # DDP-mirror device audit (last line of defense): every tensor kwarg must+ # sit on the same accelerator device as the block's parameters. A stale+ # kwarg (e.g. an entry recorded on the primary GPU while the replica+ # lives elsewhere) silently overrides the staged first-arg inside this+ # function and crashes cross-device inside the block forward -- serial+ # runs with a single shared device stay unaffected. Move the stragglers+ # onto the block's device AND name them (fail-visible).+ try:+ _block_dev = next(block.parameters()).device+ except StopIteration: # pragma: no cover - param-less blocks+ _block_dev = device++ def _audit(value, prefix, bad):+ if torch.is_tensor(value):+ if value.device.type in ("cuda", "xpu", "hpu") and value.device != _block_dev:+ bad.append(f"{prefix}:{value.device}")+ elif isinstance(value, (list, tuple)):+ for _i, _v in enumerate(value):+ _audit(_v, f"{prefix}[{_i}]", bad)+ elif isinstance(value, dict):+ for _k, _v in value.items():+ _audit(_v, f"{prefix}.{_k}", bad)++ def _relocate(value):+ # Mirror _audit's condition exactly: only accelerator tensors that+ # mismatch the block device move. A wholesale to_device() here would+ # also migrate host tensors nested in lists/dicts onto the accelerator+ # (semantic change + VRAM), which the audit never flagged.+ if torch.is_tensor(value):+ if value.device.type in ("cuda", "xpu", "hpu") and value.device != _block_dev:+ return value.to(_block_dev)+ return value+ if isinstance(value, list):+ return [_relocate(_v) for _v in value]+ if isinstance(value, tuple):+ return tuple(_relocate(_v) for _v in value)+ if isinstance(value, dict):+ return {k: _relocate(v) for k, v in value.items()}+ return value++ _bad = []+ for _k, _v in input_others.items():+ _audit(_v, str(_k), _bad)+ if _bad:+ logger.warning(+ "[block-forward] device sweep: moving %d stale kwarg tensor(s) onto %s: %s%s",+ len(_bad),+ _block_dev,+ ", ".join(_bad[:8]),+ " ..." if len(_bad) > 8 else "",+ )+ for _k in list(input_others.keys()):+ if torch.is_tensor(input_others[_k]) and input_others[_k].device.type in ("cuda", "xpu", "hpu"):+ if input_others[_k].device != _block_dev:+ input_others[_k] = input_others[_k].to(_block_dev)+ elif isinstance(input_others[_k], (list, tuple, dict)):+ input_others[_k] = _relocate(input_others[_k])+if amp:with autocast(device_type=str(device).split(":")[0], dtype=amp_dtype): # pragma: no coveroutput = block(**input_others)@@ -452,19 +510,35 @@ class IndexSampler:nsamples (int): Total number of samples.batch_size (int): Number of indices to return in each batch.index (int): Current position in the index list.- indices (List[int]): Shuffled list of indices.+ indices (list[int]): Shuffled list of indices."""- def __init__(self, nsamples: int, batch_size: int) -> None:+ def __init__(+ self,+ nsamples: int,+ batch_size: int,+ indices: "list[int] | None" = None,+ rng: "random.Random | None" = None,+ ) -> None:"""Initializes the sampler.Args:nsamples (int): Total number of samples (must be >= batch_size).batch_size (int): Number of indices per batch.+ indices (list[int], optional): Explicit index pool to shuffle+ (defaults to ``range(nsamples)``). Used by shard-constrained+ DDP sampling; ``nsamples`` must equal ``len(indices)``.+ rng (random.Random, optional): Private shuffle generator (defaults+ to the global ``random`` module). Data-parallel rank threads+ pass their own so concurrent reshuffles stay reproducible.Raises:ValueError: If batch_size is not in the range (0, nsamples]."""+ if indices is None:+ indices = list(range(nsamples))+ elif len(indices) != nsamples:+ raise ValueError(f"indices pool ({len(indices)}) does not match nsamples ({nsamples})")if batch_size <= 0 or batch_size > nsamples:raise ValueError("batch_size must be > 0 and <= nsamples")@@ -472,8 +546,9 @@ class IndexSampler:self.batch_size: int = batch_sizeself.index: int = 0- self.indices: list[int] = list(range(nsamples))- random.shuffle(self.indices)+ self._rng = rng if rng is not None else random+ self.indices: list[int] = list(indices)+ self._rng.shuffle(self.indices)def next_batch(self) -> list[int]:"""Returns the next batch of shuffled indices.@@ -485,7 +560,7 @@ class IndexSampler:list[int]: A list of size `batch_size` containing sample indices."""if self.index + self.batch_size > self.nsamples:- random.shuffle(self.indices)+ self._rng.shuffle(self.indices)self.index = 0batch = self.indices[self.index : self.index + self.batch_size]diff --git a/auto_round/context/compress.py b/auto_round/context/compress.pyindex c6f35b74b..09eb35c2b 100644--- a/auto_round/context/compress.py+++ b/auto_round/context/compress.py@@ -66,6 +66,9 @@ class CompressContext(BaseContext):self.static_attention_dtype = static_attention_dtypeself.static_kv_granularity = static_kv_granularityself.static_attention_granularity = static_attention_granularity+ # Resolved --data_parallel policy (ParallelPolicy or None = off);+ # set once at the CLI entry, read by the data-parallel engine+ self.parallel_policy = kwargs.pop("parallel_policy", None)def clear_memory(self, tensor=None):"""Clear GPU/CPU memory only when ``low_gpu_mem_usage`` is enabled."""diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 1b6744be0..fb679e89b 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -12,6 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.+import functoolsimport mathfrom collections.abc import Callablefrom functools import lru_cache@@ -199,7 +200,6 @@ def _int_search_with_thresh(search_scales: Any, q_scale_thresh: float) -> "Callaper-call closure, so resolved callables compare equal across calls and cansafely key same-shape search batches."""- import functoolsdef _clamped(weight_reshape: torch.Tensor, bits: int, imatrix: "torch.Tensor | None", *, _thresh: float@@ -372,10 +372,12 @@ def reshape_imatrix_for_weight(imatrix, weight_reshape: torch.Tensor, group_size"""if imatrix is None or not isinstance(imatrix, torch.Tensor):return torch.ones_like(weight_reshape)- imatrix = imatrix.reshape(1, -1)+ # relocate the RAW column first: moving the expanded copy after the fact+ # would pay a weight-sized cross-device transfer for sharded placements+ imatrix = imatrix.reshape(1, -1).to(weight_reshape.device)imatrix = reshape_pad_tensor_by_group_size(imatrix, group_size, val=1e-5)[0].view(1, -1)imatrix = imatrix.expand(weight_reshape.numel() // imatrix.numel(), -1)- return imatrix.reshape(weight_reshape.shape).to(weight_reshape.device)+ return imatrix.reshape(weight_reshape.shape)def compute_optimized_init_scale(diff --git a/auto_round/envs.py b/auto_round/envs.pyindex 595b17d1a..57f0f0398 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -45,6 +45,7 @@ if TYPE_CHECKING:AR_SEARCH_BATCH_GB: float | None = NoneAR_DISABLE_CROSS_DEVICE_SEARCH: bool = FalseAR_ENABLE_LOG_MINMAX: bool = False+ AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES: int = 0def _get_optional_positive_int_env(name: str) -> int | None:@@ -61,6 +62,20 @@ def _get_optional_positive_int_env(name: str) -> int | None:return value+def _get_non_negative_int_env(name: str, default: int) -> int:+ """Read an env var that must be a non-negative integer; malformed values fail fast."""+ raw = os.getenv(name)+ if raw is None:+ return default+ try:+ value = int(raw)+ except ValueError as exc:+ raise ValueError(f"{name} must be a non-negative integer, got {raw!r}") from exc+ if value < 0:+ raise ValueError(f"{name} must be a non-negative integer, got {raw!r}")+ return value++def _get_optional_positive_float_env(name: str) -> float | None:"""Read an optional env var that must be a positive float when set."""raw = os.getenv(name)@@ -172,6 +187,8 @@ environment_variables: dict[str, Callable[[], Any]] = {"AR_RESUME_DIR": lambda: os.getenv("AR_RESUME_DIR", None),"AR_DISABLE_BATCHED_SEARCH": lambda: os.getenv("AR_DISABLE_BATCHED_SEARCH", "False").strip().lower()in ("1", "true", "yes"),+ # Emit [perf] phase-breakdown log lines (per-block load/tune/pack/write/clean/offload+ # in the data-driven loop; the sharded collection breakdown for --data_parallel)."AR_PERF_COUNTERS": lambda: os.getenv("AR_PERF_COUNTERS", "False").strip().lower() in ("1", "true", "yes"),"AR_SEARCH_BATCH_GB": lambda: _get_optional_positive_float_env("AR_SEARCH_BATCH_GB"),"AR_DISABLE_CROSS_DEVICE_SEARCH": lambda: os.getenv("AR_DISABLE_CROSS_DEVICE_SEARCH", "False").strip().lower()@@ -216,9 +233,8 @@ environment_variables: dict[str, Callable[[], Any]] = {# Benchmarks (A100, W4G128 tuning fwd+bwd) show "auto" matches or beats a# fixed 16 on every Qwen3-MoE preset (e.g. 35B-A3B: 108 ms vs 156 ms).# <int > 0> - fixed group size. Fusing every active expert at once builds a working set- # that grows with the expert count; past a point that costs peak memory on- # GPU and cache locality on CPU (measured: fusing 64 experts halved CPU- # calibration throughput, and doubled the GPU calibration peak).+ # that grows with the expert count; past a point that costs peak+ # memory on GPU and cache locality on CPU.# 0 or <0 - disable chunking/tiling: "fuse everything" in one group (e.g. set -1 to turn it off).# Results are identical for any value -- rows stay independent. A fixed count and "auto"# are both torch.compile-friendly (constant fused shape -> no per-count recompile).@@ -257,6 +273,14 @@ environment_variables: dict[str, Callable[[], Any]] = {"AR_NEUQI_COARSE": lambda: int(v) if (v := os.getenv("AR_NEUQI_COARSE")) is not None else None,"AR_NEUQI_FINE": lambda: int(v) if (v := os.getenv("AR_NEUQI_FINE")) is not None else None,"AR_NEUQI_BACKEND": lambda: os.getenv("AR_NEUQI_BACKEND", "auto").lower(),+ # Cap the number of devices a hook-carrying sharded collection forward runs+ # on concurrently (hook passes keep compiled runners python-bound between+ # graph sections, which can convoy on the GIL). Only lower this on hosts+ # where hook-carrying passes convoy. Values above the engaged world are+ # no-ops. Default 0 = no cap.+ "AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES": lambda: _get_non_negative_int_env(+ "AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES", 0+ ),}diff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex 1381aeaf7..ac9e777c8 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -127,7 +127,12 @@ def _bump_dynamo_cache_limit(min_size: int | None = None):min_size = envs.AR_DYNAMO_CACHE_SIZE_LIMITfrom torch._dynamo import config as _dynamo_config- for attr in ("cache_size_limit", "accumulated_cache_size_limit", "recompile_limit"):+ for attr in (+ "cache_size_limit",+ "accumulated_cache_size_limit",+ "recompile_limit",+ "accumulated_recompile_limit", # newer torch: process-wide cap (binds on MoE shape zoos)+ ):if hasattr(_dynamo_config, attr) and getattr(_dynamo_config, attr) < min_size:setattr(_dynamo_config, attr, min_size)except Exception: # pragma: no cover - best effort@@ -1580,6 +1585,9 @@ class MemoryMonitor:self.peak_ram = 0.0 # GBself.peak_vram = {} # {device_id: peak_mb}self.enabled = True+ # devices registered for sampling BEYOND the caller's device_map list+ # (e.g. DDP replica devices the parallel lane engaged outside the map)+ self._extra_devices: set[str] = set()@staticmethoddef _process_tree_rss() -> float:@@ -1593,6 +1601,31 @@ class MemoryMonitor:continuereturn rss / 1024**3+ def register_extra_devices(self, devices) -> None:+ """Add accelerator devices to the monitored set beyond the device_map.++ The final ``'peak_vram': {...}`` summary only reports devices the+ ``update`` calls sample; callers pass the device-map list, so devices+ engaged outside the map (single-process DDP replicas on all visible+ GPUs while ``--device_map`` is single-device) would stay invisible.+ Registered devices are merged into every subsequent ``update``/+ ``update_hpu`` sample; ``reset`` clears the peaks but keeps the+ registration (engagement re-registers per block).+ """+ if not hasattr(self, "_extra_devices"): # older singleton instance+ self._extra_devices = set()+ for d in devices or []:+ key = str(d)+ if key.split(":")[0] in ("cuda", "xpu", "hpu"):+ self._extra_devices.add(key)++ def _merged_device_list(self, device_list):+ """The caller's device list plus any registered extra devices (deduped)."""+ if not getattr(self, "_extra_devices", None):+ return device_list+ seen = [str(d) for d in device_list]+ return list(device_list) + [d for d in sorted(self._extra_devices) if d not in seen]+def update(self, device_list=None):"""Update current memory usage and track peaks."""if not self.enabled:@@ -1611,6 +1644,7 @@ class MemoryMonitor:device_list = list(range(dev_mgr.device_count()))else:device_list = [0]+ device_list = self._merged_device_list(device_list)dev_mgr = get_current_device_manager()for device in device_list:@@ -1658,6 +1692,7 @@ class MemoryMonitor:device_list = list(range(count)) if count > 0 else [0]elif not isinstance(device_list, (list, tuple)):device_list = [device_list]+ device_list = self._merged_device_list(device_list)for device in device_list:if str(device) == "cpu":continuediff --git a/auto_round/utils/device_manager.py b/auto_round/utils/device_manager.pyindex 9db19f703..ba43e9033 100644--- a/auto_round/utils/device_manager.py+++ b/auto_round/utils/device_manager.py@@ -821,6 +821,9 @@ class DeviceManager:@propertydef device_list(self) -> list:"""All concrete devices selected by the configured ``device_map``."""+ scoped = self._thread_scope()+ if scoped is not None:+ return [scoped]if self._device_list is None:self.configure(self._device_map)return self._device_list@@ -828,10 +831,35 @@ class DeviceManager:@propertydef device(self) -> str:"""The major (primary, non-CPU when possible) device string."""+ scoped = self._thread_scope()+ if scoped is not None:+ return scopedif self._major_device is None:self.configure(self._device_map)return self._major_device+ def _thread_scope(self) -> str | None:+ local = self.__dict__.get("_thread_local")+ return getattr(local, "device", None) if local is not None else None++ @contextlib.contextmanager+ def thread_device_scope(self, device: str | torch.device):+ """Pin :attr:`device` / :attr:`device_list` to ``device`` for the calling thread.++ Used by the data-parallel engine: each rank thread runs the unchanged+ per-block algorithm and must see its own replica device as the major+ device. Other threads keep the configured devices.+ """+ import threading++ local = self.__dict__.setdefault("_thread_local", threading.local())+ prev = getattr(local, "device", None)+ local.device = str(device)+ try:+ yield+ finally:+ local.device = prev+@device.setterdef device(self, value: str | torch.device) -> None:"""Override the major device (e.g. an OOM fallback to ``"cpu"``)."""diff --git a/auto_round/utils/distributed.py b/auto_round/utils/distributed.pyindex 6a7936ce9..5879a8f2b 100644--- a/auto_round/utils/distributed.py+++ b/auto_round/utils/distributed.py@@ -55,6 +55,14 @@ def setup_ddp_if_needed_(ar, block: torch.nn.Module, device_list: list[int]):"""import torch.distributed as dist+ from auto_round.algorithms.parallel.spmd import current_rank++ rank = current_rank()+ if rank is not None:+ # in-process thread ranks (--data_parallel): each rank tunes+ # its own replica; the hook averages gradients across the ranks+ return block, lambda: rank.sync_gradients(block)+# The process group can be created and destroyed by individual tests or# by successive launcher invocations. Do not reuse a cached result from# a previous process-group lifetime.@@ -105,6 +113,40 @@ def _noop_sync():pass+def make_index_sampler(nsamples: int, batch_size: int):+ """The tuning loop's sample sampler.++ Serial and torchrun ranks (whose calibration data the caller already+ partitioned) draw from all ``nsamples``. An in-process thread rank draws+ from its own contiguous shard of the calibration pool.+ """+ from auto_round.algorithms.parallel.spmd import current_rank+ from auto_round.compressors.utils import IndexSampler++ rank = current_rank()+ if rank is not None:+ return rank.index_sampler(nsamples, batch_size)+ return IndexSampler(nsamples, batch_size)+++def all_reduce_loss(loss: float, weight: float = 1.0) -> float:+ """Global iteration loss across ranks; identity when not data-parallel.++ ``loss`` is the rank's normalized loss and ``weight`` its normalizer+ (element or valid-token count), so the result is the weighted mean+ sum(loss * weight) / sum(weight) -- the serial value over the union of+ the ranks' samples. Every rank must take the same best-iteration and+ early-stop decisions (it holds the same parameters after each synced+ step), so the decisions use the global loss.+ """+ from auto_round.algorithms.parallel.spmd import current_rank++ rank = current_rank()+ if rank is not None:+ return rank.all_reduce_weighted_mean(loss, weight)+ return loss++def _move_block_to_device(module: torch.nn.Module, device):"""Move *module* to *device*, used before DDP wrapping."""module.to(device)diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 331d84b1a..d154b604a 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -1063,10 +1063,15 @@ def wrapper_block(for w in anchor_pending:w._run_deferred_neuqi_anchor_now()# stacked same-shape batches per (device, shape, config, search fn);- # singletons and the kill-switch fall back to the identical per-module call- if not search_dispatch.run_batched_wrap_search(deferred_wrappers):- for w in deferred_wrappers:- w._run_deferred_search_now()+ # singletons and the kill-switch fall back to the identical per-module call.+ # Data-parallel ranks shard the searches across the ranks instead.+ from auto_round.algorithms.parallel.spmd import current_rank, run_rank_sharded_wrap_searches++ rank = current_rank()+ if not (rank is not None and run_rank_sharded_wrap_searches(rank, deferred_wrappers)):+ if not search_dispatch.run_batched_wrap_search(deferred_wrappers):+ for w in deferred_wrappers:+ w._run_deferred_search_now()quantized_layers = [n for (n, _m, _k), r in zip(work, results) if r == "q"]unquantized_layers = [n for (n, _m, _k), r in zip(work, results) if r == "u"]diff --git a/docs/environments.md b/docs/environments.mdindex 66e5f1788..74db88e88 100644--- a/docs/environments.md+++ b/docs/environments.md@@ -411,6 +411,16 @@ else:4. **Performance Optimization**: Enable `AR_ENABLE_COMPILE_PACKING=1` if you have sufficient computational resources5. **Custom Workspace**: Set `AR_WORK_SPACE` to a directory with sufficient disk space for model processing+### AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES+- **Description**: Caps the number of devices a hook-carrying sharded collection forward runs on concurrently (hook passes keep compiled runners python-bound between graph sections, which can convoy on the GIL). Only lower this on hosts where hook-carrying passes convoy; not reproduced on the validation rig up to world=8. Values above the engaged world are no-ops.+- **Default**: `0` (no cap)+- **Valid Values**: non-negative integer+- **Usage**: Limit concurrent shard-forward threads on weak-CPU hosts.++```bash+AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES=2 python -m auto_round --model ... --data_parallel 8+```+## Notes- Environment variables are evaluated lazily, meaning they are only read when first accesseddiff --git a/docs/environments_CN.md b/docs/environments_CN.mdindex 7a43b744a..1eb1f1d1d 100644--- a/docs/environments_CN.md+++ b/docs/environments_CN.md@@ -411,6 +411,16 @@ else:4. **性能优化**:如有足够算力,可启用 `AR_ENABLE_COMPILE_PACKING=1`5. **自定义工作目录**:将 `AR_WORK_SPACE` 设置为磁盘空间充足的目录+### AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES+- **描述**:限制携带 hook 的分片采集前向并发使用的设备数(hook 前向会使编译后的执行器在图分段之间退回 Python 绑定,可能在 GIL 上形成串行瓶颈)。仅在 hook 前向出现串行瓶颈的主机上才需要调低;在验证机台上至 world=8 未复现。大于已启用 world 的值等效于不限制。+- **默认值**:`0`(不限制)+- **取值**:非负整数+- **用法**:在弱 CPU 主机上限制并发分片前向线程数。++```bash+AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES=2 python -m auto_round --model ... --data_parallel 8+```+## 注意事项- 环境变量采用懒加载方式,仅在首次访问时读取diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 60bf7421a..58366832e 100755--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -936,6 +936,37 @@ CUDA_VISIBLE_DEVICES=0,1,2,3 auto-round --model "Qwen/Qwen3-0.6B" --scheme "W4A1There are typically two scenarios that require multi-GPU tuning: one is the calibration phase mainly for lm-head quantization, and the other is quantizing extremely large models (e.g., models larger than 100 GB).+#### Parallel block tuning across GPUs+When a tuning block fits on one GPU, `--data_parallel` uses the other GPUs of the same process for+data-parallel tuning: the block's tuning loop runs once per GPU (a rank, one thread each) on that GPU's copy of the+block and its share of the calibration samples, and the ranks average their gradients every iteration. This is the+same contract as the multi-process torchrun path (`sync_gradients()`), so the algorithm's loop itself is unchanged.++~~~bash+CUDA_VISIBLE_DEVICES=0,1,2,3 auto-round --model "Qwen/Qwen3-0.6B" --scheme "W4A16" --data_parallel auto+~~~++- `off` (default) keeps the serial loop. `N` pins the number of ranks: at least 2, at most the visible device count,+ and a power of two when `--iters > 0` (the gradient all-reduce is a halving-doubling exchange); checked at startup.+ `auto` takes all visible devices at `--iters 0` and the largest power of two that fits them at `--iters > 0`, and+ announces the choice in the log.+- The block must sit whole on one GPU: run without `--device_map` or with a single-device map. A block spread across+ several GPUs stops with that reason; blocks larger than one GPU are not covered.+- Every rank holds a full copy of the block plus its tuning state. Free-VRAM estimates only warn (naming the+ device); an actual shortage surfaces as an out-of-memory error.+- `--parallel_batch split` (default) keeps `--batch_size` as the global batch of an iteration; every rank draws+ `batch_size / world` samples (the division must be exact). `--parallel_batch per_replica` lets every rank draw the+ full `--batch_size`, so the global batch is `batch_size x world`.+- A block whose sample count or batch does not split evenly over the ranks, or whose per-rank batch exceeds its+ share of the samples, tunes serially and logs the reason. LFQ, dict-shaped (diffusion-style) calibration pools and+ an active torchrun group are not supported.+- The calibration forwards (reference outputs, the quantized-input chain, imatrix and activation statistics) are+ sharded across the GPUs too, also at `--iters 0`. Quantizers declare the statistics their hooks write and how they+ reduce (`calibration_stats`); a hook pass with undeclared statistics stays serial.+- Each rank samples from its own share of the calibration pool, so per-iteration loss lines and tuned weights differ+ from a serial run. Compare runs at the model level (accuracy, KL divergence against the base model).+- To exclude devices, set the backend's visibility variable (e.g. `CUDA_VISIBLE_DEVICES`) before launch.+#### Enable multiple gpus calibration in lm_head quantizationFor LM head tuning, AutoRound needs to cache the inputs to the lm-head, which requires the entire model to reside onthe GPU for efficient calibration. If there is no enough VRAM, some layers will fallback to RTN modediff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex 5514fa88d..1e708b5bc 100755--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -907,6 +907,34 @@ CUDA_VISIBLE_DEVICES=0,1,2,3 auto-round --model "Qwen/Qwen3-0.6B" --scheme "W4A1通常有两种情况需要启用多 GPU 训练:一是主要针对 lm-head 量化的标定阶段,二是参数量极大(如显存占用超 100GB)的模型。+#### 跨 GPU 并行 block 调优+当单个 block 可以放入单张 GPU 时,可通过 `--data_parallel` 在同一进程中利用其他 GPU 做数据并行调优:+block 的调优循环在每张 GPU 上各运行一次(每张 GPU 一个 rank,各占一个线程),使用该 GPU 上的 block 副本和它分到的+那部分标定样本,各 rank 在每次迭代中对梯度求平均。这与多进程 torchrun 路径的约定(`sync_gradients()`)相同,因此算法+本身的调优循环不需要修改。++```bash+CUDA_VISIBLE_DEVICES=0,1,2,3 auto-round --model "Qwen/Qwen3-0.6B" --scheme "W4A16" --data_parallel auto+```++- `off`(默认)保持串行循环。`N` 指定 rank 数:至少为 2,不超过可见设备数;`--iters > 0` 时必须为 2 的幂+ (梯度 all-reduce 采用 halving-doubling 交换),启动时校验。`auto` 在 `--iters 0` 时使用全部可见设备,在+ `--iters > 0` 时取不超过可见设备数的最大 2 的幂,并在日志中说明所选数量。+- block 必须完整放在一张 GPU 上:不指定 `--device_map`,或使用单设备的 map。跨多张 GPU 放置的 block 会带着原因+ 停止;超过单张 GPU 容量的 block 不在支持范围内。+- 每个 rank 持有完整的 block 副本及其调优状态。空闲显存估算只会给出告警(指明设备),真正的显存不足会以 OOM+ 错误的形式出现。+- `--parallel_batch split`(默认)将 `--batch_size` 作为每次迭代的全局 batch,每个 rank 抽取 `batch_size / world`+ 个样本(必须整除)。`--parallel_batch per_replica` 让每个 rank 抽取完整的 `--batch_size`,全局 batch 为+ `batch_size x world`。+- 若某个 block 的样本数或 batch 不能被 rank 数整除,或每个 rank 的 batch 超过它分到的样本数,该 block 改为串行+ 调优并在日志中给出原因。不支持 LFQ、字典形式(diffusion 类)的标定池,以及已激活的 torchrun 进程组。+- 标定前向(参考输出、量化输入链、imatrix 与激活统计)同样在各 GPU 间分片执行,`--iters 0` 时也是如此。量化器通过+ `calibration_stats` 声明其 hook 写入的统计量及归约方式;带有未声明统计量的 hook 前向保持串行。+- 每个 rank 从自己那部分标定池中抽样,因此每次迭代的 loss 日志和调优后的权重与串行运行不同。请在模型层面+ (精度、相对基础模型的 KL 散度)比较不同运行。+- 如需排除设备,请在启动前设置对应后端的可见性环境变量(如 `CUDA_VISIBLE_DEVICES`)。+#### lm_head 量化中开启多 GPU 标定量化 lm-head 时,AutoRound 需要缓存其输入数据以进行高效的标定,这要求**整个模型驻留在 GPU 显存中** ;若 GPU 显存不足,部分层会退回至 Pure RTN 模式。diff --git a/test/unit/common/compressors/test_reference_output.py b/test/unit/common/compressors/test_reference_output.pyindex b55630aa9..6f8acdecd 100644--- a/test/unit/common/compressors/test_reference_output.py+++ b/test/unit/common/compressors/test_reference_output.py@@ -34,8 +34,8 @@ def _patch_composer(composer, fp_inputs):composer.block_forward = mock_forwardcomposer.block_quantizer.quantize_block = MagicMock()composer.block_quantizer.enable_quanted_input = False- composer._get_fp_act_hooks = MagicMock(return_value=[])- composer._get_q_act_hooks = MagicMock(return_value=[])+ composer._get_fp_act_hooks = MagicMock(return_value=([], {"act_max": "max"}))+ composer._get_q_act_hooks = MagicMock(return_value=([], {"act_max": "max"}))composer.preprocessors = []return mock_forward@@ -127,8 +127,8 @@ def test_weight_changing_preprocessor_recomputes_seeded_reference():composer.block_forward = BlockForwardRunner(batch_size=1, amp=False, enable_torch_compile=False)composer.block_quantizer.quantize_block = MagicMock()composer.block_quantizer.enable_quanted_input = False- composer._get_fp_act_hooks = MagicMock(return_value=[])- composer._get_q_act_hooks = MagicMock(return_value=[])+ composer._get_fp_act_hooks = MagicMock(return_value=([], None))+ composer._get_q_act_hooks = MagicMock(return_value=([], None))composer.preprocessors = [_WeightChangingPreprocessor()]_, reference_next_input = composer.compress_block(diff --git a/test/unit/test_cpu/algorithms/test_qad_warm_start.py b/test/unit/test_cpu/algorithms/test_qad_warm_start.pyindex 18bc46ead..48ab754bc 100644--- a/test/unit/test_cpu/algorithms/test_qad_warm_start.py+++ b/test/unit/test_cpu/algorithms/test_qad_warm_start.py@@ -128,7 +128,7 @@ def test_warmup_propagates_best_quantized_output(monkeypatch, wrapper_cls, outpuexpected_outputs = []calls = []- def forward(module, samples, others):+ def forward(module, samples, others, cache_device=None):calls.append(module[0])result = [module(sample) for sample in samples]forward.last_output_dict = {"hidden_states": result} if output_dict else None@@ -163,8 +163,8 @@ def test_warmup_propagates_best_quantized_output(monkeypatch, wrapper_cls, outpucomposer.preprocessors = []composer._rotation_members = []composer.scheme = None- composer._get_fp_act_hooks = lambda block: []- composer._get_q_act_hooks = lambda block: []+ composer._get_fp_act_hooks = lambda block: ([], None)+ composer._get_q_act_hooks = lambda block: ([], None)monkeypatch.setattr("auto_round.algorithms.composer.clear_memory", lambda: None)ctx = BlockContext(model=block, block_names=["0"], block_name="0", block_index=0)quantized, reference = composer.compress_block(block, inputs, {}, ctx)diff --git a/test/unit/test_cpu/quantization/test_ddp_core.py b/test/unit/test_cpu/quantization/test_ddp_core.pynew file mode 100644index 000000000..b7123ab95--- /dev/null+++ b/test/unit/test_cpu/quantization/test_ddp_core.py@@ -0,0 +1,2220 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""CPU-tier tests for the ported single-process DDP tuning surface.++Covers the device-agnostic contract: shard-constrained samplers, plan+resolution with injected free-VRAM data, transport encoding math, pool+distribution layout, gather-on-CPU paths, env defaults, and the+block-runner device-safe concatenation. Mirror/replica behavior on real+CUDA devices lives in the CUDA tier (test_ddp_mirror.py + e2e).+"""++import logging++import pytest+import torch++from auto_round.algorithms.block_runner import _cat_device_safe+from auto_round.algorithms.parallel.data_parallel import (+ _transport_segment,+ distribute_pool,+ parallel_state,+ resolve_ddp_plan,+ resolve_parallel_world,+)+++def _with_policy(q, world=2):+ """Attach a ParallelPolicy-bearing compress_context to a fake quantizer+ (the policy rides on the compress context; env vars are gone)."""+ from types import SimpleNamespace++ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy++ q.compress_context = SimpleNamespace(parallel_policy=ParallelPolicy(world=world))+ return q+++class TestResolveDDPPlan:+ def _plan(self, world, free, footprint=100):+ return resolve_ddp_plan(+ world,+ torch.device("cuda", 0), # device OBJECTS only -- no CUDA runtime touched+ 8,+ visible_devices=[0, 1, 2, 3],+ vram_free_bytes={torch.device("cuda", i): free[i] for i in range(4)},+ mirror_footprint_bytes=footprint,+ margin_bytes=0, # toy byte values in these tests+ )++ def test_full_world_when_vram_fits(self):+ plan = self._plan(4, [1000, 1000, 1000, 1000])+ assert plan.enabled and plan.world == 4+ assert plan.shard_size == 2++ def test_device_without_mirror_fit_is_kept_with_warning_note(self):+ plan = self._plan(4, [1000, 10, 1000, 1000]) # cuda:1 estimate too small+ assert plan.enabled+ assert torch.device("cuda", 1) in plan.devices+ assert plan.world == 4 # the VRAM check is advisory: the requested world stays+ assert any("low free VRAM on" in n and "device kept" in n for n in plan.notes)++ def test_all_devices_low_vram_still_engages_full_world(self):+ plan = self._plan(4, [5, 5, 5, 5], footprint=100)+ assert plan.enabled and plan.world == 4+ assert sum(1 for n in plan.notes if "low free VRAM on" in n) == 3 # mirrors only, not home++ def test_batch_smaller_than_world_demotes_to_pow2(self):+ from auto_round.algorithms.parallel.data_parallel import resolve_ddp_plan as _rdp++ def _resolve(batch, world):+ return _rdp(+ world,+ torch.device("cuda", 0),+ batch,+ visible_devices=[0, 1, 2, 3],+ vram_free_bytes={torch.device("cuda", i): 1 << 40 for i in range(4)},+ mirror_footprint_bytes=1,+ margin_bytes=0,+ )++ # batch 1 cannot shard: serial (world collapses to 1; the reason+ # goes to the log)+ p = _resolve(1, 4)+ assert p.world == 1+ # batch 3, world 4 -> halved to 2 (3 >= 2, one replica gets 2 samples)+ p = _resolve(3, 4)+ assert p.world == 2 and any("world set to 2" in str(n) for n in p.notes)+ # batch 6, world 4 -> no demotion (6 >= 4)+ p = _resolve(6, 4)+ assert p.world == 4+ # batch divisible by world stays+ p = _resolve(8, 4)+ assert p.world == 4++ def test_batch_demotion_is_exact_when_pow2_floor_off(self):+ from auto_round.algorithms.parallel.data_parallel import resolve_ddp_plan as _rdp++ def _resolve(batch, world):+ return _rdp(+ world,+ torch.device("cuda", 0),+ batch,+ visible_devices=[0, 1, 2, 3],+ vram_free_bytes={torch.device("cuda", i): 1 << 40 for i in range(4)},+ mirror_footprint_bytes=1,+ margin_bytes=0,+ pow2_floor=False,+ )++ # iters=0 worlds carry no pow2 constraint: the demotion is exact+ p = _resolve(3, 4)+ assert p.world == 3 and any("world set to 3" in str(n) for n in p.notes)+ p = _resolve(6, 3)+ assert p.world == 3 # no demotion+ p = _resolve(1, 4)+ assert p.world == 1 # serial: a replica needs at least one sample++ def test_non_power_of_two_request_resolves_but_caller_gates(self):+ # resolve itself is policy-free: a non-pow2 world with enough+ # visible devices resolves enabled; the engagement resolver and the+ # CLI policy entry enforce the pow2 rule only for iters > 0 (the+ # gradient exchange needs it; iters=0 runs have no exchange)+ plan = resolve_ddp_plan(+ 3,+ torch.device("cuda", 0),+ 12,+ visible_devices=[0, 1, 2],+ vram_free_bytes={torch.device("cuda", i): 1 << 30 for i in range(3)},+ mirror_footprint_bytes=1,+ margin_bytes=0,+ )+ assert plan.world == 3 and plan.enabled+++class TestResolveParallelWorld:+ """Policy entry: off|auto|N resolution (the CLI delegates here)."""++ def test_off_returns_one(self):+ assert resolve_parallel_world("off", 8) == 1+ assert resolve_parallel_world(None, 8) == 1+ assert resolve_parallel_world("", 8) == 1++ def test_auto_takes_largest_pow2_fitting_visible_at_iters_above_zero(self):+ assert resolve_parallel_world("auto", 8, iters=10) == 8+ assert resolve_parallel_world("auto", 6, iters=10) == 4+ assert resolve_parallel_world("auto", 5, iters=10) == 4++ def test_auto_takes_all_visible_at_iters_zero(self):+ # no gradient exchange runs at iters=0, so every visible device works+ assert resolve_parallel_world("auto", 6, iters=0) == 6+ assert resolve_parallel_world("auto", 8, iters=0) == 8++ def test_auto_needs_two_visible(self):+ with pytest.raises(RuntimeError, match="at least 2 visible CUDA devices"):+ resolve_parallel_world("auto", 1)++ def test_explicit_valid_pow2(self):+ assert resolve_parallel_world("2", 4) == 2+ assert resolve_parallel_world("4", 4) == 4++ def test_explicit_non_pow2_rejected_at_iters_above_zero(self):+ with pytest.raises(RuntimeError, match="power of two"):+ resolve_parallel_world("6", 8, iters=10)++ def test_explicit_non_pow2_allowed_at_iters_zero(self):+ assert resolve_parallel_world("6", 8, iters=0) == 6+ assert resolve_parallel_world("3", 3, iters=0) == 3++ def test_explicit_below_two_rejected(self):+ with pytest.raises(RuntimeError, match="at least 2"):+ resolve_parallel_world("1", 8)++ def test_explicit_over_visible_fails_loudly(self):+ with pytest.raises(RuntimeError, match="requires at least 8 visible CUDA devices, got 4"):+ resolve_parallel_world("8", 4)++ def test_garbage_rejected(self):+ with pytest.raises(RuntimeError, match=r"accepts off\|auto\|N"):+ resolve_parallel_world("yes", 8)+++class TestVisibleCountShortfall:+ def test_visible_fewer_than_requested_world_reports_shortfall(self):+ plan = resolve_ddp_plan(+ 4,+ torch.device("cuda", 0),+ 8,+ visible_devices=[0, 1], # home + one other: fewer than the requested world+ vram_free_bytes=None,+ mirror_footprint_bytes=None,+ )+ assert not plan.enabled+ assert any("fewer than the requested world" in n for n in plan.notes)+++class TestTransportMath:+ def test_fp32_passthrough(self):+ t = torch.randn(16)+ out = _transport_segment(t, t.device)+ assert out is t++ def test_cross_device_copy_is_dtype_preserving(self):+ t = torch.randn(16) if not torch.cuda.is_available() else torch.randn(16, device="cuda:0")+ out = _transport_segment(t, torch.device("cpu"))+ assert out.dtype == t.dtype and out.shape == t.shape+++class TestDistributePool:+ def test_plan_layout_covers_pool_exactly(self):+ # distribute_pool gives device r the contiguous range [r*shard,(r+1)*shard);+ # on uniform CPU devices that is a no-op, so pin the layout arithmetic+ # through the plan the pool follows+ from auto_round.algorithms.parallel.data_parallel import resolve_ddp_plan++ plan = resolve_ddp_plan(+ 4,+ torch.device("cuda", 0),+ 8,+ visible_devices=[0, 1, 2, 3], # avoids torch.cuda.device_count() on CUDA-less hosts+ vram_free_bytes={torch.device("cuda", i): 1 << 40 for i in range(4)},+ mirror_footprint_bytes=1,+ margin_bytes=0,+ )+ assert plan.world == 4+ assert plan.shard_size * plan.world == 8+ # device objects in the plan are cuda:N (unusable on CUDA-less hosts);+ # exercise the scatter loop with the degenerate all-cpu device list+ distribute_pool([torch.zeros(2) for _ in range(8)], [torch.device("cpu")] * 4)+ # indivisible / too-small pools are left alone by contract (serial cats handle them)+ distribute_pool([torch.zeros(2)], [torch.device("cpu")] * 4)+++class TestSingleDevicePlacement:+ """Parallel tuning tunes whole-block mirrors: a block whose weights span+ several CUDA devices fails the resolver (validation, never a gather)."""++ @staticmethod+ def _param(dev, n=8):+ from types import SimpleNamespace++ return SimpleNamespace(device=torch.device(dev), numel=lambda: n, element_size=lambda: 4)++ @staticmethod+ def _quantizer():+ from types import SimpleNamespace++ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)+ q._get_scaler = lambda: None+ return q++ def test_block_spanning_two_cuda_devices_declines(self, monkeypatch):+ from types import SimpleNamespace++ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_++ block = SimpleNamespace(parameters=lambda: iter([self._param("cuda:0"), self._param("cuda:1")]))+ with pytest.raises(RuntimeError, match="spans 2 CUDA devices"):+ resolve_tune_ddp_plan_(_with_policy(self._quantizer()), block, [torch.zeros(1)], None, "cuda:0")++ def test_non_cuda_accelerator_home_is_eligible(self, monkeypatch):+ """cuda/xpu/hpu homes are eligible; a fake xpu home with explicit+ devices resolves a plan on CPU (no xpu runtime touched)."""+ from auto_round.algorithms.parallel import data_parallel as dp+ from auto_round.algorithms.parallel.data_parallel import resolve_ddp_plan++ free_map = dp._accel_free_bytes_map("xpu") # None on a box without xpu+ block = SimpleNamespace(+ parameters=lambda: iter(+ [SimpleNamespace(device=torch.device("xpu", 0), numel=lambda: 8, element_size=lambda: 4)]+ ),+ modules=lambda: iter([]),+ )+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)+ q._get_scaler = lambda: None+ # plan resolution itself works for an xpu home with visible devices+ plan = resolve_ddp_plan(+ 2,+ torch.device("xpu", 0),+ 2,+ visible_devices=[0, 1],+ vram_free_bytes=free_map,+ mirror_footprint_bytes=1,+ )+ assert plan.enabled and plan.world == 2+ assert all(d.type == "xpu" for d in plan.devices)+ # and the full resolver reports (decline, without raising) only on+ # real blockers:+ # an xpu home on a box without xpu resolves without the CUDA-only error+ try:+ resolved = dp.resolve_tune_ddp_plan_(_with_policy(q), block, [torch.zeros(1)], None, "xpu:0")+ except RuntimeError as e:+ assert "not a supported accelerator" not in str(e)+ resolved = None+ if resolved is not None:+ assert resolved.world >= 1++ def test_cpu_resident_weights_pass_span_rule(self, monkeypatch):+ """CPU-pinned subtrees alongside the CUDA home are legal placement."""+ from types import SimpleNamespace++ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_++ block = SimpleNamespace(+ parameters=lambda: iter([self._param("cpu"), self._param("cuda:0")]),+ modules=list,+ )+ # the span rule passes; the resolve then proceeds (it may decline+ # later on the free-VRAM probe, which is CPU-safe) -- the assertion+ # checks the decline list for the placement reason+ try:+ resolve_tune_ddp_plan_(_with_policy(self._quantizer()), block, [torch.zeros(1)], None, "cuda:0")+ except RuntimeError as e:+ assert "spans" not in str(e)++ def test_iters0_engages_non_pow2_world(self, monkeypatch):+ """A world=3 plan engages when the tune loop is empty (iters=0):+ no gradient exchange runs, so the halving-doubling pow2 rule lifts."""+ from auto_round.algorithms.parallel import data_parallel as dp++ q = SimpleNamespace(iters=0, gradient_accumulate_steps=1, enable_lfq=False)+ q._get_scaler = lambda: None+ _with_policy(q, world=3)+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 3)+ monkeypatch.setattr(+ dp,+ "_accel_free_bytes_map",+ lambda _t: {torch.device("cuda", i): 1 << 40 for i in range(3)},+ )+ block = SimpleNamespace(+ parameters=lambda: iter(+ [SimpleNamespace(device=torch.device("cuda", 0), numel=lambda: 8, element_size=lambda: 4)]+ ),+ modules=lambda: iter([]),+ )+ plan = dp.resolve_tune_ddp_plan_(q, block, [torch.zeros(1)] * 3, None, "cuda:0", log=False)+ assert plan.enabled and plan.world == 3++ def test_iters_above_zero_declines_non_pow2_world(self, monkeypatch):+ """The same world=3 with a live tune loop raises: the gradient+ exchange (recursive halving-doubling) needs a power-of-two world."""+ from auto_round.algorithms.parallel import data_parallel as dp++ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)+ q._get_scaler = lambda: None+ _with_policy(q, world=3)+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 3)+ monkeypatch.setattr(+ dp,+ "_accel_free_bytes_map",+ lambda _t: {torch.device("cuda", i): 1 << 40 for i in range(3)},+ )+ block = SimpleNamespace(+ parameters=lambda: iter(+ [SimpleNamespace(device=torch.device("cuda", 0), numel=lambda: 8, element_size=lambda: 4)]+ ),+ modules=lambda: iter([]),+ )+ with pytest.raises(RuntimeError, match="power of two"):+ dp.resolve_tune_ddp_plan_(q, block, [torch.zeros(1)] * 3, None, "cuda:0", log=False)++ def test_ddp_engagement_raises_dynamo_cache_limit_by_world(self, monkeypatch):+ """Engaging a parallel plan bumps the dynamo cache limit to base x world."""+ from auto_round.algorithms.parallel import data_parallel as dp+ from auto_round.utils import device as _dev++ calls = []+ monkeypatch.setattr(_dev, "_bump_dynamo_cache_limit", lambda n=None: calls.append(n))+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)+ q._get_scaler = lambda: None+ _with_policy(q)+ # fake a two-device visible set (this host has at most one CUDA device)+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 2)+ monkeypatch.setattr(+ dp, "_accel_free_bytes_map", lambda _t: {torch.device("cuda", 0): 1 << 40, torch.device("cuda", 1): 1 << 40}+ )+ block = SimpleNamespace(+ parameters=lambda: iter(+ [SimpleNamespace(device=torch.device("cuda", 0), numel=lambda: 8, element_size=lambda: 4)]+ ),+ modules=lambda: iter([]),+ )+ plan = dp.resolve_tune_ddp_plan_(q, block, [torch.zeros(1)] * 2, None, "cuda:0", log=False)+ assert plan.world == 2+ assert calls and calls[-1] == 16 * 2 # default env base 16 x engaged world++ def test_accumulation_engages(self, monkeypatch):+ """gradient_accumulate_steps > 1 no longer gates the lane."""+ from auto_round.algorithms.parallel import data_parallel as dp+ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_++ q = SimpleNamespace(iters=10, gradient_accumulate_steps=2, enable_lfq=False)+ q._get_scaler = lambda: None+ _with_policy(q)+ # fake a two-device visible set (this host has at most one CUDA device)+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 2)+ monkeypatch.setattr(+ dp, "_accel_free_bytes_map", lambda _t: {torch.device("cuda", 0): 1 << 40, torch.device("cuda", 1): 1 << 40}+ )+ block = SimpleNamespace(+ parameters=lambda: iter(+ [SimpleNamespace(device=torch.device("cuda", 0), numel=lambda: 8, element_size=lambda: 4)]+ ),+ modules=lambda: iter([]),+ )+ plan = resolve_tune_ddp_plan_(q, block, [torch.zeros(1)] * 2, None, "cuda:0", log=False)+ # engages with a requested world; the accumulation gate is gone+ assert plan.world == 2+ assert not any("accumulate" in str(n) for n in plan.notes)++ def test_accumulated_grad_sign_matches_serial(self):+ """Two replicas, sum-reduced shard losses; the full-value exchange+ leaves each replica with the averaged gradient -- a positive rescale+ of the serial accumulated (summed) gradient, so sign-SGD updates and+ momentum directions match serial."""+ from auto_round.algorithms.parallel import data_parallel as dp++ world = 2+ # replicas hold the SAME parameters (mirrors); each accumulates the+ # sum-reduced gradient over its own data shard. Shard grads derived+ # analytically: grad of (x[shard]**2).sum() lives on the shard slice.+ x = torch.randn(16, requires_grad=False)+ grad0 = torch.zeros(16)+ grad0[:8] = 2 * x[:8]+ grad1 = torch.zeros(16)+ grad1[8:] = 2 * x[8:]+ bufs = [grad0.clone(), grad1.clone()]+ dp.halving_doubling_allreduce(bufs, scale=1.0 / world)+ serial_accumulated = 2 * x # serial: grads summed over both shards+ for buf in bufs:+ # exchanged buffer x world == serial accumulated gradient+ assert torch.allclose(buf * world, serial_accumulated, atol=1e-6)+++class TestCatDeviceSafe:+ def test_same_device_is_plain_cat(self):+ parts = [torch.zeros(2, 3), torch.ones(2, 3)]+ out = _cat_device_safe(parts, dim=0)+ assert torch.equal(out, torch.cat(parts, dim=0))++ def test_empty_selection_raises(self):+ with pytest.raises(ValueError):+ _cat_device_safe([], dim=0)+++class TestPolicyDefaults:+ """The policy rides on the compress context; a quantizer without one is+ serial (off), and the resolver reads the policy's world."""++ def test_quantizer_without_context_defaults_to_off(self):+ from types import SimpleNamespace++ from auto_round.algorithms.parallel.data_parallel import _quantizer_policy++ assert _quantizer_policy(SimpleNamespace()).world == 1++ def test_policy_world_round_trip(self):+ from types import SimpleNamespace++ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy, _quantizer_policy++ q = _with_policy(SimpleNamespace(), world=4)+ assert _quantizer_policy(q).world == 4+ assert isinstance(_quantizer_policy(q), ParallelPolicy)++ def test_declared_lfq_constraint_declines(self):+ """Eligibility constraints are declared by the quantizer+ (parallel_constraints); LFQ is a real blocker."""+ from types import SimpleNamespace++ import torch++ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_++ q = _with_policy(SimpleNamespace(iters=10, gradient_accumulate_steps=1))+ q.parallel_constraints = lambda: {"enable_lfq": True}+ block = torch.nn.Sequential(torch.nn.Linear(4, 4))+ with pytest.raises(RuntimeError, match="enable_lfq"):+ resolve_tune_ddp_plan_(q, block, [torch.zeros(1)], None, "cpu")++ def test_base_quantizer_declares_constraints(self):+ """BaseQuantizer.parallel_constraints reports LFQ without the resolver+ introspecting quantizer internals; a grad scaler is no constraint."""+ from auto_round.algorithms.quantization.base import BaseQuantizer++ q = BaseQuantizer.__new__(BaseQuantizer)+ assert q.parallel_constraints() == {"enable_lfq": False}+ q.enable_lfq = True+ assert q.parallel_constraints()["enable_lfq"] is True+++class TestTupleKwargSlicing:+ """transformers-v5 rope arrives as position_embeddings=(cos, sin) with a+ per-sample batch dim; shard/batch forwards smaller than the cached batch+ must slice tuple-of-tensors kwargs elementwise or crash in rope."""++ def _runner(self, n=8, batch_size=4):+ from auto_round.algorithms.block_runner import BlockForwardRunner++ inputs = [torch.randn(1, 6, 4) for _ in range(n)]+ return (+ BlockForwardRunner(+ batch_dim=0, batch_size=batch_size, device=torch.device("cpu"), cache_device="cpu", amp=False+ ),+ inputs,+ )++ def test_tuple_kwargs_sliced_to_batch_indices(self):+ runner, inputs = self._runner()+ cos = torch.randn(8, 6, 2)+ sin = torch.randn(8, 6, 2)+ _, others = runner.select_batch(inputs, {"position_embeddings": (cos, sin)}, [3, 5])+ assert isinstance(others["position_embeddings"], tuple)+ assert others["position_embeddings"][0].shape == (2, 6, 2)+ assert others["position_embeddings"][1].shape == (2, 6, 2)+ assert torch.equal(others["position_embeddings"][0][0], cos[3])+ assert torch.equal(others["position_embeddings"][0][1], cos[5])++ def test_broadcast_tuple_elements_pass_through(self):+ runner, inputs = self._runner()+ table = torch.randn(1, 6, 2) # broadcast-shaped: not per-sample+ _, others = runner.select_batch(inputs, {"position_embeddings": (table, table)}, [3, 5])+ assert others["position_embeddings"][0] is table # unsliced, broadcastable++ def test_mixed_and_non_tensor_tuples_untouched(self):+ runner, inputs = self._runner()+ val = (torch.randn(8, 6, 2), "flag")+ _, others = runner.select_batch(inputs, {"weird": val}, [3, 5])+ assert others["weird"] is val # mixed tuple treated as opaque+++class TestSharedCacheShardPick:+ """shared_cache_keys kwargs arrive as ONE ENTRY PER BATCH (list[n_pool/batch]).+ A sub-batch draw (DDP shard) must pick the owning batch's entry and slice its+ within-batch rows; full-batch draws keep the legacy pick exactly."""++ def _runner(self, batch_size=8):+ from auto_round.algorithms.block_runner import BlockForwardRunner++ inputs = [torch.randn(1, 6, 4) for _ in range(128)]+ return (+ BlockForwardRunner(+ batch_dim=0,+ batch_size=batch_size,+ device=torch.device("cpu"),+ cache_device="cpu",+ amp=False,+ shared_cache_keys=("position_embeddings",),+ ),+ inputs,+ )++ def _pe(self):+ # 16 batch entries, each an (cos, sin) tuple of [8, S, D] per-sample tensors+ return [(torch.full((8, 6, 2), float(b)), torch.full((8, 6, 2), -float(b))) for b in range(16)]++ def test_tensor_and_list_indices_pick_identically(self):+ # ``indices`` normally ride on the inputs' device (a CUDA tensor on GPU+ # runs); the host-int hoist must yield the same picks as a plain list+ runner, inputs = self._runner()+ pe = self._pe()+ t_in, t_other = runner.select_batch(inputs, {"position_embeddings": pe}, torch.tensor([32, 33]))+ l_in, l_other = runner.select_batch(inputs, {"position_embeddings": pe}, [32, 33])+ assert torch.equal(t_in, l_in)+ assert torch.equal(t_other["position_embeddings"][0], l_other["position_embeddings"][0])+ assert torch.equal(t_other["position_embeddings"][1], l_other["position_embeddings"][1])++ def test_shard_picks_owning_batch_and_rows(self):+ runner, inputs = self._runner()+ pe = self._pe()+ _, others = runner.select_batch(inputs, {"position_embeddings": pe}, [32, 33])+ cos, sin = others["position_embeddings"]+ assert cos.shape == (2, 6, 2) and sin.shape == (2, 6, 2)+ assert torch.all(cos == 4.0) and torch.all(sin == -4.0) # entry 4, rows 0-1++ def test_shard_wrapping_rows_across_batch_boundary_fails_safe(self):+ runner, inputs = self._runner()+ pe = self._pe()+ _, others = runner.select_batch(inputs, {"position_embeddings": pe}, [7, 8])+ # batch 0 entry (indices[0]=7), rows 7 and 0 -- within-entry slice+ cos, _ = others["position_embeddings"]+ assert cos.shape == (2, 6, 2) and torch.all(cos == 0.0)++ def test_full_batch_draw_keeps_legacy_pick(self):+ runner, inputs = self._runner()+ pe = self._pe()+ _, others = runner.select_batch(inputs, {"position_embeddings": pe}, list(range(8, 16)))+ # legacy: multi-index full-batch -> val[0], untouched (batch 0's entry, 8 rows)+ cos, _ = others["position_embeddings"]+ assert isinstance(cos, torch.Tensor) and cos.shape[0] == 8 and torch.all(cos == 0.0)++ def test_single_index_sub_batch_draw_per_batch_list(self):+ runner, inputs = self._runner()+ pe = self._pe()+ _, others = runner.select_batch(inputs, {"position_embeddings": pe}, [5])+ # per-batch list: sample 5 -> batch 0 entry, row 5+ cos, _ = others["position_embeddings"]+ assert cos.shape == (1, 6, 2) and torch.all(cos == 0.0)++ def test_per_sample_list_sub_batch_draw(self):+ runner, inputs = self._runner()+ # per-sample layout: 128 entries, each a [1, 6, 2] tensor+ ps = [torch.full((1, 6, 2), float(i)) for i in range(128)]+ _, others = runner.select_batch(inputs, {"position_embeddings": ps}, [32, 33])+ assert others["position_embeddings"].shape == (2, 6, 2)+ assert torch.all(others["position_embeddings"][0] == 32.0)++ @pytest.mark.parametrize("rank_batch", [1, 2])+ def test_rank_runner_with_its_batch_share(self, rank_batch):+ # a thread rank tunes with its share of the batch (8 // world), while+ # the cache was collected 8 per entry: the collection batch comes from+ # the cache, not from the rank runner's batch_size (regression: at 8+ # ranks a 1-row draw got a whole 8-row entry)+ runner, inputs = self._runner(batch_size=rank_batch)+ pe = self._pe()+ draw = [37, 38][:rank_batch]+ _, others = runner.select_batch(inputs, {"position_embeddings": pe}, draw)+ cos, sin = others["position_embeddings"]+ assert cos.shape == (rank_batch, 6, 2) and torch.all(cos == 4.0) and torch.all(sin == -4.0)++ def test_broadcast_entry_passes_through(self):+ # [1, S, D] tables broadcast over the batch: never row-sliced (on CUDA an+ # out-of-range index_select is a device-side assert, not an exception)+ runner, inputs = self._runner(batch_size=1)+ pe = [(torch.full((1, 6, 2), float(b)), torch.full((1, 6, 2), -float(b))) for b in range(16)]+ _, others = runner.select_batch(inputs, {"position_embeddings": pe}, [37])+ cos, _ = others["position_embeddings"]+ assert cos.shape == (1, 6, 2) and torch.all(cos == 4.0)+++class TestLoggingGlobals:+ def test_engaged_log_globals_initialized(self):+ """Regression: the env-strip once dropped the module-level+ ``_ENGAGED_LOGGED_SIG`` init, and the ``global`` read in+ resolve_tune_ddp_plan_ then NameError'd -- but only on GPU runs,+ because a CPU home declines before the sig block (CPU tests cannot+ reach the engaged path)."""+ from auto_round.algorithms.parallel import data_parallel as dp+ from auto_round.algorithms.quantization.search_dispatch import _ENGAGED_LOGGED++ assert dp._ENGAGED_LOGGED_SIG is None+ # _coll_mirror_setup_logged was removed with its once-per-process+ # INFO line; search_dispatch._ENGAGED_LOGGED remains the registry+ assert isinstance(_ENGAGED_LOGGED, set)+++class TestActivationPricing:+ """Mirror pricing charges the per-forward activation working set on top+ of the weights (estimate_tuning_block_mem's SIX-value return contract).++ Regression: the rebase onto main left the DDP seam unpacking the+ estimator's pre-main FOUR-tuple, so every block logged ``activation+ pricing skipped (too many values to unpack)`` at INFO and the plan+ budget silently dropped to weights-only."""++ @staticmethod+ def _block():+ layer = torch.nn.Linear(32, 32, bias=False)+ layer.global_name = "m.blk.proj"+ layer.to_quantized = True+ layer.bits = 4+ layer.act_bits = 16+ return torch.nn.Sequential(layer)++ @staticmethod+ def _quantizer():+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)+ q._get_scaler = lambda: None+ return _with_policy(q)++ def test_pricing_succeeds_and_adds_bytes_to_the_budget(self, monkeypatch, caplog, _autoround_log_propagate):+ """The resolver's pricing call runs (no skip warning) and its bytes+ enter the mirror budget: cuda:1 fits the weights-only mirror (plus+ the 2GiB margin) but misses once the activation charge (>= the flat+ 1GiB ``additional_memory``) lands on top."""+ from auto_round.algorithms.parallel import data_parallel as dp++ block = self._block()+ weight_bytes = sum(p.numel() * p.element_size() for p in block.parameters())+ free = {+ torch.device("cuda", 0): 1 << 40,+ # weights + margin + 64MiB slack: passes weights-only pricing,+ # fails once >=1GiB activation bytes are charged+ torch.device("cuda", 1): weight_bytes + (2 << 30) + (64 << 20),+ }+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 2)+ monkeypatch.setattr(dp, "_accel_free_bytes_map", lambda _t: free)+ inputs = [torch.randn(1, 16, 32) for _ in range(2)]+ with caplog.at_level("WARNING"):+ plan = dp.resolve_tune_ddp_plan_(self._quantizer(), block, inputs, None, "cuda:0", log=False)+ assert plan.enabled and plan.world == 2+ assert not any("activation pricing skipped" in r.message for r in caplog.records)+ assert any(+ "low free VRAM on cuda:1" in n for n in plan.notes+ ), "activation bytes never entered the budget (weights-only pricing)"++ def test_estimator_failure_is_a_warning(self, monkeypatch, caplog, _autoround_log_propagate):+ """A failed estimate is a degraded state (weights-only budget): the+ skip line must be WARNING-level, not buried at INFO."""+ from auto_round.algorithms.parallel import data_parallel as dp+ from auto_round.utils import device as _dev++ def _boom(*_a, **_k):+ raise RuntimeError("estimator exploded")++ monkeypatch.setattr(_dev, "estimate_tuning_block_mem", _boom)+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 2)+ monkeypatch.setattr(+ dp, "_accel_free_bytes_map", lambda _t: {torch.device("cuda", i): 1 << 40 for i in range(2)}+ )+ inputs = [torch.randn(1, 16, 32) for _ in range(2)]+ with caplog.at_level("WARNING"):+ plan = dp.resolve_tune_ddp_plan_(self._quantizer(), self._block(), inputs, None, "cuda:0", log=False)+ assert plan.enabled and plan.world == 2+ assert any("activation pricing skipped" in r.message and r.levelno == logging.WARNING for r in caplog.records)+++class TestEarlyDeviceMapEligibility:+ """F3: the multi-device-map ineligibility fires at policy resolution+ (before model load) with the same message the runtime resolver raises at+ block 0; the runtime check remains the second line of defense."""++ def test_multi_device_map_with_live_loop_raises_before_load(self):+ from auto_round.algorithms.parallel.data_parallel import check_parallel_device_map_compat_++ with pytest.raises(RuntimeError, match="spans 2 CUDA devices; parallel tuning requires"):+ check_parallel_device_map_compat_(4, {"a": "cuda:0", "b": "cuda:1"}, iters=10)++ def test_message_lists_the_fixes(self):+ from auto_round.algorithms.parallel.data_parallel import check_parallel_device_map_compat_++ with pytest.raises(RuntimeError, match="--data_parallel off"):+ check_parallel_device_map_compat_(4, {"a": "cuda:0", "b": "cuda:1"}, iters=10)++ def test_single_device_map_passes(self):+ from auto_round.algorithms.parallel.data_parallel import check_parallel_device_map_compat_++ check_parallel_device_map_compat_(4, {"a": "cuda:0"}, iters=10) # one device+ check_parallel_device_map_compat_(4, "0", iters=10) # CLI default+ check_parallel_device_map_compat_(4, None, iters=10) # no map++ def test_iters0_and_world1_are_not_cli_blocked(self):+ from auto_round.algorithms.parallel.data_parallel import check_parallel_device_map_compat_++ multi = {"a": "cuda:0", "b": "cuda:1"}+ check_parallel_device_map_compat_(4, multi, iters=0) # no tune loop: runtime decides+ check_parallel_device_map_compat_(1, multi, iters=10) # lane off++ def test_cpu_entries_do_not_count(self):+ from auto_round.algorithms.parallel.data_parallel import check_parallel_device_map_compat_++ check_parallel_device_map_compat_(4, {"a": "cpu", "b": "cpu"}, iters=10)++ def test_cli_policy_resolution_calls_the_early_check(self):+ """Wiring: the CLI calls the early check right after resolving the+ world, before the model loads."""+ import inspect++ from auto_round.cli import main as cli_main++ src = inspect.getsource(cli_main)+ assert "check_parallel_device_map_compat_" in src+ assert src.index("check_parallel_device_map_compat_(") > src.index("resolve_parallel_world(")++ def test_runtime_span_check_still_exists(self):+ """Second line of defense: the resolver's per-block span check."""+ from types import SimpleNamespace++ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_++ block = SimpleNamespace(+ parameters=lambda: iter(+ [+ SimpleNamespace(device=torch.device("cuda:0"), numel=lambda: 8, element_size=lambda: 4),+ SimpleNamespace(device=torch.device("cuda:1"), numel=lambda: 8, element_size=lambda: 4),+ ]+ )+ )+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)+ q._get_scaler = lambda: None+ with pytest.raises(RuntimeError, match="spans 2 CUDA devices"):+ resolve_tune_ddp_plan_(_with_policy(q), block, [torch.zeros(1)], None, "cuda:0")+++class TestZeroShotParallelNotice:+ """F4: an explicit --data_parallel N on the zero-shot RTN path+ (no calibration forward) emits exactly ONE INFO line; auto/off stay+ silent; nothing refuses to run."""++ def test_explicit_request_logs_one_info_line(self, caplog, _autoround_log_propagate):+ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy, log_zero_shot_parallel_noop_++ with caplog.at_level("INFO"):+ log_zero_shot_parallel_noop_(ParallelPolicy(world=4, source="explicit"))+ infos = [r for r in caplog.records if "has no effect on this run" in r.message]+ assert len(infos) == 1 and infos[0].levelno == logging.INFO+ assert "--data_parallel 4" in infos[0].message+ assert "zero-shot RTN needs no calibration forward" in infos[0].message+ assert "quantizing serially" in infos[0].message++ def test_auto_and_off_are_silent(self, caplog, _autoround_log_propagate):+ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy, log_zero_shot_parallel_noop_++ with caplog.at_level("INFO"):+ log_zero_shot_parallel_noop_(ParallelPolicy(world=8, source="auto"))+ log_zero_shot_parallel_noop_(ParallelPolicy(world=1, source="off"))+ log_zero_shot_parallel_noop_(None)+ assert not any("has no effect" in r.message for r in caplog.records)++ def test_zero_shot_path_wires_the_notice(self):+ """Wiring: _quantize_zero_shot consults the compress-context policy."""+ import inspect++ from auto_round.compressors import orchestrator++ src = inspect.getsource(orchestrator.CompressionOrchestrator._quantize_zero_shot)+ assert "log_zero_shot_parallel_noop_" in src+++class TestPeakVramReplicaDevices:+ """F5: the final peak-VRAM summary must cover DDP replica devices engaged+ outside the device map (the monitor's callers pass the device-map list)."""++ def test_registered_devices_are_sampled_beyond_the_caller_list(self, monkeypatch, _memory_monitor_clean):+ from auto_round.utils import device as dev_mod++ class _FakeMgr:+ type = "cuda"++ @staticmethod+ def is_available():+ return True++ @staticmethod+ def memory_reserved(i):+ return (i + 1) * 1024**3 # cuda:i reports (i+1) GiB reserved++ mm = _memory_monitor_clean+ mm.register_extra_devices([torch.device("cuda", 3)])+ monkeypatch.setattr(dev_mod, "get_current_device_manager", lambda: _FakeMgr())+ mm.update(["cuda:0"]) # the device-map list, WITHOUT the replica device+ assert "0" in mm.peak_vram and mm.peak_vram["0"] == 1.0+ assert "3" in mm.peak_vram and mm.peak_vram["3"] == 4.0 # replica now visible+ assert "cuda:3" in mm.get_summary() or "'3': 4.0GB" in mm.get_summary()++ def test_registration_survives_reset_and_cpu_entries_are_ignored(self, _memory_monitor_clean):+ mm = _memory_monitor_clean+ mm.register_extra_devices([torch.device("cuda", 2), torch.device("cpu"), None])+ assert mm._extra_devices == {"cuda:2"}+ mm.reset()+ assert mm._extra_devices == {"cuda:2"} # peaks reset, registration kept++ def test_engaging_plan_registers_its_devices(self, monkeypatch, _memory_monitor_clean):+ from auto_round.algorithms.parallel import data_parallel as dp++ mm = _memory_monitor_clean+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 2)+ monkeypatch.setattr(+ dp, "_accel_free_bytes_map", lambda _t: {torch.device("cuda", i): 1 << 40 for i in range(2)}+ )+ block = SimpleNamespace(+ parameters=lambda: iter(+ [SimpleNamespace(device=torch.device("cuda", 0), numel=lambda: 8, element_size=lambda: 4)]+ ),+ modules=lambda: iter([]),+ )+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)+ q._get_scaler = lambda: None+ _with_policy(q)+ plan = dp.resolve_tune_ddp_plan_(q, block, [torch.zeros(1)] * 2, None, "cuda:0", log=False)+ assert plan.enabled+ assert mm._extra_devices == {"cuda:0", "cuda:1"}++ def test_serial_plan_registers_nothing(self, _memory_monitor_clean):+ from auto_round.algorithms.parallel.data_parallel import resolve_ddp_plan++ assert resolve_ddp_plan(1, torch.device("cuda", 0), 8).world == 1+ assert _memory_monitor_clean._extra_devices == set()+++class TestPerfPrepSpan:+ """F6: the per-block phases line must account for the region between the+ reload mark and the first tick (materialize + dtype convert ++ dispatch_block placement + pool-placement attach) -- ~8s/block on big+ MoE blocks that tqdm counted but total= missed."""++ def test_prep_span_opens_before_attach_and_prints(self):+ import inspect++ from auto_round.compressors import orchestrator++ src = inspect.getsource(orchestrator.CompressionOrchestrator._quantize_blocks)+ first_prep = src.index('_perf.tick("prep")')+ first_attach = src.index('_perf.tick("attach")')+ assert first_prep < first_attach, "the measured chain must open with the prep span"+ assert "prep=" in src++ def test_each_tick_opens_the_region_it_names(self):+ """tick(key) closes the running span and opens ``key``: each tick must+ sit right before the code it names (a tick placed after its region+ shifts every printed label one phase late)."""+ import inspect++ from auto_round.compressors import orchestrator++ src = inspect.getsource(orchestrator.CompressionOrchestrator._quantize_blocks)+ chain = [+ ('_perf.tick("prep")', "materialize_model_(m)"),+ ('_perf.tick("attach")', "self._attach_pool_placement("),+ ('_perf.tick("compress")', "self.alg_composer.compress_block("),+ ('_perf.tick("swap")', "if q_input is not None:"),+ ('_perf.tick("sweep")', "_mm._deferred_search_inputs = None"),+ ('_perf.tick("hooks")', "accelerate.hooks.remove_hook_from_submodules(m)"),+ ('_perf.tick("pack")', "if self.compress_context.is_immediate_packing:"),+ ('_perf.tick("mv")', "mv_module_from_gpu(m)"),+ ('_perf.tick("clear")', "clear_memory(device_list=device_manager.device_list)"),+ ('_perf.tick("monitor")', "memory_monitor.log_summary()"),+ ('_perf.tick("write")', "_bg_writer.dispatch(_bg_write)"),+ ]+ pos = 0+ for tick, region in chain:+ t = src.index(tick, pos)+ r = src.index(region, t)+ assert (+ src.find("_perf.tick(", t + len(tick)) > r or src.find("_perf.tick(", t + len(tick)) == -1+ ), f"{tick} must directly precede {region!r}"+ pos = r+++class TestEngagedLineBatchMode:+ """F7: the engaged line logs the batch mode and the effective global batch."""++ def test_engaged_line_reports_batch_mode_and_effective_batch(self, monkeypatch, caplog, _autoround_log_propagate):+ """F7: the engaged line logs the batch mode and the effective global+ batch (per_replica scales it by the world)."""+ from auto_round.algorithms.parallel import data_parallel as dp+ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy++ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 2)+ monkeypatch.setattr(+ dp, "_accel_free_bytes_map", lambda _t: {torch.device("cuda", i): 1 << 40 for i in range(2)}+ )+ block = SimpleNamespace(+ parameters=lambda: iter(+ [SimpleNamespace(device=torch.device("cuda", 0), numel=lambda: 8, element_size=lambda: 4)]+ ),+ modules=lambda: iter([]),+ )++ def _q(mode):+ q = SimpleNamespace(+ iters=10,+ gradient_accumulate_steps=1,+ enable_lfq=False,+ calibration_context=SimpleNamespace(batch_size=8),+ )+ q._get_scaler = lambda: None+ q.compress_context = SimpleNamespace(parallel_policy=ParallelPolicy(world=2, batch_mode=mode))+ return q++ with caplog.at_level("INFO"):+ dp._ENGAGED_LOGGED_SIG = None # the engaged line dedups by signature+ dp.resolve_tune_ddp_plan_(_q("split"), block, [torch.zeros(1)] * 32, None, "cuda:0", log=True)+ dp._ENGAGED_LOGGED_SIG = None+ dp.resolve_tune_ddp_plan_(_q("per_replica"), block, [torch.zeros(1)] * 32, None, "cuda:0", log=True)+ engaged = [r.message for r in caplog.records if "[tune-ddp] engaged:" in r.message]+ assert len(engaged) == 2+ assert "batch=split" in engaged[0] and "effective global batch=8" in engaged[0]+ assert "batch=per_replica" in engaged[1] and "effective global batch=16" in engaged[1]+++class TestSyncMirrorsToHome:+ """iters=0 RTN/OptRTN under DDP: the quantizer runs main's code on the HOME+ block and never touches the pool; the engine then syncs the quantized home+ state onto the mirrors (sync_mirrors_to_home, called from the composer's+ _settle_block_pool) so the Step-6 cascade still runs sharded."""++ @staticmethod+ def _block():+ class _L(torch.nn.Linear):+ def __init__(self, i):+ super().__init__(4, 4, bias=False)+ self.global_name = f"m.blk.l{i}"+ self.to_quantized = True+ self.bits = 4+ self.sym = True+ self.data_type = "int"+ self.group_size = 4+ self.act_bits = 16+ self.imatrix = torch.rand(4) + 0.5+ self.imatrix_cnt = 1++ class _B(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.l0 = _L(0)+ self.l1 = _L(1)++ def forward(self, x):+ return self.l0(x) + self.l1(x)++ return _B()++ @staticmethod+ def _ddp_quantizer(quantizer_cls, block, with_pool=True):+ from types import SimpleNamespace++ from auto_round.algorithms.parallel.data_parallel import MirrorPool++ devices = [torch.device("cpu", 0), torch.device("cpu", 1)]+ q = quantizer_cls.__new__(quantizer_cls)+ q._BaseAlgorithm__run_ctx = None # unbound: q.model is None (the driver is stubbed)+ pool = MirrorPool(block, devices) if with_pool else None+ q.parallel_state = SimpleNamespace(plan=SimpleNamespace(world=2, devices=devices), pool=pool, pool_used=False)+ return q, pool++ @staticmethod+ def _qdq(layer):+ with torch.no_grad(): # a REAL quantize-dequantize transform on the home layer+ layer.weight.data = ((layer.weight.data * 4).round() / 4) + 0.125++ @staticmethod+ def _mirror_pairs(block, pool):+ for n, m in block.named_modules():+ if hasattr(m, "global_name"):+ for r in range(pool.world):+ if pool.reps[r] is not block:+ yield pool.mirror_layer(r, n), m++ def test_optrtn_quantize_block_stays_on_home(self, monkeypatch):+ import auto_round.algorithms.quantization.rtn.batched_search as _bs+ from auto_round.algorithms.quantization.rtn.quantizer import OptimizedRTNQuantizer++ block = self._block()+ home_ids = {id(m) for _, m in block.named_modules() if hasattr(m, "global_name")}+ q, pool = self._ddp_quantizer(OptimizedRTNQuantizer, block)+ staged_on = []++ def _staged(layer, disable_opt_rtn=None, defer_search=False):+ staged_on.append((id(layer), defer_search))+ return layer++ def _driver(model, staged, max_batch=None):+ for _name, layer in staged:+ self._qdq(layer)++ q._quantize_layer_via_rtn = _staged+ monkeypatch.setattr(_bs, "run_batched_rtn_search", _driver)+ q.quantize_block(block, None, {}, None, None, None)+ # main's staged lane ran on the HOME layers only, and the quantizer+ # left the pool alone (the sync is the engine's job)+ assert len(staged_on) == 2+ assert all(iid in home_ids and deferred for iid, deferred in staged_on)+ assert q.parallel_state.pool_used is False+ assert all(not torch.equal(lm.weight.data, hm.weight.data) for lm, hm in self._mirror_pairs(block, pool))++ def test_sync_pushes_home_state_and_marks_pool_consumed(self):+ from auto_round.algorithms.parallel.mirror_sync import sync_mirrors_to_home+ from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer++ block = self._block()+ q, pool = self._ddp_quantizer(RTNQuantizer, block)+ for _n, m in block.named_modules():+ if hasattr(m, "global_name"):+ self._qdq(m)+ m.scale = torch.full((4, 1), 0.25)+ assert sync_mirrors_to_home(q, block) is True+ for lm, hm in self._mirror_pairs(block, pool):+ assert torch.equal(lm.weight.data, hm.weight.data)+ assert torch.equal(lm.scale, hm.scale)+ assert q.parallel_state.pool_used is True+ assert not any(hasattr(m, "imatrix") for _n, m in block.named_modules() if hasattr(m, "global_name"))++ def test_sync_without_aligned_pool_is_a_noop(self):+ from auto_round.algorithms.parallel.data_parallel import MirrorPool+ from auto_round.algorithms.parallel.mirror_sync import sync_mirrors_to_home+ from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer++ block = self._block()+ q, _pool = self._ddp_quantizer(RTNQuantizer, block, with_pool=False)+ assert sync_mirrors_to_home(q, block) is False+ # a pool built for another block must not be written either+ other = self._block()+ q.parallel_state.pool = MirrorPool(other, q.parallel_state.plan.devices)+ assert sync_mirrors_to_home(q, block) is False+ assert q.parallel_state.pool_used is False+ assert all(hasattr(m, "imatrix") for _n, m in block.named_modules() if hasattr(m, "global_name"))++ def test_composer_settle_syncs_rtn_family_and_keeps_pool(self):+ from auto_round.algorithms.composer import AlgorithmComposer+ from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer++ block = self._block()+ q, pool = self._ddp_quantizer(RTNQuantizer, block)+ for _n, m in block.named_modules():+ if hasattr(m, "global_name"):+ self._qdq(m)+ released = []+ composer = AlgorithmComposer.__new__(AlgorithmComposer)+ composer.block_quantizer = q+ composer._coll_ctx = type("_Ctx", (), {"release_pool": lambda self: released.append(1)})()+ assert composer._settle_block_pool(block, pool) is pool+ assert released == []+ for lm, hm in self._mirror_pairs(block, pool):+ assert torch.equal(lm.weight.data, hm.weight.data)++ def test_composer_settle_releases_pool_for_other_quantizers(self):+ from auto_round.algorithms.composer import AlgorithmComposer+ from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer++ block = self._block()+ q, pool = self._ddp_quantizer(SignRoundQuantizer, block)+ released = []+ composer = AlgorithmComposer.__new__(AlgorithmComposer)+ composer.block_quantizer = q+ composer._coll_ctx = type("_Ctx", (), {"release_pool": lambda self: released.append(1)})()+ # not the RTN family and nothing consumed the pool: no sync, released+ assert composer._settle_block_pool(block, pool) is None+ assert released == [1]+ assert q.parallel_state.pool_used is False+++class TestRequestedWorldErrors:+ """A requested parallel world is a requirement: infeasible -> RuntimeError, never silent serial."""++ def _fake_quantizer(self):+ from types import SimpleNamespace++ q = SimpleNamespace(+ iters=10,+ gradient_accumulate_steps=1,+ enable_lfq=False,+ )+ q._get_scaler = lambda: None+ return q++ def test_infeasible_world_raises(self, monkeypatch):+ import torch++ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_++ block = torch.nn.Sequential(torch.nn.Linear(4, 4))+ with pytest.raises(RuntimeError, match="ineligible"):+ resolve_tune_ddp_plan_(_with_policy(self._fake_quantizer()), block, [torch.zeros(1)], None, "cpu")++ def test_iters0_is_not_a_decline_reason(self, monkeypatch):+ """The DDP world shards the collection at iters=0 too (campaign+ semantics restored): with a fake quantizer at iters=0 the only+ ineligibility reason left must be the non-CUDA home, never iters."""+ import torch++ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_++ q = self._fake_quantizer()+ q.iters = 0+ block = torch.nn.Sequential(torch.nn.Linear(4, 4))+ with pytest.raises(RuntimeError) as excinfo:+ resolve_tune_ddp_plan_(_with_policy(q), block, [torch.zeros(1)], None, "cpu")+ assert "iters" not in str(excinfo.value)+ assert "not a supported accelerator" in str(excinfo.value)++ def test_no_policy_stays_serial(self, monkeypatch):+ import torch++ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_++ block = torch.nn.Sequential(torch.nn.Linear(4, 4))+ plan = resolve_tune_ddp_plan_(self._fake_quantizer(), block, [torch.zeros(1)], None, "cpu")+ assert plan.world == 1+++def _memory_monitor_clean():+ """Isolate the process-singleton memory monitor's peaks and extra-device+ registration so tests are order-independent."""+ from auto_round.utils.device import memory_monitor++ saved = (+ dict(memory_monitor.peak_vram),+ float(memory_monitor.peak_ram),+ set(getattr(memory_monitor, "_extra_devices", set())),+ )+ memory_monitor.peak_vram = {}+ memory_monitor.peak_ram = 0.0+ memory_monitor._extra_devices = set()+ yield memory_monitor+ memory_monitor.peak_vram, memory_monitor.peak_ram = saved[0], saved[1]+ memory_monitor._extra_devices = saved[2]+++def _autoround_log_propagate():+ """Temporarily enable propagation on the ``autoround`` logger so pytest's+ caplog fixture (handler at the root logger) can capture warnings; the+ logger is configured with propagate=False in production."""+ import logging++ logger = logging.getLogger("autoround")+ original = logger.propagate+ logger.propagate = True+ yield+ logger.propagate = original+++def _expert_layer(name):+ """An exact-nn.Linear module that satisfies _split_expert_batches' gates."""+ m = torch.nn.Linear(4, 4)+ m.global_name = name+ m.to_quantized = True+ m.bits = 4+ m.group_size = 4+ m.sym = True # sym batches under BOTH neuqi and plain opt-RTN; plain asym has no search+ m.act_bits = 16+ return m+++def _expert_moe_block():+ """4 same-key experts + one dense module: the standard sharded-search fixture."""++ class _B(torch.nn.Module):+ def __init__(self):+ super().__init__()+ mods = [+ _expert_layer("m.blk.experts.0.gate_proj"),+ _expert_layer("m.blk.experts.1.gate_proj"),+ _expert_layer("m.blk.experts.2.up_proj"),+ _expert_layer("m.blk.experts.3.up_proj"),+ ]+ dense = _expert_layer("m.blk.mlp.down_proj")+ for i, m in enumerate(mods + [dense]):+ setattr(self, f"m{i}", m)++ return _B()+++class TestMirrorPoolUnit:+ """MirrorPool construction, hook sync, and release."""++ def _block_with_hook(self):+ class _B(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.lin = torch.nn.Linear(4, 4)+ self.lin.global_name = "m.lin"+ self.lin.to_quantized = True+ self.lin.bits = 4+ self.lin.group_size = 4+ self.lin.sym = False+ self.lin.act_bits = 16++ def forward(self, x):+ return self.lin(x)++ b = _B()+ seen = []+ b.lin.register_forward_hook(lambda mod, inp, out: seen.append(out) or out)+ b._seen = seen+ return b++ def test_sync_hooks_copies_and_clears(self):+ from auto_round.algorithms.parallel.data_parallel import MirrorPool++ block = self._block_with_hook()+ pool = MirrorPool(block, [torch.device("cpu", 0), torch.device("cpu", 1)])+ rep = pool.reps[-1]+ assert rep is not block+ # deepcopy carried the hook; sync must keep it working...+ pool.sync_hooks_from(block)+ x = torch.randn(2, 4)+ rep(x)+ assert len(block._seen) == 1+ # ...and after the home hook is gone, sync drops it from the mirror+ block.lin._forward_hooks.clear()+ pool.sync_hooks_from(block)+ rep(x)+ assert len(block._seen) == 1, "stale hook must not fire after sync"++ def test_release_drops_mirrors(self):+ from auto_round.algorithms.parallel.data_parallel import MirrorPool++ block = self._block_with_hook()+ pool = MirrorPool(block, [torch.device("cpu", 0), torch.device("cpu", 1)])+ assert pool.world == 2+ pool.release()+ assert pool.reps == [] and pool.world == 0+++class TestComposerPoolHandoff:+ """Step 3.6: the collection pool is handed to the iters=0 quantizer and+ released around quantize_block (kept for the cascade at iters=0, dropped+ before the tune's own mirrors at iters>0, dropped on failure too)."""++ def _parts(self, iters, consumes=True):+ from auto_round.algorithms.composer import AlgorithmComposer+ from auto_round.algorithms.quantization.rtn.quantizer import OptimizedRTNQuantizer++ composer = AlgorithmComposer.__new__(AlgorithmComposer)+ events = []++ class _Ctx:+ devices = None # no DDP devices -> ensure_pool is a no-op++ def __init__(self, pool):+ self.pool = pool++ def ensure_pool(self, block):+ events.append("ensure")++ def mark(self, what):+ events.append(what)++ def release_pool(self):+ if self.pool is not None:+ events.append("release")+ self.pool = None++ class _Pool:+ pass++ pool = _Pool()++ q = OptimizedRTNQuantizer.__new__(OptimizedRTNQuantizer)+ q.iters = iters+ q.enable_quanted_input = False+ q._consumes = consumes++ def _qb(*a, **k):+ # instance-attribute functions stay unbound: close over q directly+ st = parallel_state(q)+ seen = st.pool if st is not None else None+ events.append(("saw_pool", seen))+ if seen is not None and q._consumes:+ parallel_state(q, create=True).pool_used = True+ if q._consumes == "adopt":+ # adoption transfers ownership (drops the pool ref)+ parallel_state(q).pool = None++ q.quantize_block = _qb+ ctx = _Ctx(pool)+ ctx.devices = [torch.device("cpu", 0), torch.device("cpu", 1)] # DDP on+ composer._coll_ctx = ctx+ composer._block = None+ composer.block_quantizer = q+ composer.preprocessors = []+ return composer, ctx, pool, q, events++ def _run_handoff(self, composer):+ # mirrors the Step 3.6 handoff (pool to the quantizer, ANY consumer:+ # OptRTN home quantize or SignRound adoption) + the real post-Step-4+ # _settle_block_pool + the try/finally in compress_block+ if getattr(composer._coll_ctx, "devices", None) and len(composer._coll_ctx.devices) >= 2:+ composer._coll_ctx.ensure_pool(composer._block)+ _pool = getattr(composer._coll_ctx, "pool", None)+ if _pool is not None:+ _state = parallel_state(composer.block_quantizer, create=True)+ _state.pool = _pool+ _state.pool_used = False+ try:+ composer.block_quantizer.quantize_block(None)+ _pool = composer._settle_block_pool(composer._block, _pool)+ composer._coll_ctx.mark("step5+6")+ finally:+ _state = parallel_state(composer.block_quantizer)+ if _state is not None:+ _state.pool = None+ composer._coll_ctx.release_pool()++ def test_iters0_pool_handed_then_kept_for_cascade(self):+ # RTN-lane consumption: attr KEPT + used -> the pool survives into+ # step5+6 (its sync made the mirrors exactly-home for the cascade)+ composer, ctx, pool, q, events = self._parts(iters=0, consumes=True)+ self._run_handoff(composer)+ assert events == ["ensure", ("saw_pool", pool), "step5+6", "release"]+ assert parallel_state(q).pool is None++ def test_iters0_unconsumed_pool_released_before_cascade(self):+ composer, ctx, pool, q, events = self._parts(iters=0, consumes=False)+ self._run_handoff(composer)+ # quantizer left unconsumed (e.g. plan vanished): released right+ # after quantize_block, before the cascade step (the finally release+ # is then a no-op on None)+ assert events == ["ensure", ("saw_pool", pool), "release", "step5+6"]+ assert ctx.pool is None++ def test_iters_positive_pool_adopted_by_tune(self):+ composer, ctx, pool, q, events = self._parts(iters=10, consumes="adopt")+ self._run_handoff(composer)+ # the tune adopts the pool (engage_ wraps mirrors in place) -> kept+ # through the tune, released once at the very end+ assert events == ["ensure", ("saw_pool", pool), "release", "step5+6"]++ def test_failure_still_releases_pool(self):+ composer, ctx, pool, q, events = self._parts(iters=0)++ def _boom(*a, **k):+ raise RuntimeError("quantize failed")++ q.quantize_block = _boom+ try:+ self._run_handoff(composer)+ raised = False+ except RuntimeError:+ raised = True+ assert raised+ assert parallel_state(q).pool is None+ assert ctx.pool is None+++class TestRotationHookPoolInteraction:+ """Real Hadamard input-location hooks x MirrorPool: the online rotation+ rides on the mirrors (deepcopy + sync_hooks_from) and mirror forwards are+ bit-identical to the home block; removing the home hook and re-syncing+ drops it from the mirrors (no stale rotation)."""++ def test_hadamard_input_hook_parity_and_resync(self):+ from auto_round.algorithms.parallel.data_parallel import MirrorPool+ from auto_round.algorithms.transforms import RotationConfig+ from auto_round.algorithms.transforms.hadamard.apply import _apply_input_transform++ class _B(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.lin = torch.nn.Linear(32, 32)++ def forward(self, x):+ return self.lin(x)++ b = _B()+ _apply_input_transform(b.lin, RotationConfig(hadamard_type="hadamard"), data_type="int")+ pool = MirrorPool(b, [torch.device("cpu", 0), torch.device("cpu", 1)])+ rep = pool.reps[-1]+ assert rep is not b+ assert len(rep.lin._forward_pre_hooks) == 1+ x = torch.randn(2, 32)+ with torch.no_grad():+ y_home = b(x)+ y_rep = rep(x)+ assert torch.equal(y_home, y_rep), "mirror forward must match home under the rotation hook"+ # stale-hook hygiene: removing the home hook + re-sync drops it+ b.lin._forward_pre_hooks.clear()+ pool.sync_hooks_from(b)+ assert len(rep.lin._forward_pre_hooks) == 0+ with torch.no_grad():+ y_rep2 = rep(x)+ assert not torch.equal(y_home, y_rep2), "resync must drop the removed rotation hook"+++class TestHomeQuantizeThenCascade:+ """End-to-end seam: the quantizer quantizes the HOME block, the engine+ syncs the mirrors to it, and the cascade forward on the SAME pool outputs+ exactly the serial forward of the quantized home block."""++ def test_cascade_on_pool_matches_serial_home(self, monkeypatch):+ from types import SimpleNamespace++ from auto_round.algorithms.parallel.data_parallel import (+ MirrorPool,+ sharded_nograd_forward,+ )+ from auto_round.algorithms.parallel.mirror_sync import sync_mirrors_to_home+ from auto_round.algorithms.quantization.rtn.quantizer import OptimizedRTNQuantizer++ torch.manual_seed(0)++ class _B(torch.nn.Module):+ def __init__(self):+ super().__init__()+ for i in range(4):+ setattr(self, f"l{i}", _expert_layer(f"m.blk.l{i}"))++ def forward(self, x):+ for i in range(4):+ x = getattr(self, f"l{i}")(x)+ return x++ block = _B() # weights stay RANDOM: FP and quantized forwards differ+ originals = {n: m.weight.data.clone() for n, m in block.named_modules() if hasattr(m, "global_name")}+ devices = [torch.device("cpu", 0), torch.device("cpu", 1)]+ pool = MirrorPool(block, devices)++ q = OptimizedRTNQuantizer.__new__(OptimizedRTNQuantizer)+ q._BaseAlgorithm__run_ctx = None # unbound: q.model is None (the driver is stubbed)+ q.parallel_state = SimpleNamespace(plan=SimpleNamespace(world=2, devices=devices), pool=pool, pool_used=False)+ import auto_round.algorithms.quantization.rtn.batched_search as _bs++ def _staged(layer, disable_opt_rtn=None, defer_search=False):+ return layer++ def _driver(model, staged, max_batch=None):+ # a REAL quantize-dequantize: round-to-grid + offset, so an+ # FP (unsynced) mirror layer would produce different outputs+ with torch.no_grad():+ for _name, layer in staged:+ layer.weight.data = ((layer.weight.data * 4).round() / 4) + 0.125++ q._quantize_layer_via_rtn = _staged+ monkeypatch.setattr(_bs, "run_batched_rtn_search", _driver)+ q.quantize_block(block, None, {}, None, None, None)+ assert sync_mirrors_to_home(q, block) is True++ # every home layer carries the EXACT quantize transform+ for n, m in block.named_modules():+ if hasattr(m, "global_name"):+ assert torch.equal(m.weight.data, ((originals[n] * 4).round() / 4) + 0.125)++ inputs = [torch.randn(1, 4) for _ in range(4)] # n=4, world=2 -> shards++ def _runner(rep, inp, oth, idxs, cache_device=None):+ out = []+ with torch.no_grad():+ for i in idxs:+ out.append(rep(inp[i]))+ return torch.cat(out, dim=0)++ # mirror layers must carry the exact home state (the post-quantize+ # sync): FP mirrors would diverge here+ for r in range(pool.world):+ if pool.reps[r] is block:+ continue+ for name, m in block.named_modules():+ if hasattr(m, "global_name"):+ assert torch.equal(pool.mirror_layer(r, name).weight.data, m.weight.data)+ outs = sharded_nograd_forward(_runner, block, inputs, {}, torch.device("cpu"), devices, pool=pool)+ with torch.no_grad():+ serial = [block(t) for t in inputs]+ for a, b in zip(outs, serial):+ assert torch.equal(a, b), "cascade on quantized mirrors must match the serial home forward"+ pool.release()+++class TestMirrorPoolMultiPass:+ """The pool must SURVIVE its passes: sharded_nograd_forward's teardown+ once cleared the aliased reps list, silently disabling every later reuse+ (searches, cascade) while still pinning the mirror VRAM."""++ def test_two_pool_passes_keep_the_pool_alive(self):+ import torch as _t++ from auto_round.algorithms.parallel.data_parallel import (+ MirrorPool,+ sharded_nograd_forward,+ )++ class _B(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.lin = torch.nn.Linear(4, 4)++ def forward(self, x):+ return self.lin(x)++ block = _B()+ devices = [torch.device("cpu", 0), torch.device("cpu", 1)]+ pool = MirrorPool(block, devices)+ n_rep_objects = [id(r) for r in pool.reps]+ inputs = [_t.randn(1, 4) for _ in range(4)]++ def _runner(rep, inp, oth, idxs, cache_device=None):+ out = []+ with _t.no_grad():+ for i in idxs:+ out.append(rep(inp[i]))+ return _t.cat(out, dim=0)++ for pass_idx in range(2):+ outs = sharded_nograd_forward(_runner, block, inputs, {}, torch.device("cpu"), devices, pool=pool)+ assert len(outs) == 4+ # the pool still holds its replicas after the pass teardown+ assert pool.world == 2, f"pass {pass_idx}: pool lost its reps"+ assert [id(r) for r in pool.reps] == n_rep_objects, "replica objects were rebuilt"+ pool.release()+ assert pool.world == 0+++class TestMergeMirrorStats:+ """_merge_mirror_stats must keep scalar stats co-traveling with their tensor+ sibling: imatrix_cnt (a python int) was silently dropped, so a cold expert+ (routed rows on mirror shards, zero on home's) merged its imatrix home+ while cnt went missing -- the normalize step then crashed with+ AttributeError: no attribute 'imatrix_cnt'."""++ _STATS = {"imatrix": "sum", "imatrix_cnt": "sum", "act_max": "max"}++ def test_cold_expert_cnt_cotravels_with_imatrix(self):+ from auto_round.algorithms.parallel.data_parallel import _merge_mirror_stats++ home = torch.nn.Linear(4, 4)+ mirror = torch.nn.Linear(4, 4)+ mirror.imatrix = torch.ones(4)+ mirror.imatrix_cnt = 7 # python int, exactly as both collectors write it+ _merge_mirror_stats(home, [mirror], self._STATS)+ assert hasattr(home, "imatrix") and torch.equal(home.imatrix, torch.ones(4))+ assert getattr(home, "imatrix_cnt", None) == 7++ def test_cnt_sums_across_mirrors(self):+ from auto_round.algorithms.parallel.data_parallel import _merge_mirror_stats++ home = torch.nn.Linear(4, 4)+ home.imatrix = torch.zeros(4)+ home.imatrix_cnt = 2+ m1, m2 = torch.nn.Linear(4, 4), torch.nn.Linear(4, 4)+ m1.imatrix_cnt, m2.imatrix_cnt = 3, 5+ m1.imatrix = torch.ones(4)+ m2.imatrix = torch.ones(4)+ _merge_mirror_stats(home, [m1, m2], self._STATS)+ assert home.imatrix_cnt == 10+ assert torch.equal(home.imatrix, torch.full((4,), 2.0))++ def test_tensor_cnt_also_merges(self):+ from auto_round.algorithms.parallel.data_parallel import _merge_mirror_stats++ home = torch.nn.Linear(4, 4)+ mirror = torch.nn.Linear(4, 4)+ mirror.imatrix = torch.ones(4)+ mirror.imatrix_cnt = torch.tensor(7)+ _merge_mirror_stats(home, [mirror], self._STATS)+ assert home.imatrix_cnt == 7++ def test_non_numeric_attrs_ignored(self):+ from auto_round.algorithms.parallel.data_parallel import _merge_mirror_stats++ home = torch.nn.Linear(4, 4)+ mirror = torch.nn.Linear(4, 4)+ mirror.imatrix_cnt = "bogus"+ _merge_mirror_stats(home, [mirror], self._STATS)+ assert not hasattr(home, "imatrix_cnt")++ def test_mirror_values_are_removed_once_folded(self):+ from auto_round.algorithms.parallel.data_parallel import _merge_mirror_stats++ home = torch.nn.Linear(4, 4)+ mirror = torch.nn.Linear(4, 4)+ mirror.imatrix, mirror.imatrix_cnt, mirror.act_max = torch.ones(4), 3, torch.ones(4)+ _merge_mirror_stats(home, [mirror], self._STATS)+ _merge_mirror_stats(home, [mirror], self._STATS) # a later pass finds nothing left to add+ assert torch.equal(home.imatrix, torch.ones(4)) and home.imatrix_cnt == 3+ assert not any(hasattr(mirror, a) for a in ("imatrix", "imatrix_cnt", "act_max"))++ def test_hook_pass_then_hookless_pass_matches_serial(self):+ """Pool mirrors live across a block's collection passes: a sharded hook+ pass followed by any other sharded pass must leave the serial totals+ (the second merge used to add the mirrors' share again)."""+ from auto_round.algorithms.parallel.data_parallel import MirrorPool, sharded_nograd_forward++ class _B(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.lin = torch.nn.Linear(4, 4)++ def forward(self, x):+ return self.lin(x)++ def _imatrix_hook(module, inp, _out):+ sq = (inp[0].reshape(-1, inp[0].shape[-1]).float() ** 2).sum(0)+ if hasattr(module, "imatrix"):+ module.imatrix += sq+ else:+ module.imatrix = sq++ def _runner(rep, inp, _oth, idxs, cache_device=None):+ with torch.no_grad():+ return torch.cat([rep(inp[i]) for i in idxs], dim=0)++ torch.manual_seed(0)+ block = _B()+ devices = [torch.device("cpu", i) for i in range(2)]+ pool = MirrorPool(block, devices)+ inputs = [torch.randn(1, 4) for _ in range(8)]+ serial = sum((t.reshape(-1, 4) ** 2).sum(0) for t in inputs)+ stats = {"imatrix": "sum"}+ handle = block.lin.register_forward_hook(_imatrix_hook)+ sharded_nograd_forward(_runner, block, inputs, {}, torch.device("cpu"), devices, merge_stats=stats, pool=pool)+ handle.remove()+ assert torch.allclose(block.lin.imatrix, serial)+ sharded_nograd_forward(_runner, block, inputs, {}, torch.device("cpu"), devices, merge_stats=stats, pool=pool)+ assert torch.allclose(block.lin.imatrix, serial)+ pool.release()+++class TestCalibrationStatsContract:+ """A block quantizer declares the statistics its calibration hooks write+ (``calibration_stats``: attr -> "sum" | "max"); the engine folds exactly+ those from the mirrors, keeps undeclared hook passes serial, and warns+ when a sharded pass changes an undeclared attribute on the mirrors."""++ def test_merge_map_is_engine_act_max_plus_declaration(self):+ from types import SimpleNamespace++ from auto_round.algorithms.parallel.data_parallel import calibration_merge_stats++ q = SimpleNamespace(calibration_stats={"hess": "sum"})+ assert calibration_merge_stats(q, ["h"]) == {"act_max": "max", "hess": "sum"}++ def test_undeclared_hooks_keep_the_pass_serial(self):+ from types import SimpleNamespace++ from auto_round.algorithms.parallel.data_parallel import calibration_merge_stats++ assert calibration_merge_stats(SimpleNamespace(calibration_stats=None), ["h"]) is None+ # no quantizer hooks: nothing to declare, the engine's act_max still folds+ assert calibration_merge_stats(SimpleNamespace(calibration_stats=None), []) == {"act_max": "max"}++ def test_unknown_reduction_is_rejected(self):+ from types import SimpleNamespace++ from auto_round.algorithms.parallel.data_parallel import calibration_merge_stats++ with pytest.raises(ValueError, match="supported shard reductions"):+ calibration_merge_stats(SimpleNamespace(calibration_stats={"hess": "mean"}), ["h"])++ def test_imatrix_quantizers_declare_their_stats(self):+ from auto_round.algorithms.quantization.base import BaseQuantizer+ from auto_round.algorithms.quantization.rtn.quantizer import OptimizedRTNQuantizer+ from auto_round.algorithms.quantization.sign_roundv2.quantizer import SignRoundV2Quantizer++ assert BaseQuantizer.calibration_stats is None+ assert OptimizedRTNQuantizer.calibration_stats == {"imatrix": "sum", "imatrix_cnt": "sum"}+ assert SignRoundV2Quantizer.calibration_stats == {"imatrix": "sum"}++ @staticmethod+ def _sharded_hook_pass(hook, stats):+ from auto_round.algorithms.parallel.data_parallel import MirrorPool, sharded_nograd_forward++ class _B(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.lin = torch.nn.Linear(4, 4)++ def forward(self, x):+ return self.lin(x)++ def _runner(rep, inp, _oth, idxs, cache_device=None):+ with torch.no_grad():+ return torch.cat([rep(inp[i]) for i in idxs], dim=0)++ block = _B()+ devices = [torch.device("cpu", i) for i in range(2)]+ pool = MirrorPool(block, devices)+ handle = block.lin.register_forward_hook(hook)+ inputs = [torch.randn(1, 4) for _ in range(8)]+ sharded_nograd_forward(_runner, block, inputs, {}, torch.device("cpu"), devices, merge_stats=stats, pool=pool)+ handle.remove()+ pool.release()+ return block++ def test_undeclared_mirror_write_warns(self, caplog, _autoround_log_propagate):+ def _hook(module, inp, _out):+ module.f9_undeclared_hess = inp[0].sum()++ with caplog.at_level("WARNING"):+ self._sharded_hook_pass(_hook, {"act_max": "max"})+ msgs = [r.message for r in caplog.records if "not a declared calibration statistic" in r.message]+ assert msgs and "f9_undeclared_hess" in msgs[0]++ def test_bookkeeping_list_change_does_not_warn(self, caplog, _autoround_log_propagate):+ # a hook that clears its own handle list (the MoE shape recorder in+ # tune_memory does this) is bookkeeping, not a statistic+ def _hook(module, _inp, _out):+ module.f9_rec_handles_ = []++ with caplog.at_level("WARNING"):+ self._sharded_hook_pass(_hook, {"act_max": "max"})+ assert not [r for r in caplog.records if "not a declared calibration statistic" in r.message]++ def test_undeclared_tensor_list_append_warns(self, caplog, _autoround_log_propagate):+ def _hook(module, inp, _out):+ module.__dict__.setdefault("f9_rows", []).append(inp[0].sum())++ with caplog.at_level("WARNING"):+ self._sharded_hook_pass(_hook, {"act_max": "max"})+ msgs = [r.message for r in caplog.records if "not a declared calibration statistic" in r.message]+ assert msgs and "f9_rows" in msgs[0]++ def test_declared_mirror_write_is_folded_silently(self, caplog, _autoround_log_propagate):+ def _hook(module, inp, _out):+ sq = (inp[0].reshape(-1, 4) ** 2).sum(0)+ if hasattr(module, "f9_declared_stat"):+ module.f9_declared_stat += sq+ else:+ module.f9_declared_stat = sq++ with caplog.at_level("WARNING"):+ block = self._sharded_hook_pass(_hook, {"act_max": "max", "f9_declared_stat": "sum"})+ assert not [r for r in caplog.records if "not a declared calibration statistic" in r.message]+ assert hasattr(block.lin, "f9_declared_stat")+++class TestCappedPassReusesPool:+ """A capped hook pass (AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES) must REUSE+ the full-width pool's first K replicas, skipping ephemeral mirrors (the+ old behavior paid a per-pass block deepcopy plus a late full pool+ rebuild per block)."""++ def test_capped_pass_runs_subset_of_pool_replicas(self):+ import torch as _t++ from auto_round.algorithms.parallel.data_parallel import (+ MirrorPool,+ sharded_nograd_forward,+ )++ class _B(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.lin = torch.nn.Linear(4, 4)++ def forward(self, x):+ return self.lin(x)++ block = _B()+ devices = [torch.device("cpu", i) for i in range(4)]+ pool = MirrorPool(block, devices)+ rep_ids = [id(r) for r in pool.reps]+ inputs = [_t.randn(1, 4) for _ in range(8)]+ ran = []++ def _runner(rep, inp, oth, idxs, cache_device=None):+ ran.append(id(rep))+ out = []+ with _t.no_grad():+ for i in idxs:+ out.append(rep(inp[i]))+ return _t.cat(out, dim=0)++ outs = sharded_nograd_forward(+ _runner, block, inputs, {}, torch.device("cpu"), devices, pool=pool, max_devices=2+ )+ with torch.no_grad():+ serial = [block(t) for t in inputs]+ for a, b in zip(outs, serial):+ assert _t.equal(a, b), "capped pool pass must match the serial forward"+ # only the first K replicas ran, and they are the pool's own objects+ assert set(ran) == set(rep_ids[:2]), f"capped pass must run pool.reps[:2], ran={ran}"+ # the pool keeps its full width and identity set afterwards+ assert pool.world == 4+ assert [id(r) for r in pool.reps] == rep_ids, "replica objects were rebuilt"+ pool.release()++ def test_full_width_pass_with_pool_unchanged(self):+ import torch as _t++ from auto_round.algorithms.parallel.data_parallel import (+ MirrorPool,+ sharded_nograd_forward,+ )++ class _B(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.lin = torch.nn.Linear(4, 4)++ def forward(self, x):+ return self.lin(x)++ block = _B()+ devices = [torch.device("cpu", i) for i in range(2)]+ pool = MirrorPool(block, devices)+ rep_ids = [id(r) for r in pool.reps]+ inputs = [_t.randn(1, 4) for _ in range(4)]++ def _runner(rep, inp, oth, idxs, cache_device=None):+ out = []+ with _t.no_grad():+ for i in idxs:+ out.append(rep(inp[i]))+ return _t.cat(out, dim=0)++ outs = sharded_nograd_forward(_runner, block, inputs, {}, torch.device("cpu"), devices, pool=pool)+ assert len(outs) == 4+ assert [id(r) for r in pool.reps] == rep_ids+ pool.release()+++class TestEnsurePool:+ """The pool is block-wide: built even when no collection pass was+ eligible (n % world != 0 -> serial forwards), because the searches and+ the tune consume resident mirrors regardless of forward eligibility."""++ def test_ensure_pool_builds_once(self):+ from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext++ class _B(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.lin = torch.nn.Linear(4, 4)++ ctx = TuneParallelContext()+ ctx.devices = [torch.device("cpu", 0), torch.device("cpu", 1)]+ assert ctx.pool is None+ blk = _B()+ ctx.ensure_pool(blk)+ first = ctx.pool+ assert first is not None and first.world == 2+ ctx.ensure_pool(blk) # idempotent: same pool object+ assert ctx.pool is first++ def test_ensure_pool_noop_without_devices(self):+ from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext++ ctx = TuneParallelContext()+ ctx.devices = None+ ctx.ensure_pool(object())+ assert ctx.pool is None+++class TestContiguousShardSplit:+ """The ceil/floor split: ANY sample count shards (a remainder lands on+ an earlier replica); fewer samples than devices shrinks the pass world+ instead of running serial; pools are withheld when the pass narrows."""++ def test_bounds_remainder_on_earlier_shards(self):+ from auto_round.algorithms.parallel.data_parallel import contiguous_shard_bounds++ assert contiguous_shard_bounds(129, 4) == [0, 33, 65, 97, 129] # 33/32/32/32+ assert contiguous_shard_bounds(128, 4) == [0, 32, 64, 96, 128]+ assert contiguous_shard_bounds(3, 4) == [0, 1, 2, 3, 3] # caller clamps world first++ def test_uneven_n_shards_instead_of_serial(self):+ import torch as _t++ from auto_round.algorithms.parallel.data_parallel import sharded_nograd_forward++ class _B(_t.nn.Module):+ def __init__(self):+ super().__init__()+ self.lin = _t.nn.Linear(4, 4)++ def forward(self, x):+ return self.lin(x)++ seen_shards = []++ def _runner(rep, inp, oth, idxs, cache_device=None):+ seen_shards.append(list(idxs))+ out = []+ with _t.no_grad():+ for i in idxs:+ out.append(rep(inp[i]))+ return _t.cat(out, dim=0)++ devices = [_t.device("cpu", 0), _t.device("cpu", 1)]+ inputs = [_t.randn(1, 4) for _ in range(5)] # 5 % 2 != 0 -> 3/2 split+ outs = sharded_nograd_forward(_runner, _B(), inputs, {}, _t.device("cpu"), devices)+ assert seen_shards == [[0, 1, 2], [3, 4]]+ assert len(outs) == 5++ def test_fewer_samples_than_devices_shrinks_world(self):+ import torch as _t++ from auto_round.algorithms.parallel.data_parallel import MirrorPool, sharded_nograd_forward++ class _B(_t.nn.Module):+ def __init__(self):+ super().__init__()+ self.lin = _t.nn.Linear(4, 4)++ def forward(self, x):+ return self.lin(x)++ def _runner(rep, inp, oth, idxs, cache_device=None):+ out = []+ with _t.no_grad():+ for i in idxs:+ out.append(rep(inp[i]))+ return _t.cat(out, dim=0)++ devices = [_t.device("cpu", 0), _t.device("cpu", 1), _t.device("cpu", 2)]+ inputs = [_t.randn(1, 4) for _ in range(2)] # n=2 < world=3+ pool = MirrorPool(_B(), devices) # full-width pool exists+ outs = sharded_nograd_forward(_runner, _B(), inputs, {}, _t.device("cpu"), devices, pool=pool)+ assert len(outs) == 2 # sharded on 2 devices, not serial+ assert pool.world == 3 # pool untouched by the narrowed pass+++class TestResolverRtnSafe:+ """The resolver must not assume SignRound-family quantizer methods."""++ def test_quantizer_without_get_scaler_resolves(self, monkeypatch):+ from types import SimpleNamespace++ import torch++ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_++ q = SimpleNamespace(iters=0, gradient_accumulate_steps=1, enable_lfq=False)+ # missing parallel_constraints and calibration_context must still+ # resolve cleanly+ with pytest.raises(RuntimeError, match="not a supported accelerator"):+ resolve_tune_ddp_plan_(+ _with_policy(q), torch.nn.Sequential(torch.nn.Linear(4, 4)), [torch.zeros(1)], None, "cpu"+ )+++class TestCollectionShardingFailVisible:+ """The collection context must not swallow the resolver's requirement error."""++ def _composer(self):+ from auto_round.algorithms.composer import AlgorithmComposer++ composer = AlgorithmComposer.__new__(AlgorithmComposer)+ composer.block_forward = type("R", (), {"output_config": ["hidden_states"]})()+ composer.block_quantizer = object() # opaque; the resolver is monkeypatched+ return composer++ def test_requirement_error_propagates(self, monkeypatch):+ import torch++ import auto_round.algorithms.parallel.tune_parallel as tp++ def _raise(quantizer, block, fp_inputs, fp_outputs, home, world=None, log=True):+ raise RuntimeError("parallel tuning with world=4 is ineligible: no mirror device")++ monkeypatch.setattr(tp, "resolve_tune_ddp_plan_", _raise)+ block = torch.nn.Sequential(torch.nn.Linear(4, 4))+ with pytest.raises(RuntimeError, match="ineligible"):+ self._composer()._collection_context(block, [torch.zeros(1)])++ def test_reachability_error_warns_and_declines(self, monkeypatch, caplog, _autoround_log_propagate):+ import torch++ import auto_round.algorithms.parallel.tune_parallel as tp++ def _raise(quantizer, block, fp_inputs, fp_outputs, home, world=None, log=True):+ raise OSError("boom")++ monkeypatch.setattr(tp, "resolve_tune_ddp_plan_", _raise)+ block = torch.nn.Sequential(torch.nn.Linear(4, 4))+ with caplog.at_level("WARNING"):+ out = self._composer()._collection_context(block, [torch.zeros(1)])+ assert out.devices is None+ assert any("resolver unreachable" in r.message for r in caplog.records)+++import unittest # noqa: E402+from types import SimpleNamespace # noqa: E402+++def _stackable_search_fn(weight, bits, imatrix):+ """Row-independent deterministic search: first element of each leading row."""+ return weight.reshape(weight.shape[0], -1)[:, 0]+++class TestSyncPoolDevices:+ """Phase barriers synchronize every distinct CUDA pool device once (CPU skipped)."""++ def test_syncs_each_cuda_device_once(self):+ from auto_round.algorithms.parallel.mirror_sync import _sync_pool_devices++ devices = [+ torch.device("cpu"),+ torch.device("cuda:0"),+ torch.device("cuda:1"),+ torch.device("cuda:0"),+ ]+ calls = []+ orig = torch.cuda.synchronize+ try:+ torch.cuda.synchronize = lambda d: calls.append(str(d))+ elapsed = _sync_pool_devices(devices)+ finally:+ torch.cuda.synchronize = orig+ # one synchronize per distinct cuda device, cpu skipped, elapsed returned+ assert calls == ["cuda:0", "cuda:1"]+ assert isinstance(elapsed, float)+++class TestDistributeSearchStats:+ """Tune adoption: the merged home stats land on every mirror."""++ def _pool(self, block, world):+ mirrors = [self._build_mirror(block) for _ in range(world)]+ reps = [block] + mirrors # home at index 0++ def mirror_layer(r, name):+ return reps[r].get_submodule(name)++ return SimpleNamespace(world=len(reps), reps=reps, mirror_layer=mirror_layer)++ @staticmethod+ def _build_mirror(block):+ import copy as _pcopy++ return _pcopy.deepcopy(block)++ def test_distributes_to_every_mirror(self):+ import torch.nn as nn++ from auto_round.algorithms.parallel.mirror_sync import _distribute_search_stats++ block = nn.Module()+ block.l1 = nn.Linear(4, 4, bias=False)+ block.l1.imatrix = torch.ones(4) * 2+ pool = self._pool(block, 3)+ _distribute_search_stats(pool, block, [("l1", block.l1)])+ for r in (1, 2):+ assert torch.equal(pool.mirror_layer(r, "l1").imatrix, torch.ones(4) * 2)+++class TestHomeQuantizeMirrorParity:+ """REAL end-to-end parity (no spies) on CPU mirrors: under a world=2 plan+ the quantizer's own quantize_block (main's code) plus the engine's mirror+ sync must leave the home block AND every mirror bit-identical to a plain+ serial quantize_block run. Every quantized weight/scale/zp/data_type must+ be torch.equal."""++ WIDTHS = ((16, 32), (16, 32), (8, 32), (12, 24)) # two same-shape pairs -> real stacking++ def _layer(self, name, out, inn, gen):+ m = torch.nn.Linear(inn, out, bias=False)+ with torch.no_grad():+ m.weight.copy_(torch.randn(out, inn, generator=gen))+ m.global_name = name+ m.to_quantized = True+ m.bits = 4+ m.group_size = 32+ m.sym = True+ m.data_type = "int"+ m.iters = 0+ m.act_bits = 16+ m.scale_dtype = torch.float16+ m.super_bits = None+ m.super_group_size = None+ m.imatrix = torch.rand(inn, generator=gen) + 0.5+ m.imatrix_cnt = 1 # pre-normalized: quantize_block's divide is identity+ return m++ def _block(self):+ import copy as _copy++ gen = torch.Generator().manual_seed(1234)++ blk = torch.nn.Module()+ for i, (out, inn) in enumerate(self.WIDTHS):+ blk.add_module(f"l{i}", self._layer(f"m.blk.l{i}", out, inn, gen))+ return blk, _copy.deepcopy(blk)++ def _model(self, blk):+ # the tree must resolve the layers' GLOBAL names (m.blk.l<i>): the+ # driver re-attaches quantized results by that path+ model = torch.nn.Module()+ m = torch.nn.Module()+ m.blk = blk+ model.m = m+ return model++ def _quantizer(self, cls, config, model):+ from types import SimpleNamespace++ q_cls = type(+ f"_ParityQ_{cls.__name__}",+ (cls,),+ {+ "config": config,+ "model": model,+ "model_context": SimpleNamespace(amp_dtype=torch.float32, is_moe_model=False),+ "compress_context": SimpleNamespace(enable_torch_compile=False),+ },+ )+ return q_cls.__new__(q_cls)++ def _assert_same(self, ref, got, tag):+ assert torch.equal(ref.weight.data, got.weight.data), f"{tag} weight diverged"+ assert torch.equal(ref.scale, got.scale), f"{tag} scale diverged"+ rz, gz = getattr(ref, "zp", None), getattr(got, "zp", None)+ assert (rz is None) == (gz is None), f"{tag} zp presence diverged"+ if rz is not None:+ if isinstance(rz, torch.Tensor):+ assert torch.equal(rz, gz), f"{tag} zp diverged"+ else:+ assert rz == gz, f"{tag} zp diverged"+ assert getattr(ref, "data_type", None) == getattr(got, "data_type", None), f"{tag} data_type diverged"+ assert getattr(ref, "q_scale_thresh", None) == getattr(got, "q_scale_thresh", None)++ def _run_parity(self, quantizer_cls, config):+ from types import SimpleNamespace++ from auto_round.algorithms.parallel.data_parallel import MirrorPool+ from auto_round.algorithms.parallel.mirror_sync import sync_mirrors_to_home++ blk_s, blk_d = self._block()+ model_s, model_d = self._model(blk_s), self._model(blk_d)+ self._quantizer(quantizer_cls, config, model_s).quantize_block(blk_s, None, {}, None, None, None)++ devices = [torch.device("cpu", 0), torch.device("cpu", 1)]+ pool = MirrorPool(blk_d, devices)+ q = self._quantizer(quantizer_cls, config, model_d)+ q.parallel_state = SimpleNamespace(plan=SimpleNamespace(world=2, devices=devices), pool=pool, pool_used=False)+ q.quantize_block(blk_d, None, {}, None, None, None)+ assert sync_mirrors_to_home(q, blk_d) is True++ for i in range(len(self.WIDTHS)):+ ref = model_s.get_submodule(f"m.blk.l{i}")+ self._assert_same(ref, blk_d.get_submodule(f"l{i}"), f"home l{i}")+ for r in range(pool.world):+ if pool.reps[r] is not blk_d:+ self._assert_same(ref, pool.mirror_layer(r, f"l{i}"), f"mirror{r} l{i}")++ def test_opt_rtn_home_and_mirrors_match_serial(self):+ from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig+ from auto_round.algorithms.quantization.rtn.quantizer import OptimizedRTNQuantizer++ cfg = OptimizedRTNConfig(bits=4, group_size=32, sym=True, disable_opt_rtn=False, enable_neuqi=False)+ self._run_parity(OptimizedRTNQuantizer, cfg)++ def test_plain_rtn_home_and_mirrors_match_serial(self):+ from auto_round.algorithms.quantization.rtn.config import RTNConfig+ from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer++ cfg = RTNConfig(bits=4, group_size=32, sym=True, disable_opt_rtn=True)+ self._run_parity(RTNQuantizer, cfg)diff --git a/test/unit/test_cpu/quantization/test_search_dispatch.py b/test/unit/test_cpu/quantization/test_search_dispatch.pyindex 610072819..01612c868 100644--- a/test/unit/test_cpu/quantization/test_search_dispatch.py+++ b/test/unit/test_cpu/quantization/test_search_dispatch.py@@ -791,3 +791,62 @@ class TestPlanCapacityChunks(unittest.TestCase):plans, _ = self._plan([("cuda:1", per, 0, 8, group_home1), ("cuda:2", per, 0, 8, group_home2)], probes, picks)for worker, home, chunk in plans:self.assertEqual(worker, home)+++class TestSearchSummary(unittest.TestCase):+ """One summary line per search phase: per-device busy time (stacked and+ per-module calls alike) and the elapsed time of the phase."""++ def _line(self, summary, label="[batched-search] wrap search:"):+ from unittest import mock++ from auto_round.algorithms.quantization import search_dispatch++ with (+ mock.patch.object(search_dispatch, "perf_counters_enabled", return_value=True),+ mock.patch.object(search_dispatch.logger, "info") as info,+ ):+ summary.log(label)+ fmt, *args = info.call_args[0]+ return fmt % tuple(args)++ def test_line_keeps_the_summary_grammar(self):+ from auto_round.algorithms.quantization.search_dispatch import SearchSummary++ s = SearchSummary()+ st = s.device("cuda:0")+ st.update(wall=1.5, modules=10, batches=2, singletons=0)+ st = s.device("cuda:1")+ st.update(wall=0.5, modules=2, batches=0, singletons=2)+ s.elapsed = 1.75+ self.assertEqual(+ self._line(s),+ "[batched-search] wrap search: 12 modules, 2 batch + 2 singleton search calls, "+ "1.75s device-wall (2.00s summed) walls {'0': 1.50, '1': 0.50}",+ )++ def test_per_module_calls_are_timed(self):+ import time as _time++ from auto_round.algorithms.quantization.search_dispatch import SearchSummary++ s = SearchSummary()+ with s.timed("cuda:0") as st:+ _time.sleep(0.02)+ st["singletons"] += 1+ st["modules"] += 1+ s.finish()+ self.assertGreater(s.devices["cuda:0"]["wall"], 0.01)+ self.assertGreaterEqual(s.elapsed, s.devices["cuda:0"]["wall"])++ def test_merged_ranks_add_counters_and_keep_the_slowest_elapsed(self):+ from auto_round.algorithms.quantization.search_dispatch import SearchSummary++ a, b = SearchSummary(), SearchSummary()+ a.device("cuda:0").update(wall=1.0, modules=3, batches=1, singletons=0)+ b.device("cuda:1").update(wall=2.0, modules=2, batches=0, singletons=2)+ a.elapsed, b.elapsed = 1.1, 2.2+ m = SearchSummary.merged([a, b])+ self.assertEqual(m.elapsed, 2.2)+ self.assertEqual(m.devices["cuda:0"]["modules"] + m.devices["cuda:1"]["modules"], 5)+ self.assertIn("5 modules, 1 batch + 2 singleton search calls, 2.20s device-wall (3.00s summed)", self._line(m))diff --git a/test/unit/test_cpu/quantization/test_spmd_ranks.py b/test/unit/test_cpu/quantization/test_spmd_ranks.pynew file mode 100644index 000000000..e1a1b5572--- /dev/null+++ b/test/unit/test_cpu/quantization/test_spmd_ranks.py@@ -0,0 +1,377 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Thread ranks (auto_round.algorithms.parallel.spmd): collectives, turnstile,+samplers, failure handling, and the unchanged SignRound loop run per rank+on CPU against the serial run."""++import logging+import threading++import pytest+import torch++from auto_round.algorithms.parallel import spmd++CPU2 = [torch.device("cpu"), torch.device("cpu", 1)]+++def _run(fns, devices=None):+ devices = devices or CPU2[: len(fns)]+ return spmd.run_ranks(fns, devices, spmd.RankGroup(devices))+++class TestCollectives:+ def test_weighted_mean_reproduces_the_union(self):+ # rank 0: 2 tokens with loss 1.0, rank 1: 6 tokens with loss 3.0+ out = _run(+ [+ lambda: spmd.current_rank().all_reduce_weighted_mean(1.0, 2),+ lambda: spmd.current_rank().all_reduce_weighted_mean(3.0, 6),+ ]+ )+ assert out == [2.5, 2.5]++ def test_mean_is_identical_on_every_rank(self):+ vals = [1.0, 2.0, 4.0, 8.0]+ devs = [torch.device("cpu")] + [torch.device("cpu", i) for i in range(1, 4)]+ out = _run([lambda v=v: spmd.current_rank().all_reduce_mean(v) for v in vals], devs)+ assert out == [3.75] * 4++ def test_grads_average_across_ranks_like_one_batch(self):+ torch.manual_seed(0)+ base = torch.nn.Linear(4, 2)+ xs = [torch.randn(3, 4), torch.randn(3, 4)]+ reps = [base, torch.nn.Linear(4, 2)]+ reps[1].load_state_dict(base.state_dict())++ def _fn(r):+ def _go():+ reps[r](xs[r]).pow(2).mean().backward()+ spmd.current_rank().sync_gradients(reps[r])++ return _go++ _run([_fn(0), _fn(1)])+ ref = torch.nn.Linear(4, 2)+ ref.load_state_dict(base.state_dict())+ ref(torch.cat(xs)).pow(2).mean().backward()+ for p_ref, p0, p1 in zip(ref.parameters(), reps[0].parameters(), reps[1].parameters()):+ assert torch.equal(p0.grad, p1.grad)+ assert torch.allclose(p0.grad, p_ref.grad, atol=1e-6)++ def test_rank0_runs_in_the_calling_thread(self):+ main = threading.current_thread()+ out = _run([lambda: threading.current_thread() is main, lambda: threading.current_thread() is main])+ assert out == [True, False]+++class TestTurnstileAndFailures:+ def test_first_phases_run_in_rank_order(self):+ events = []+ lock = threading.Lock()++ def _fn(r):+ def _go():+ with lock:+ events.append(("start", r))+ spmd.current_rank().all_reduce_mean(r)+ with lock:+ events.append(("after", r))++ return _go++ devs = [torch.device("cpu")] + [torch.device("cpu", i) for i in range(1, 4)]+ _run([_fn(r) for r in range(4)], devs)+ starts = [r for kind, r in events if kind == "start"]+ assert starts == [0, 1, 2, 3]+ # nobody passes the collective before the last rank started+ assert events.index(("start", 3)) < min(events.index(("after", r)) for r in range(4))++ def test_warm_blocks_start_every_rank_at_once(self):+ started = threading.Event()++ def _rank0():+ # rank 1 must start before rank 0 reaches its first collective+ assert started.wait(timeout=5)+ spmd.current_rank().all_reduce_mean(0.0)++ def _rank1():+ started.set()+ spmd.current_rank().all_reduce_mean(1.0)++ spmd.run_ranks([_rank0, _rank1], CPU2, spmd.RankGroup(CPU2, serial_first_phase=False))++ def test_block_signature_tracks_shapes_and_devices(self):+ a, b = torch.nn.Linear(4, 4), torch.nn.Linear(4, 8)+ assert spmd.block_signature(a, CPU2) == spmd.block_signature(torch.nn.Linear(4, 4), CPU2)+ assert spmd.block_signature(a, CPU2) != spmd.block_signature(b, CPU2)+ assert spmd.block_signature(a, CPU2) != spmd.block_signature(a, CPU2[:1])++ def test_peer_failure_raises_the_real_error_without_hanging(self):+ def _ok():+ spmd.current_rank().all_reduce_mean(1.0)+ spmd.current_rank().all_reduce_mean(2.0)++ def _boom():+ spmd.current_rank().all_reduce_mean(1.0)+ raise ValueError("rank 1 failed")++ with pytest.raises(ValueError, match="rank 1 failed"):+ _run([_ok, _boom])++ def test_failure_before_first_collective_releases_the_next_rank(self):+ def _boom():+ raise KeyError("early")++ with pytest.raises(KeyError):+ _run([_boom, lambda: spmd.current_rank().all_reduce_mean(1.0)])+++class TestRankScope:+ def test_device_manager_sees_the_rank_device_only_inside_the_rank(self):+ from auto_round.utils.device_manager import device_manager++ outside = device_manager.device+ seen = _run([lambda: device_manager.device, lambda: (device_manager.device, device_manager.device_list)])+ assert seen[0] == "cpu"+ assert seen[1] == ("cpu:1", ["cpu:1"])+ assert device_manager.device == outside++ def test_index_sampler_draws_from_the_rank_shard(self):+ from auto_round.utils.distributed import make_index_sampler++ def _draws():+ s = make_index_sampler(8, 2)+ return sorted(i for _ in range(2) for i in s.next_batch())++ out = _run([_draws, _draws])+ assert out == [[0, 1, 2, 3], [4, 5, 6, 7]]+ assert make_index_sampler(8, 2).nsamples == 8 # outside ranks: the whole pool++ def test_non_zero_ranks_log_warnings_only(self, caplog):+ from auto_round.logger import logger++ logger.propagate = True+ try:+ with caplog.at_level(logging.INFO, logger="autoround"):++ def _log(r):+ def _go():+ logger.info("info from rank %d", r)+ logger.warning("warning from rank %d", r)++ return _go++ _run([_log(0), _log(1)])+ finally:+ logger.propagate = False+ msgs = [r.getMessage() for r in caplog.records]+ assert "info from rank 0" in msgs and "warning from rank 0" in msgs+ assert "info from rank 1" not in msgs and "warning from rank 1" in msgs+++_DS = [+ "CPU-tier rank parity sample with enough tokens for quantization",+ "Another parity sample with a different length padding the batch",+ "Third sample keeps the calibration pool at four entries",+ "Fourth sample completes the pinned draw schedule",+]+++class _Fixed:+ def __init__(self, draws):+ self._draws, self._i = list(draws), 0++ def next_batch(self):+ out = self._draws[self._i % len(self._draws)]+ self._i += 1+ return out+++def _pinned_sampler(nsamples, batch_size):+ # serial: batches [0, 1] then [2, 3]; rank r draws the r-th sample of the+ # same batch, so both runs see the same samples per iteration+ rank = spmd.current_rank()+ if rank is None:+ return _Fixed([[0, 1], [2, 3]])+ return _Fixed([[0], [2]]) if rank.rank == 0 else _Fixed([[1], [3]])+++def _quantize(model_path, monkeypatch, world, **extra):+ import auto_round.algorithms.parallel.tune_parallel as tp+ import auto_round.algorithms.quantization.sign_round.quantizer as sr+ from auto_round import AutoRound+ from auto_round.algorithms.parallel.data_parallel import DDPPlan, ParallelPolicy++ monkeypatch.setattr(sr, "make_index_sampler", _pinned_sampler)+ engaged = []+ if world > 1:+ # CPU ranks: the resolvers decline non-accelerator homes, so pin the+ # tune plan to (cpu, cpu:1) and keep the collection serial+ monkeypatch.setattr(spmd, "resolve_tune_ddp_plan_", lambda *a, **k: DDPPlan(2, CPU2, 2))+ monkeypatch.setattr(tp, "resolve_tune_ddp_plan_", lambda *a, **k: DDPPlan(1, CPU2[:1], 0))+ orig = spmd.quantize_block_on_ranks++ def _counted(*a, **k):+ res = orig(*a, **k)+ engaged.append((res[0], type(a[0]).__name__))+ return res++ monkeypatch.setattr(spmd, "quantize_block_on_ranks", _counted)+ ar = AutoRound(+ model_path,+ bits=4,+ group_size=32,+ sym=extra.pop("sym", False),+ iters=2,+ nsamples=4,+ seqlen=8,+ batch_size=2,+ dataset=list(_DS),+ device_map="cpu",+ enable_torch_compile=False,+ disable_model_free=True,+ **extra,+ )+ if world > 1:+ ar.compress_context.parallel_policy = ParallelPolicy(world=world)+ from auto_round.logger import logger++ losses = []++ class _LossLines(logging.Handler):+ def emit(self, record):+ msg = record.getMessage()+ if "loss iter" in msg:+ losses.append(msg[msg.index("loss iter") :])++ handler = _LossLines(level=logging.INFO)+ logger.addHandler(handler)+ try:+ model, _ = ar.quantize()+ finally:+ logger.removeHandler(handler)+ weights = {k: v.detach().clone() for k, v in model.state_dict().items()}+ return weights, engaged, losses+++def test_signround_on_two_cpu_ranks_matches_serial(tiny_opt_model_path, monkeypatch):+ """The unchanged SignRound loop, run once per rank with averaged+ gradients, lands on the serial run's quantized weights bit-exactly."""+ serial, _, serial_losses = _quantize(tiny_opt_model_path, monkeypatch, world=1)+ ranks, engaged, rank_losses = _quantize(tiny_opt_model_path, monkeypatch, world=2)+ assert engaged and all(e for e, _ in engaged)+ assert set(serial) == set(ranks)+ diff = [k for k in serial if not torch.equal(serial[k], ranks[k])]+ assert not diff, f"rank run diverged from serial: {diff[:8]}"+ # the padded samples carry a valid-token mask: the global loss (masked+ # sums over valid tokens, weighted by each rank's token count) reports+ # the serial value+ assert serial_losses and rank_losses == serial_losses+++def test_signroundv2_on_two_cpu_ranks_matches_serial(tiny_opt_model_path, monkeypatch):+ """SignRoundV2 defers its wrap-time init-scale searches; warm ranks shard+ them (each rank searches its share, the scales are exchanged), and the+ result still matches the serial run bit-exactly."""+ sharded = []+ orig = spmd.run_rank_sharded_wrap_searches++ def _counted(rank, wrappers):+ res = orig(rank, wrappers)+ sharded.append(res)+ return res++ monkeypatch.setattr(spmd, "run_rank_sharded_wrap_searches", _counted)+ serial, _, _ = _quantize(tiny_opt_model_path, monkeypatch, world=1, sym=True, enable_alg_ext=True)+ ranks, engaged, _ = _quantize(tiny_opt_model_path, monkeypatch, world=2, sym=True, enable_alg_ext=True)+ assert engaged and all(e and name == "SignRoundV2Quantizer" for e, name in engaged), engaged+ assert any(sharded), "no block sharded its wrap searches"+ diff = [k for k in serial if not torch.equal(serial[k], ranks[k])]+ assert not diff, f"rank run diverged from serial: {diff[:8]}"+++def test_pull_tuning_pool_keeps_the_rank_shards_in_place():+ """A rank must not bulk-pull the shared, device-distributed pool onto its+ own device (every other rank's shard would be copied there)."""+ from auto_round.algorithms.quantization.base import BaseQuantizer++ q = BaseQuantizer.__new__(BaseQuantizer)+ pool = [torch.zeros(1), torch.zeros(1)]+ out = _run([lambda: q.pull_tuning_pool(pool, "cpu", None, 1, "pool"), lambda: None])+ assert out[0] is pool+++class _FakeWrapper:+ def __init__(self, idx, searched):+ self.idx, self._searched, self.init_scale, self.final = idx, searched, None, None++ def _run_deferred_search_now(self):+ self._searched.append((spmd.current_rank().rank, self.idx))+ self.init_scale = self.final = torch.tensor(float(self.idx))++ def finalize_batched_search(self, init_scale):+ self.init_scale = self.final = init_scale+++def test_wrap_searches_shard_across_warm_ranks(monkeypatch):+ from auto_round.algorithms.quantization import search_dispatch+ from auto_round.logger import logger++ monkeypatch.setattr(search_dispatch, "run_batched_wrap_search", lambda ws, **kw: False)+ monkeypatch.setattr(search_dispatch, "perf_counters_enabled", lambda: True)+ searched = []+ per_rank = [[_FakeWrapper(i, searched) for i in range(5)] for _ in range(2)]++ def _fn(r):+ return lambda: spmd.run_rank_sharded_wrap_searches(spmd.current_rank(), per_rank[r])++ lines = []++ class _Lines(logging.Handler):+ def emit(self, record):+ lines.append(record.getMessage())++ handler = _Lines(level=logging.INFO)+ logger.addHandler(handler)+ try:+ out = spmd.run_ranks([_fn(0), _fn(1)], CPU2, spmd.RankGroup(CPU2, serial_first_phase=False))+ finally:+ logger.removeHandler(handler)+ assert out == [True, True]+ # every wrapper searched exactly once, by its owner rank (index % world)+ assert sorted(searched, key=lambda t: t[1]) == [(i % 2, i) for i in range(5)]+ for ws in per_rank:+ assert [float(w.final) for w in ws] == [0.0, 1.0, 2.0, 3.0, 4.0]+ # one summary line for the block, covering every rank's share and device+ summary = [m for m in lines if m.startswith("[batched-search] wrap search:")]+ assert len(summary) == 1, lines+ assert "5 modules, 0 batch + 5 singleton search calls" in summary[0], summary+ assert "'cpu': " in summary[0] and "'cpu:1': " in summary[0], summary+++def test_wrap_searches_stay_local_in_the_serial_first_phase():+ out = _run([lambda: spmd.run_rank_sharded_wrap_searches(spmd.current_rank(), []), lambda: None])+ assert out[0] is False+++def test_adamround_on_two_cpu_ranks_matches_serial(tiny_opt_model_path, monkeypatch):+ """AdamRound runs the same unchanged loop per rank; its update uses+ gradient magnitudes, so the averaged rank gradients must reproduce the+ serial update."""+ serial, _, _ = _quantize(tiny_opt_model_path, monkeypatch, world=1, enable_adam=True)+ ranks, engaged, _ = _quantize(tiny_opt_model_path, monkeypatch, world=2, enable_adam=True)+ assert engaged and all(e and name == "AdamRoundQuantizer" for e, name in engaged), engaged+ diff = [k for k in serial if not torch.equal(serial[k], ranks[k])]+ assert not diff, f"rank run diverged from serial: {diff[:8]}"diff --git a/test/unit/test_cpu/quantization/test_tune_parallel_context.py b/test/unit/test_cpu/quantization/test_tune_parallel_context.pynew file mode 100644index 000000000..c55cca634--- /dev/null+++ b/test/unit/test_cpu/quantization/test_tune_parallel_context.py@@ -0,0 +1,130 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Unit tests for the collection-phase TuneParallelContext.++The engine primitives (sharded_nograd_forward, ...) are faked: these tests+pin the context's collection-path semantics and serial fallbacks. The tune+itself runs on thread ranks (test_spmd_ranks).+"""++import logging+from unittest import mock++import pytest+import torch++from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext+++class TestCollectionContext:+ def test_for_collection_non_list_inputs_keeps_serial(self):+ composer = mock.Mock()+ ctx = TuneParallelContext.for_collection(composer, block=torch.nn.Linear(2, 2), fp_inputs={"a": [1]})+ assert ctx.devices is None++ def test_collect_forward_serial_when_devices_none(self):+ ctx = TuneParallelContext()+ bf = mock.Mock(return_value="out")+ assert ctx.collect_forward(bf, "blk", "inp", "oth", out_dev="cpu") == "out"+ bf.assert_called_once_with("blk", "inp", "oth", cache_device="cpu")++ def test_collect_forward_sharded_when_devices_set(self):+ ctx = TuneParallelContext()+ ctx.devices = [torch.device("cpu"), torch.device("cpu")]+ with mock.patch(+ "auto_round.algorithms.parallel.tune_parallel.sharded_nograd_forward",+ return_value="sharded",+ ) as snf:+ assert ctx.collect_forward("bf", "blk", "inp", "oth") == "sharded"+ from auto_round.algorithms.parallel.data_parallel import ENGINE_CALIBRATION_STATS++ snf.assert_called_once_with(+ "bf",+ "blk",+ "inp",+ "oth",+ None,+ ctx.devices,+ merge_stats=ENGINE_CALIBRATION_STATS,+ pool=None,+ max_devices=0,+ )++ def test_collect_forward_undeclared_stats_stay_serial(self):+ """merge_stats=None (quantizer hooks without a calibration_stats+ declaration): the pass must not shard -- the mirrors' statistics+ could not be folded back."""+ ctx = TuneParallelContext()+ ctx.devices = [torch.device("cpu"), torch.device("cpu")]+ bf = mock.Mock(return_value="serial")+ with mock.patch("auto_round.algorithms.parallel.tune_parallel.sharded_nograd_forward") as snf:+ assert ctx.collect_forward(bf, "blk", "inp", "oth", hook_pass=True, merge_stats=None) == "serial"+ snf.assert_not_called()+ bf.assert_called_once_with("blk", "inp", "oth", cache_device=None)++ def test_collect_forward_hook_pass_cap_from_policy(self):+ ctx = TuneParallelContext()+ ctx.devices = [torch.device("cpu")] * 8+ with mock.patch(+ "auto_round.algorithms.parallel.tune_parallel.sharded_nograd_forward",+ return_value="sharded",+ ) as snf:+ ctx.collect_forward("bf", "blk", "inp", "oth", hook_pass=True)+ # default: no cap (full-width hook passes)+ assert snf.call_args.kwargs["max_devices"] == 0+ # the cap is set on the context (folded from the env into the+ # ParallelPolicy at the entry)+ ctx.collect_forward_cap = 2+ ctx.collect_forward("bf", "blk", "inp", "oth", hook_pass=True)+ assert snf.call_args.kwargs["max_devices"] == 2+ # non-hook passes stay uncapped+ ctx.collect_forward("bf", "blk", "inp", "oth", hook_pass=False)+ assert snf.call_args.kwargs["max_devices"] == 0++ def test_capped_hook_pass_keeps_the_pool(self):+ """The cap limits WHICH pool replicas run, never pool eligibility:+ the full-width pool is passed through with max_devices set."""+ ctx = TuneParallelContext()+ ctx.devices = [torch.device("cpu")] * 8+ ctx.collect_forward_cap = 2+ sentinel = object()+ ctx.pool = sentinel+ blk = torch.nn.Linear(2, 2) # pool eligibility requires a real Module+ with mock.patch(+ "auto_round.algorithms.parallel.tune_parallel.sharded_nograd_forward",+ return_value="sharded",+ ) as snf:+ assert ctx.collect_forward("bf", blk, [1] * 8, "oth", hook_pass=True) == "sharded"+ assert snf.call_args.kwargs["pool"] is sentinel+ assert snf.call_args.kwargs["max_devices"] == 2++ def test_distribute_pools_noop_without_devices(self):+ ctx = TuneParallelContext()+ pool = [torch.zeros(1), torch.zeros(1)]+ with mock.patch("auto_round.algorithms.parallel.tune_parallel.distribute_pool") as dp:+ ctx.distribute_pools(pool, None)+ dp.assert_not_called()+++def _autoround_log_propagate():+ """Temporarily enable propagation on the ``autoround`` logger so pytest's+ caplog fixture (handler at the root logger) can capture warnings; the+ logger is configured with propagate=False in production."""++ _logger = logging.getLogger("autoround")+ original = _logger.propagate+ _logger.propagate = True+ yield+ _logger.propagate = originaldiff --git a/test/unit/test_cuda/quantization/test_ddp_mirror.py b/test/unit/test_cuda/quantization/test_ddp_mirror.pynew file mode 100644index 000000000..e4c276ed4--- /dev/null+++ b/test/unit/test_cuda/quantization/test_ddp_mirror.py@@ -0,0 +1,87 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""CUDA-tier tests for DDP mirror building.++The resolver requires single-device block placement: a block whose weights+span several CUDA devices is declined loudly.+"""++import pytest+import torch++from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_++_requires_multi_cuda = pytest.mark.skipif(+ torch.cuda.device_count() < 2, reason="needs >=2 CUDA devices for a sharded source block"+)+++class _TinyBlock(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.l0 = torch.nn.Linear(8, 8, bias=False)+ self.l1 = torch.nn.Linear(8, 8, bias=False)++ def forward(self, x):+ return self.l1(torch.nn.functional.relu(self.l0(x)))+++def _make_sharded_block():+ block = _TinyBlock().cuda(0)+ block.l0.tuning_device = "cuda:0"+ block.l1.tuning_device = "cuda:1"+ block.l1.to("cuda:1") # emulate the auto block device map shard+ # plain-attr tensors + a params-dict entry, the wrapper state _relocate covers+ block.l0.anchor = torch.zeros(4, device="cuda:0")+ block.l1.anchor = torch.zeros(4, device="cuda:1")+ block.l0.params = {"v": torch.nn.Parameter(torch.ones(8, device="cuda:0"), requires_grad=True)}+ block.l1.params = {"v": torch.nn.Parameter(torch.ones(8, device="cuda:1"), requires_grad=True)}+ return block+++def _make_whole_block():+ block = _TinyBlock().cuda(0)+ block.l0.tuning_device = "cuda:0"+ block.l1.tuning_device = "cuda:0"+ block.l0.anchor = torch.zeros(4, device="cuda:0")+ block.l1.anchor = torch.zeros(4, device="cuda:0")+ block.l0.params = {"v": torch.nn.Parameter(torch.ones(8, device="cuda:0"), requires_grad=True)}+ block.l1.params = {"v": torch.nn.Parameter(torch.ones(8, device="cuda:0"), requires_grad=True)}+ return block+++def _quantizer():+ from types import SimpleNamespace++ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)+ q._get_scaler = lambda: None+ return q+++def _quantizer_with_policy(world=2):+ from types import SimpleNamespace++ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy++ q = _quantizer()+ q.compress_context = SimpleNamespace(parallel_policy=ParallelPolicy(world=world))+ return q+++@_requires_multi_cuda+class TestShardedSourceDeclines:+ def test_spanning_block_raises_with_placement_reason(self, monkeypatch):+ block = _make_sharded_block()+ with pytest.raises(RuntimeError, match="spans 2 CUDA devices"):+ resolve_tune_ddp_plan_(_quantizer_with_policy(), block, [torch.zeros(1, 8)], None, "cuda:0")diff --git a/test/unit/test_cuda/quantization/test_ddp_parity.py b/test/unit/test_cuda/quantization/test_ddp_parity.pynew file mode 100644index 000000000..4fe03f57c--- /dev/null+++ b/test/unit/test_cuda/quantization/test_ddp_parity.py@@ -0,0 +1,135 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""CUDA-tier serial-vs-DDP result parity: RTN iters=0 and the default+SignRound tune at iters=2.++Both arms quantize the SAME tiny model (same fixture path -> identical+weights) with the SAME calibration samples. The DDP arm engages the+parallel lane via a world=2 ParallelPolicy on the compress context and receives the SAME samples per+rank through pinned draws: each iteration the serial lane processes batch+[a, b] while rank 0 draws [a] and rank 1 draws [b]+-- mean-of-shard-means equals the full-batch mean, so SignSGD consumes+identical gradients and both lanes evolve identically.++The final quantized weights are asserted BIT-EXACT (torch.equal). For+iters=0 this is guaranteed (the same deterministic per-module quantize on+identical weights). For iters=2 the two lanes are different floating-point+programs (batched reduction vs shard+average, fp32 gradient exchange), so+bitwise equality is expected via the discrete sign steps + the final+round-to-grid; a mismatch is a real signal to investigate (transport+precision or reduction order), matched exactly.+"""++import pytest+import torch++pytestmark = pytest.mark.skipif(torch.cuda.device_count() < 2, reason="needs 2+ CUDA devices")++_DS = [+ "CUDA-tier DDP parity sample with enough tokens for quantization",+ "Another parity sample with a different length padding the batch",+ "Third sample keeps the calibration pool at four entries",+ "Fourth sample completes the pinned draw schedule",+]+++class _FixedSampler:+ """Yields a fixed global-index schedule (list of lists) in order."""++ def __init__(self, draws):+ self._draws = list(draws)+ self._i = 0++ def next_batch(self):+ out = self._draws[self._i]+ self._i += 1+ return out+++def _autoround(model_path, iters, disable_opt_rtn=False):+ from auto_round import AutoRound++ return AutoRound(+ model_path,+ bits=4,+ group_size=32,+ sym=False,+ iters=iters,+ nsamples=4,+ seqlen=8,+ batch_size=2,+ dataset=list(_DS),+ device_map="0",+ enable_torch_compile=False,+ disable_opt_rtn=disable_opt_rtn,+ disable_model_free=True,+ )+++def _run(model_path, iters, disable_opt_rtn, monkeypatch, pin_draws, world=1):+ import auto_round.algorithms.quantization.sign_round.quantizer as v1+ from auto_round.algorithms.parallel import spmd+ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy++ if pin_draws:+ # serial lane: batches [0, 1] then [2, 3]; rank r draws the r-th+ # sample of the SAME global batch per iteration+ def _pinned(nsamples, batch):+ rank = spmd.current_rank()+ if rank is None:+ return _FixedSampler([[0, 1], [2, 3]])+ return _FixedSampler([[0], [2]]) if rank.rank == 0 else _FixedSampler([[1], [3]])++ monkeypatch.setattr(v1, "make_index_sampler", _pinned)+ ar = _autoround(model_path, iters, disable_opt_rtn)+ if world > 1:+ # the policy is set on the compress context (the CLI does it the same way)+ ar.compress_context.parallel_policy = ParallelPolicy(world=world)+ model, layer_config = ar.quantize()+ sd = model.state_dict()+ state = {}+ for name, cfg in layer_config.items():+ if cfg.get("data_type") != "int":+ continue+ key = name + ".weight"+ assert key in sd, f"quantized layer {name} has no weight in state_dict"+ state[name] = sd[key].detach().cpu().clone()+ assert state, "no quantized layers found"+ return state+++def _assert_same_weights(serial, ddp):+ assert set(serial) == set(ddp), "quantized layer sets differ"+ for name in serial:+ assert torch.equal(serial[name], ddp[name]), (+ f"layer {name}: serial vs DDP final weights differ "+ f"(max abs diff {float((serial[name] - ddp[name]).abs().max())})"+ )+++def test_rtn_iters0_serial_vs_ddp_bitexact(tiny_opt_model_path, monkeypatch):+ """Plain RTN (iters=0, disable_opt_rtn): sharded collection forwards plus+ the home quantize and mirror sync must reproduce the serial lane bit-exactly."""+ serial = _run(tiny_opt_model_path, 0, True, monkeypatch, pin_draws=False)+ ddp = _run(tiny_opt_model_path, 0, True, monkeypatch, pin_draws=False, world=2)+ _assert_same_weights(serial, ddp)+++def test_signround_iters2_serial_vs_ddp_bitexact(tiny_opt_model_path, monkeypatch):+ """Default SignRound tune at iters=2: pinned shard draws give every+ replica the serial lane's samples, so both lanes must land on identical+ final quantized weights."""+ serial = _run(tiny_opt_model_path, 2, False, monkeypatch, pin_draws=True)+ ddp = _run(tiny_opt_model_path, 2, False, monkeypatch, pin_draws=True, world=2)+ _assert_same_weights(serial, ddp)