Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat: data-parallel block tuning via --data_parallel (#2351)

avtc6c824f741be3
diff --git a/auto_round/algorithms/block_runner.py b/auto_round/algorithms/block_runner.py
index 358303600..5c889809f 100644
--- a/auto_round/algorithms/block_runner.py
+++ b/auto_round/algorithms/block_runner.py
@@ -28,6 +28,7 @@ from typing import TYPE_CHECKING, Any
import torch
from auto_round.compressors.utils import block_forward
+from auto_round.logger import logger
from auto_round.utils.device_manager import device_manager
if TYPE_CHECKING:
@@ -87,6 +88,28 @@ register_block_output("GlmMoeDsaDecoderLayer", ["hidden_states", "prev_topk_indi
# TODO wenhuach better follow heng's imp to decouple llm/diffusion
+def _cat_device_safe(tensors: list, dim: int) -> torch.Tensor:
+ """Concatenate per-sample tensors that may live on different devices.
+
+ Distributed calibration pools keep each sample on its owning DDP device;
+ batch selection that crosses owners moves the minority to the first
+ piece's device (same-device selections -- the common DDP case -- stay
+ copy-free).
+ """
+ if not tensors:
+ raise ValueError("_cat_device_safe: empty selection")
+ dev = tensors[0].device
+ if any(t.device != dev for t in tensors):
+ logger.debug(
+ "_cat_device_safe: moved %d/%d pieces to %s",
+ sum(1 for t in tensors if t.device != dev),
+ len(tensors),
+ dev,
+ )
+ tensors = [t.to(dev) for t in tensors]
+ return torch.cat(tensors, dim=dim)
+
+
class BlockForwardRunner:
"""Stateless block-forward execution engine shared across quantizer & compressor.
@@ -191,6 +214,15 @@ class BlockForwardRunner:
if indices is not None, this func returns tensor, otherwise list
Normalized output tensor on ``cache_device`` (or ``self.cache_device``).
"""
+ if cache_device is None:
+ from auto_round.algorithms.parallel.spmd import current_rank
+
+ rank = current_rank()
+ if rank is not None:
+ # a data-parallel rank keeps its outputs on its own device: the
+ # loss is staged there, and parking them on the shared cache
+ # device would copy every rank's output out and back each step
+ cache_device = rank.device
out_device = cache_device if cache_device is not None else self.cache_device
# Per-chunk calibration-data placement (--calibration_data_device): list mode (indices is
# None -- the cat at the end of indices-mode requires same-device outputs)
@@ -305,32 +337,58 @@ class BlockForwardRunner:
# ── Private ──────────────────────────────────────────────────────────────
+ def _forward_device(self, block) -> torch.device:
+ """Staging device for a forward pass through ``block``.
+
+ Normally the runner's global device. The one exception is a block
+ that sits WHOLE on a single accelerator device other than the runner
+ device -- a DDP mirror replica: staging its inputs to the global
+ (primary) device would put the hidden states on a different GPU than
+ the replica's weights. Blocks that span several accelerator devices
+ (upstream sharded placement, routed by their align hooks) and
+ mixed CPU/GPU blocks (CPU-pinned tables) keep the global device, so
+ serial behavior is unchanged.
+ """
+ try:
+ devs = {pp.device for pp in block.parameters()}
+ except Exception as e: # pragma: no cover - exotic modules
+ logger.warning("block device sniff failed (%s); assuming the runner device", e)
+ return self.device
+ acc = {d for d in devs if d.type in ("cuda", "xpu", "hpu")}
+ if len(acc) == 1 and devs <= (acc | {torch.device("cpu")}):
+ (d,) = acc
+ if d != self.device:
+ return d
+ return self.device
+
def _forward_one_batch(self, block, batch_inputs, batch_others) -> Any:
"""Forward one already-selected batch through the block (raw output)."""
+ # Stage on the block's actual device (see _forward_device). The shared
+ # kwargs follow the hidden states so masks/position ids reach the same
+ # GPU as the weights; for the serial path both are already there and
+ # the moves are no-ops.
+ from auto_round.utils.model import to_device
+
+ fwd_device = self._forward_device(block)
if isinstance(batch_inputs, dict):
batch_inputs = dict(batch_inputs)
batch_others = dict(batch_others)
hidden_states = batch_inputs.pop("hidden_states")
batch_others.update(batch_inputs)
- return self.block_forward(
- block,
- hidden_states,
- batch_others,
- self.amp,
- self.amp_dtype,
- self.device,
- None,
- )
else:
- return self.block_forward(
- block,
- batch_inputs,
- batch_others,
- self.amp,
- self.amp_dtype,
- self.device,
- None,
- )
+ hidden_states = batch_inputs
+ if torch.is_tensor(hidden_states) and hidden_states.device != fwd_device:
+ hidden_states = hidden_states.to(fwd_device)
+ batch_others = to_device(batch_others, fwd_device)
+ return self.block_forward(
+ block,
+ hidden_states,
+ batch_others,
+ self.amp,
+ self.amp_dtype,
+ fwd_device,
+ None,
+ )
def _count_samples(self, inputs: Any) -> int:
if isinstance(inputs, dict):
@@ -400,6 +458,11 @@ class BlockForwardRunner:
"""Select a subset of inputs by indices."""
batch_dim = self.batch_dim
shared_cache_keys = self.shared_cache_keys
+ # Host ints, resolved once. ``indices`` rides on the inputs' device, so a
+ # per-element ``int(i)`` read syncs an accelerator once per element;
+ # every host-side consumer below draws from this list instead. Device
+ # consumers (index_select on the inputs) keep the original tensor.
+ idx_list = indices.tolist() if isinstance(indices, torch.Tensor) else list(indices)
if isinstance(inputs, dict):
selected_inputs = {}
@@ -408,13 +471,13 @@ class BlockForwardRunner:
if isinstance(val, list) and len(val) == 1:
selected_inputs[key] = val[0]
elif isinstance(val, list) and len(val) > 1:
- idx = int(indices[0]) if len(indices) == 1 else 0
+ idx = idx_list[0] if len(idx_list) == 1 else 0
selected_inputs[key] = val[idx] if idx < len(val) else val[0]
else:
selected_inputs[key] = val
else:
if isinstance(val, list):
- batch_vals = self._gather_same_device([val[i] for i in indices], self.device)
+ batch_vals = self._gather_same_device([val[i] for i in idx_list], self.device)
selected_inputs[key] = torch.cat(batch_vals, dim=batch_dim)
elif isinstance(val, torch.Tensor):
# relocate indices like the input_others branch below:
@@ -426,35 +489,105 @@ class BlockForwardRunner:
selected_inputs[key] = val
else:
if isinstance(inputs, list):
- batch_vals = self._gather_same_device([inputs[i] for i in indices], self.device)
+ batch_vals = self._gather_same_device([inputs[i] for i in idx_list], self.device)
selected_inputs = torch.cat(batch_vals, dim=batch_dim)
else:
selected_inputs = torch.index_select(inputs, batch_dim, indices)
selected_others = {"positional_inputs": input_others.get("positional_inputs")}
+ def _take_rows(t, rows):
+ """Rows of ``t`` on the batch dim. A broadcast (size-1) batch dim
+ passes through unsliced, as does a tensor too short for ``rows``
+ (with a warning). The range is checked on the host: on CUDA an
+ out-of-range index_select is a device-side assert, not an exception."""
+ rows = rows.cpu() # host-side bounds check (no-op for host rows)
+ size = t.shape[batch_dim] if t.dim() > batch_dim else 0
+ if size == 1:
+ return t
+ if not len(rows) or int(rows.max()) >= size:
+ logger.warning_once(
+ "kwarg row slicing fell back to the unsliced tensor (batch size %d, rows up to %d); "
+ "sub-batch draws may see the wrong batch's entry",
+ size,
+ int(rows.max()) if len(rows) else -1,
+ )
+ return t
+ return t.index_select(batch_dim, rows.to(device=t.device))
+
+ def _slice_shared_entry(entry, rows):
+ """Within-batch row slice of one shared-cache entry (batch pick already done)."""
+ if isinstance(entry, torch.Tensor):
+ return _take_rows(entry, rows)
+ if isinstance(entry, tuple) and entry and all(torch.is_tensor(t) for t in entry):
+ return tuple(_take_rows(t, rows) for t in entry)
+ return entry
+
for key, val in input_others.items():
if "positional_inputs" in key:
continue
if key in shared_cache_keys:
if isinstance(val, list) and len(val) == 1:
selected_others[key] = val[0]
+ elif isinstance(val, list) and len(val) > 1 and inputs is not None and len(indices) >= 1:
+ # Two cached-list layouts exist: per-SAMPLE (len == pool size)
+ # and per-BATCH (len == pool / collection batch; entries are
+ # whole collection-batch tensors/tuples). The collection batch
+ # comes from the cache itself: a data-parallel rank tunes with
+ # its share of the batch, so its own batch_size is smaller.
+ _n = self._count_samples(inputs)
+ _per_entry = -(-_n // len(val)) if _n > len(val) else 1
+ if len(val) == _n and len(indices) > 1:
+ # per-sample list: pick the drawn samples' entries
+ _picks = [val[i] if i < len(val) else val[0] for i in idx_list]
+ if len(_picks) == 1:
+ selected_others[key] = _picks[0]
+ elif all(isinstance(pp, tuple) and pp and all(torch.is_tensor(t) for t in pp) for pp in _picks):
+ selected_others[key] = tuple(
+ _cat_device_safe([pp[slot] for pp in _picks], dim=batch_dim)
+ for slot in range(len(_picks[0]))
+ )
+ elif all(torch.is_tensor(pp) for pp in _picks):
+ selected_others[key] = _cat_device_safe(_picks, dim=batch_dim)
+ else:
+ selected_others[key] = val[0]
+ elif len(indices) < _per_entry:
+ # draw smaller than the collection batch (a rank's share):
+ # the owning batch's entry, sliced to the drawn rows
+ _b = idx_list[0] // _per_entry
+ _entry = val[_b] if 0 <= _b < len(val) else val[0]
+ _rows = torch.tensor([i % _per_entry for i in idx_list], dtype=torch.long)
+ selected_others[key] = _slice_shared_entry(_entry, _rows)
+ else:
+ idx = idx_list[0] if len(idx_list) == 1 else 0
+ selected_others[key] = val[idx] if idx < len(val) else val[0]
elif isinstance(val, list) and len(val) > 1:
- idx = int(indices[0]) if len(indices) == 1 else 0
+ idx = idx_list[0] if len(idx_list) == 1 else 0
selected_others[key] = val[idx] if idx < len(val) else val[0]
else:
selected_others[key] = val
elif isinstance(val, list):
- batch_vals = self._gather_same_device([val[i] for i in indices], self.device)
+ batch_vals = self._gather_same_device([val[i] for i in idx_list], self.device)
if len(batch_vals) == 1:
selected_others[key] = batch_vals[0]
else:
- selected_others[key] = torch.cat(batch_vals, dim=batch_dim)
+ selected_others[key] = _cat_device_safe(batch_vals, dim=batch_dim)
elif isinstance(val, torch.Tensor):
# ``batch_indices`` are created on CPU by the sampler. XPU
# (and other accelerator backends) require index tensors on
# the same device as the indexed value.
selected_others[key] = torch.index_select(val, batch_dim, indices.to(device=val.device))
+ elif isinstance(val, tuple) and val and all(torch.is_tensor(t) for t in val):
+ # tuple-of-tensors kwargs (transformers v5 passes rope as
+ # ``position_embeddings=(cos, sin)`` with a per-sample batch
+ # dim). Slice each element like the tensor branch; elements
+ # that are shared across samples (e.g. broadcast [1, S, D]
+ # tables) fail the select and pass through unsliced, preserving their
+ # broadcast behavior. Without this, shard/batch forwards
+ # smaller than the cached batch crash in apply_rotary_pos_emb
+ # (hidden batch N vs cos batch full).
+ _idx = torch.tensor(idx_list, dtype=torch.long)
+ selected_others[key] = tuple(_take_rows(t, _idx) for t in val)
elif isinstance(val, (str, bool, type(None))):
selected_others[key] = val
else:
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.py
index 98f850100..7d5001dd4 100644
--- a/auto_round/algorithms/composer.py
+++ b/auto_round/algorithms/composer.py
@@ -42,6 +42,11 @@ from auto_round.algorithms.config_resolver import (
resolve_shared_config_values,
split_quantization_configs,
)
+from auto_round.algorithms.parallel.data_parallel import (
+ ENGINE_CALIBRATION_STATS,
+ calibration_merge_stats,
+ parallel_state,
+)
from auto_round.algorithms.utils import _has_nvfp4_layer
from auto_round.logger import logger
from auto_round.utils import clear_memory
@@ -360,22 +365,30 @@ class AlgorithmComposer:
return register_act_max_hooks(block, getattr(self.block_quantizer.config, "act_data_type", None))
- def _get_fp_act_hooks(self, block: torch.nn.Module) -> list:
- """Register FP-input act_max + quantizer forward hooks."""
+ def _get_fp_act_hooks(self, block: torch.nn.Module) -> tuple[list, dict[str, str] | None]:
+ """Register FP-input act_max + quantizer forward hooks.
+
+ Returns ``(handles, merge_stats)``: the shard-merge map of the
+ statistics these hooks write (``None``: the pass stays serial, see
+ ``calibration_merge_stats``).
+ """
if not self.need_quanted_input():
# If having q_input, act_max will be collected in q_input forward hook,
# no need to collect in fp_input forward hook
handles = self._register_act_max_hooks(block)
else:
handles = []
- handles.extend(self.block_quantizer.register_fp_input_forward_hooks(block))
- return handles
+ quantizer_hooks = self.block_quantizer.register_fp_input_forward_hooks(block)
+ handles.extend(quantizer_hooks)
+ return handles, calibration_merge_stats(self.block_quantizer, quantizer_hooks)
- def _get_q_act_hooks(self, block: torch.nn.Module) -> list:
- """Register Q-input act_max + quantizer forward hooks."""
+ def _get_q_act_hooks(self, block: torch.nn.Module) -> tuple[list, dict[str, str] | None]:
+ """Register Q-input act_max + quantizer forward hooks; returns
+ ``(handles, merge_stats)`` like ``_get_fp_act_hooks``."""
handles = self._register_act_max_hooks(block)
- handles.extend(self.block_quantizer.register_qinput_forward_hooks(block))
- return handles
+ quantizer_hooks = self.block_quantizer.register_qinput_forward_hooks(block)
+ handles.extend(quantizer_hooks)
+ return handles, calibration_merge_stats(self.block_quantizer, quantizer_hooks)
def _attach_act_max_for_outside_layer(self, layer: torch.nn.Module, fp_inputs, q_inputs) -> None:
"""Compute and attach act_max for an outside-block layer from cached inputs.
@@ -432,6 +445,62 @@ class AlgorithmComposer:
def compress_embedding_layer(self):
return self.block_quantizer.quantize_embedding_layer()
+ def _collection_context(self, block, fp_inputs):
+ """TuneParallelContext for the collection phase (P3): mirror devices
+ for sharding the no-grad collection forwards, with the same engagement
+ resolver as the tune so the gates can never diverge."""
+ from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext
+
+ return TuneParallelContext.for_collection(self, block, fp_inputs)
+
+ def _collect_forward_timed(self, *args, **kwargs):
+ """_collect_forward with wall accumulation into self.last_collect_wall."""
+ import time as _ctime
+
+ _t0 = _ctime.perf_counter()
+ try:
+ return self._collect_forward(*args, **kwargs)
+ finally:
+ self.last_collect_wall = getattr(self, "last_collect_wall", 0.0) + (_ctime.perf_counter() - _t0)
+
+ def _collect_forward(
+ self,
+ block,
+ inputs,
+ input_others,
+ out_dev=None,
+ allow_shard: bool = True,
+ hook_pass: bool = False,
+ merge_stats=ENGINE_CALIBRATION_STATS,
+ ):
+ """Collection forward: sharded across DDP mirrors when eligible.
+
+ ``out_dev`` optionally overrides the runner's output cache device.
+ ``allow_shard=False`` forces the serial path. ``merge_stats`` is the
+ shard-merge map of the statistics the pass's hooks write (from
+ ``_get_fp_act_hooks`` / ``_get_q_act_hooks``); ``None`` keeps the
+ pass serial (undeclared quantizer statistics).
+
+ ``hook_pass=True`` may cap the concurrent shards (env-driven; default
+ no cap): forward hooks
+ force dynamo graph breaks, leaving the compiled runner as
+ python-bound eager sections that GIL-convoy under many threads.
+ """
+ if self._coll_ctx is None:
+ from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext
+
+ self._coll_ctx = TuneParallelContext()
+ return self._coll_ctx.collect_forward(
+ self.block_forward,
+ block,
+ inputs,
+ input_others,
+ out_dev,
+ allow_shard=allow_shard,
+ hook_pass=hook_pass,
+ merge_stats=merge_stats,
+ )
+
# ── Per-block pipeline orchestration ─────────────────────────────────────
def compress_block(
@@ -470,8 +539,15 @@ class AlgorithmComposer:
``enable_quanted_input`` is ``False``).
- *reference_output*: FP reference output collected before optimization.
"""
+ self.last_collect_wall = 0.0
block_forward_fn = self.block_forward
+ self._coll_ctx = self._collection_context(block, fp_inputs)
+ # distributed calibration pool: each DDP device owns its sample
+ # shard (matching the tune shards), so shard-local reads stay
+ # device-local; serial consumers use device-safe cats
+ self._coll_ctx.distribute_pools(fp_inputs, q_inputs)
+
# ── Step 0: Layer-wise rotation (before any reference/calibration) ────
# Each rotation member rotates this block's weights and installs online
# hooks so all downstream calibration and reference collection operate on
@@ -537,9 +613,18 @@ class AlgorithmComposer:
# ── Step 3: Quantizer calibration (act_max, imatrix, etc.) ─────────────
if fp_inputs is not None:
with torch.no_grad():
- quant_hooks = self._get_fp_act_hooks(block)
+ quant_hooks, merge_stats = self._get_fp_act_hooks(block)
if reference_output is None or reference_invalidated:
- reference_output = block_forward_fn(block, fp_inputs, input_others)
+ reference_output = self._collect_forward_timed(
+ block,
+ fp_inputs,
+ input_others,
+ # the declared statistics (act_max, the quantizer's
+ # calibration_stats) are folded from the mirrors, so
+ # hook-carrying passes may shard
+ hook_pass=bool(quant_hooks),
+ merge_stats=merge_stats,
+ )
reference_next_input = getattr(block_forward_fn, "last_output_dict", None) or reference_output
else:
# No forward happened in this step, so last_output_dict may
@@ -549,9 +634,15 @@ class AlgorithmComposer:
h.remove()
if self.block_quantizer.enable_quanted_input:
- q_hooks = self._get_q_act_hooks(block)
+ q_hooks, q_merge_stats = self._get_q_act_hooks(block)
if q_hooks:
- block_forward_fn(block, q_inputs if q_inputs is not None else fp_inputs, input_others)
+ self._collect_forward_timed(
+ block,
+ q_inputs if q_inputs is not None else fp_inputs,
+ input_others,
+ hook_pass=bool(q_hooks),
+ merge_stats=q_merge_stats,
+ )
for h in q_hooks:
h.remove()
@@ -588,6 +679,32 @@ class AlgorithmComposer:
# lanes without a separate q-input pool (first block; quanted-input
# disabled) still get the pre-tuning cache clear the base code had
clear_memory()
+ # ── Step 3.6: mirror-pool handoff ──────────────────────────────────────
+ # The collection passes may have left a per-block mirror pool alive
+ # (built once, reused across the collection forwards). At iters=0 the
+ # RTN/OptRTN quantize runs on the home block, the mirrors are synced
+ # to it afterwards and the cascade forwards run on them; at iters>0
+ # the tune ranks tune the mirrors (spmd.quantize_block_on_ranks).
+ from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer
+ from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer
+
+ # pool readers: the RTN family (its mirrors are synced to the
+ # quantized home block for the sharded cascade) and the SignRound
+ # family (its tune ranks run on the mirrors). NOTE: this ensure
+ # is only the SAFETY NET for when no collection pass was
+ # shard-eligible -- ref-collect sharding builds the pool lazily for
+ # EVERY lane and is untouched by this gate.
+ _pool_consumer = isinstance(self.block_quantizer, (RTNQuantizer, SignRoundQuantizer))
+ if _pool_consumer and getattr(self._coll_ctx, "devices", None) and len(self._coll_ctx.devices) >= 2:
+ self._coll_ctx.ensure_pool(block)
+ _pool = getattr(self._coll_ctx, "pool", None)
+ if _pool is not None:
+ # the tune ranks take the resident mirrors (no second weight copy);
+ # the RTN family leaves them to _settle_block_pool. A pool nobody
+ # consumed is released before the post-quantize steps below.
+ _state = parallel_state(self.block_quantizer, create=True)
+ _state.pool = _pool
+ _state.pool_used = False
# ── Step 4: quantize_block ──────────────────────────────────────────────
# When quantized input is available from the previous block, use it;
# otherwise fall back to the FP input.
@@ -600,36 +717,89 @@ class AlgorithmComposer:
output = block_forward_fn(wrapped_block, effective_input, input_others)
warm_start_outputs.append(getattr(block_forward_fn, "last_output_dict", None) or output)
- self._run_tuning(
- block,
- lambda: self.block_quantizer.quantize_block(
+ def tune_block():
+ # data-parallel tune: the unchanged quantize_block runs once per
+ # rank (one thread per device); otherwise the serial call
+ from auto_round.algorithms.parallel.spmd import quantize_block_on_ranks
+
+ engaged, rank0_params = quantize_block_on_ranks(
+ self.block_quantizer,
+ block,
+ effective_input,
+ input_others,
+ reference_output,
+ q_inputs,
+ block_ctx,
+ input_ids=input_ids,
+ )
+ if engaged:
+ return rank0_params
+ return self.block_quantizer.quantize_block(
block, effective_input, input_others, reference_output, q_inputs, block_ctx, input_ids=input_ids
- ),
- collect_warm_start_output,
- )
+ )
- # Quantizer-agnostic block finalization (e.g. NVFP MoE act_max), kept
- # out of the per-algorithm tuning loop.
- self.block_quantizer.finalize_block(block, block_ctx)
+ try:
+ self._run_tuning(block, tune_block, collect_warm_start_output)
- # ── Step 5: post_quantize_block ─────────────────────────────────────────
- for pre in self.preprocessors:
- pre.post_quantize_block(block_ctx)
-
- # ── Step 6: Collect quantized-block outputs for the next block ──────────
- if warm_start_outputs:
- # Warm-up has restored FP weights by now; use the output collected
- # while the wrappers still held the best quantization parameters.
- new_q_input = warm_start_outputs[0]
- elif self.block_quantizer.enable_quanted_input:
- with torch.no_grad():
- new_q_input = block_forward_fn(block, effective_input, input_others)
- new_q_input = getattr(block_forward_fn, "last_output_dict", None) or new_q_input
- else:
- new_q_input = None
+ # Quantizer-agnostic block finalization (e.g. NVFP MoE act_max), kept
+ # out of the per-algorithm tuning loop.
+ self.block_quantizer.finalize_block(block, block_ctx)
+
+ # the pool is block-scoped and belongs to the shared tune state:
+ # settle it (sync the mirrors for the RTN family, keep or release
+ # it) before the post-quantize steps and the Step-6 cascade
+ _pool = self._settle_block_pool(block, _pool)
+
+ # ── Step 5: post_quantize_block ─────────────────────────────────────
+ for pre in self.preprocessors:
+ pre.post_quantize_block(block_ctx)
+
+ # ── Step 6: Collect quantized-block outputs for the next block ──────
+ if warm_start_outputs:
+ # Warm-up has restored FP weights by now; use the output collected
+ # while the wrappers still held the best quantization parameters.
+ new_q_input = warm_start_outputs[0]
+ elif self.block_quantizer.enable_quanted_input:
+ with torch.no_grad():
+ new_q_input = self._collect_forward_timed(block, effective_input, input_others)
+ new_q_input = getattr(block_forward_fn, "last_output_dict", None) or new_q_input
+ else:
+ new_q_input = None
+ finally:
+ # the pool is block-scoped: drop it even on failures so the next
+ # block's pool starts clean
+ _state = parallel_state(self.block_quantizer)
+ if _state is not None:
+ _state.pool = None
+ self._coll_ctx.release_pool()
return new_q_input, reference_next_input
+ def _settle_block_pool(self, block, pool):
+ """Pool disposition after quantize_block; returns the pool kept for the Step-6 cascade.
+
+ - zero-shot RTN/OptRTN quantized the HOME block through main's search
+ lane (which spreads its searches over idle devices itself) -> sync
+ the quantized home state onto the mirrors and keep the pool, so the
+ cascade still runs sharded;
+ - the tune ranks took the mirrors (they end the block in their own
+ tuned state while home was unwrapped with the BEST params) -> drop
+ the pool; the cascade runs on the canonical home block;
+ - unconsumed (declined lane / vanished plan) -> release.
+ """
+ if pool is None:
+ return None
+ from auto_round.algorithms.parallel.mirror_sync import sync_mirrors_to_home
+ from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer
+
+ if isinstance(self.block_quantizer, RTNQuantizer):
+ sync_mirrors_to_home(self.block_quantizer, block)
+ _state = parallel_state(self.block_quantizer)
+ if _state is None or _state.pool is None or not _state.pool_used:
+ self._coll_ctx.release_pool()
+ return None
+ return pool
+
def compress_layer_outside_block(
self,
layer: torch.nn.Module,
diff --git a/auto_round/algorithms/parallel/__init__.py b/auto_round/algorithms/parallel/__init__.py
new file mode 100644
index 000000000..efef15e6b
--- /dev/null
+++ b/auto_round/algorithms/parallel/__init__.py
@@ -0,0 +1,16 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Engine-level single-process data-parallel machinery (policy, plans,
+mirroring, collectives, sharded execution). Algorithm code consumes it
+through ``TuneParallelContext``; it owns no algorithm specifics."""
diff --git a/auto_round/algorithms/parallel/data_parallel.py b/auto_round/algorithms/parallel/data_parallel.py
new file mode 100644
index 000000000..e7e0cb7f4
--- /dev/null
+++ b/auto_round/algorithms/parallel/data_parallel.py
@@ -0,0 +1,1197 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Single-process data parallelism for the SignRound tuning loop.
+
+Runs one block's iteration loop on the home GPU plus W-1 mirror replicas
+(persistent deep copies of the wrapped block). Per iteration the global
+calibration batch is sharded across replicas; each replica computes its
+forward/loss/backward on its own device; gradients are exchanged with a
+halving-doubling all-reduce over flat fp32 buffers so every replica ends
+with the same averaged gradient; the deterministic SignSGD step is then
+applied locally on every replica (identical inputs -> identical update),
+which keeps mirrors in sync without any parameter broadcast.
+
+World size 1 (default) executes none of this code.
+"""
+
+from __future__ import annotations
+
+import copy
+import threading
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from types import MappingProxyType
+
+import torch
+
+from auto_round.logger import logger
+
+
+@dataclass
+class DDPPlan:
+ """Resolved data-parallel plan for one block."""
+
+ world: int
+ devices: list[torch.device]
+ shard_size: int
+ notes: list[str] = field(default_factory=list)
+
+ @property
+ def enabled(self) -> bool:
+ return self.world > 1
+
+
+def _parse_device_token(tok: str, dev_type: str = "cuda") -> torch.device:
+ """Parse an explicit-device token: ``3`` -> ``<dev_type>:3``, ``cuda:3``/``xpu:3`` as-is."""
+ tok = tok.strip()
+ if tok.isdigit():
+ return torch.device(dev_type, int(tok))
+ return torch.device(tok)
+
+
+_SUPPORTED_ACCEL_TYPES = ("cuda", "xpu", "hpu")
+
+
+def _accel_device_count(dev_type: str) -> int:
+ """Device count for an accelerator backend, 0 when the backend is absent.
+
+ ``getattr(torch, ...)`` can raise during backend module init on builds
+ compiled without the backend, so the whole probe is guarded.
+ """
+ try:
+ mod = getattr(torch, dev_type, None)
+ fn = getattr(mod, "device_count", None)
+ if callable(fn):
+ return int(fn())
+ except Exception: # pragma: no cover - backend-specific probe failure
+ pass
+ return 0
+
+
+def _accel_free_bytes_map(dev_type: str):
+ """Per-device free-memory map for the backend, or None when unavailable.
+
+ cuda/xpu expose ``mem_get_info``; other backends (and missing APIs) return
+ None so the plan resolves without VRAM filtering (explicit devices then
+ decide the fleet).
+ """
+ try:
+ mod = getattr(torch, dev_type, None)
+ if mod is not None and getattr(mod, "is_available", lambda: False)():
+ mem = getattr(mod, "mem_get_info", None)
+ if callable(mem):
+ return {torch.device(dev_type, i): mem(i)[0] for i in range(_accel_device_count(dev_type))}
+ except Exception as e: # pragma: no cover - backend-specific probe failure
+ logger.warning(
+ "[tune-ddp] free-memory probe failed for %s (%s); resolving the plan without VRAM filtering",
+ dev_type,
+ e,
+ )
+ return None
+
+
+@dataclass(frozen=True)
+class ParallelPolicy:
+ """Resolved ``--data_parallel`` policy, set once at the entry.
+
+ ``world`` is the requested DDP world (1 = off/serial); ``source`` records
+ where it came from ("off" | "auto" | "explicit"); ``collect_forward_cap``
+ mirrors AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES (0 = uncapped);
+ ``batch_mode`` carries ``--parallel_batch`` ("split" = the global batch
+ shards across replicas, serial-equivalent updates; "per_replica" = every
+ replica draws the full --batch_size from its own pool shard, torchrun-like
+ effective batch = batch_size x world). The policy is set on the
+ CompressContext once at the entry; the engine reads it from there.
+ """
+
+ world: int = 1
+ source: str = "off"
+ collect_forward_cap: int = 0
+ batch_mode: str = "split"
+
+ @property
+ def enabled(self) -> bool:
+ return self.world > 1
+
+
+def _quantizer_policy(quantizer) -> ParallelPolicy:
+ """The run's ParallelPolicy as visible to a quantizer (default: off)."""
+ ctx = getattr(quantizer, "compress_context", None)
+ policy = getattr(ctx, "parallel_policy", None)
+ return policy if isinstance(policy, ParallelPolicy) else ParallelPolicy()
+
+
+class ParallelTuneState:
+ """Engine-owned parallel state for one quantizer run.
+
+ The quantizer hosts it as ``parallel_state`` (a mount point only -- every
+ field is engine state): the cached engagement plan, the block-scoped
+ collection mirror pool handed over by the composer, and the flag that a
+ lane consumed that pool.
+ """
+
+ __slots__ = ("plan", "pool", "pool_used")
+
+ def __init__(self) -> None:
+ self.plan: DDPPlan | None = None
+ self.pool: MirrorPool | None = None
+ self.pool_used: bool = False
+
+
+def parallel_state(quantizer, create: bool = False) -> ParallelTuneState | None:
+ """The quantizer's engine state holder (created on demand)."""
+ st = getattr(quantizer, "parallel_state", None)
+ if st is None and create:
+ st = ParallelTuneState()
+ quantizer.parallel_state = st
+ return st
+
+
+def resolve_parallel_world(parallel: str, n_visible: int, iters: int = 0) -> int:
+ """Resolve the ``--data_parallel`` argument into a DDP world size.
+
+ Policy entry point (called by the CLI): ``off`` -> 1, ``auto`` -> all
+ visible devices, ``N`` -> N. The power-of-two rule applies only when the
+ tune loop runs (``iters > 0``): the gradient exchange is a recursive
+ halving-doubling reduction over the replicas and needs a power-of-two
+ rank count. At ``iters == 0`` the loop is empty, every phase that runs
+ (sharded collection, searches) is world-count-agnostic, so any device
+ count works and ``auto`` takes all visible devices. A requested world is
+ a requirement: an invalid value, or more devices requested than are
+ visible, raises instead of silently shrinking the request.
+ """
+ parallel = str(parallel or "off").strip().lower()
+ if parallel == "off":
+ return 1
+ if n_visible < 2:
+ raise RuntimeError("--data_parallel auto needs at least 2 visible CUDA devices")
+ if parallel == "auto":
+ if iters > 0:
+ return 1 << (max(n_visible, 1).bit_length() - 1) # largest power of two <= visible
+ return n_visible # no exchange at iters=0: every visible device works
+ if parallel.isdigit():
+ world = int(parallel)
+ if world < 2:
+ raise RuntimeError(f"--data_parallel world must be at least 2, got {world}")
+ if world > n_visible:
+ raise RuntimeError(
+ f"--data_parallel {world} requires at least {world} visible CUDA devices, got {n_visible}"
+ )
+ if world & (world - 1) and iters > 0:
+ raise RuntimeError(
+ f"--data_parallel world must be a power of two when iters > 0 (the gradient "
+ f"exchange is a halving-doubling reduction), got {world}"
+ )
+ return world
+ raise RuntimeError(f"--data_parallel accepts off|auto|N, got {parallel!r}")
+
+
+def log_zero_shot_parallel_noop_(policy) -> None:
+ """ONE INFO line when an explicit ``--data_parallel N`` lands on
+ the zero-shot RTN path.
+
+ Plain RTN without calibration (``need_calib == False``) runs
+ ``Orchestrator._quantize_zero_shot`` -- no calibration forward, so the
+ parallel engine never engages and the request is silently ignored. Emit
+ exactly one INFO line for an EXPLICIT request (``auto``/``off`` add no
+ user expectation and stay silent). No refusal: weights-only RTN has
+ nothing to shard.
+ """
+ if isinstance(policy, ParallelPolicy) and getattr(policy, "source", "") == "explicit" and policy.world > 1:
+ logger.info(
+ "[tune-ddp] --data_parallel %d has no effect on this run: "
+ "zero-shot RTN needs no calibration forward (weights only); quantizing serially",
+ policy.world,
+ )
+
+
+def check_parallel_device_map_compat_(world: int, device_map, iters: int) -> None:
+ """Stop at policy resolution (before model load) when a run cannot parallel-tune.
+
+ A multi-device ``--device_map`` with a live tune loop (``iters > 0``)
+ places blocks across several accelerator devices (pipeline-style
+ stages); the data-parallel lane mirrors whole blocks on single devices,
+ so without this check the run loads the model and fails only at block 0
+ in ``resolve_tune_ddp_plan_`` ("block spans N devices; parallel tuning
+ requires single-device block placement"). Detect the same condition
+ from the device map alone and stop with the same message, before the
+ model loads. The runtime per-block check stays as the second line of
+ defense (API users, and any map the resolver observes differently).
+ Device-map semantics are untouched: a later PP+DP design keeps
+ device_map as the per-group pipeline map.
+ """
+ if world < 2 or iters <= 0 or device_map is None:
+ return
+ from auto_round.utils.device import parse_available_devices
+
+ try:
+ devs = [d for d in parse_available_devices(device_map) if not str(d).startswith("cpu")]
+ except Exception: # pragma: no cover - unparsable map: the runtime check decides
+ return
+ if len(devs) > 1:
+ dev_type = str(devs[0]).split(":")[0]
+ dev_type = dev_type.upper() if ":" in str(devs[0]) else "CUDA"
+ raise RuntimeError(
+ f"block spans {len(devs)} {dev_type} devices; parallel tuning requires "
+ "single-device block placement (run with no --device_map or a single-device "
+ "--device_map; multi-device/auto maps may shard a block across GPUs -- or run "
+ "with --data_parallel off)"
+ )
+
+
+def resolve_ddp_plan(
+ world: int,
+ home: torch.device,
+ batch_size: int,
+ visible_devices: Sequence[int] | None = None,
+ vram_free_bytes: int | None = None,
+ mirror_footprint_bytes: int | None = None,
+ margin_bytes: int = 2 * 1024**3,
+ pow2_floor: bool = True,
+) -> DDPPlan:
+ """Pick mirror devices for ``world``-way data parallelism of one block.
+
+ Rules (each adjustment is recorded in ``notes``):
+ - world <= 1 or unsupported home -> disabled
+ - batch smaller than the world -> demote to the largest world that fits
+ the batch: with ``pow2_floor`` (the iters>0 contract) the largest power
+ of two that fits; with it off (iters=0, no gradient exchange) the exact
+ batch size; a batch of 1 runs serial (logged; a tiny batch is a
+ legitimate configuration)
+ - home + next devices in ascending visible order
+ - visible devices fewer than the requested world -> reported (the
+ engagement layer raises)
+ - per-mirror VRAM check: advisory only -- a WARNING names devices whose
+ free memory is below the estimated mirror footprint; the requested
+ world stays intact (a wrong estimate failing the run loudly is
+ preferable to silently shrinking it)
+ """
+ notes: list[str] = []
+ if world is None or world <= 1:
+ return DDPPlan(1, [home], batch_size, notes)
+ if home.type not in _SUPPORTED_ACCEL_TYPES:
+ notes.append(f"home device {home} is not a supported accelerator")
+ return DDPPlan(1, [home], batch_size, notes)
+ world = int(world)
+ if batch_size < world:
+ # sum-reduced losses make uneven shards exact, so the batch may
+ # split unevenly -- with a live tune loop the exchange world itself
+ # must stay a power of two (pow2_floor); at iters=0 there is no
+ # exchange and the demotion is exact. A replica needs at least one
+ # sample; below two usable replicas the lane runs serial (logged; a
+ # tiny batch is a legitimate configuration)
+ if pow2_floor:
+ _w = world
+ while _w > 1 and batch_size < _w:
+ _w //= 2
+ else:
+ _w = min(world, max(batch_size, 1))
+ if _w < 2:
+ logger.info("[tune-ddp] batch %d smaller than any usable world; running serial", batch_size)
+ return DDPPlan(1, [home], batch_size, notes)
+ if _w < world:
+ notes.append(f"batch {batch_size} smaller than world {world}; world set to {_w}")
+ logger.info("[tune-ddp] %s", notes[-1])
+ world = _w
+
+ if visible_devices:
+ order = [torch.device(home.type, i) for i in sorted(visible_devices)]
+ else:
+ order = [torch.device(home.type, i) for i in range(_accel_device_count(home.type))]
+
+ rotated = [home] + [d for d in order if d != home]
+ if len(rotated) < world:
+ # the requested world is a requirement: fewer visible devices than
+ # requested is reported to the caller (the engagement layer raises);
+ # the world stays as asked
+ notes.append(f"visible {home.type} devices ({len(rotated)}) fewer than the requested world {world}")
+ return DDPPlan(1, [home], batch_size, notes)
+ devices = rotated[:world]
+ if vram_free_bytes is not None and mirror_footprint_bytes is not None:
+ for dev in devices:
+ if dev == home:
+ continue # the home already holds the block; this check prices mirrors
+ free = vram_free_bytes.get(dev, 0) if hasattr(vram_free_bytes, "get") else vram_free_bytes
+ if free < mirror_footprint_bytes + margin_bytes:
+ notes.append(
+ f"low free VRAM on {dev}: free {free / 2**30:.1f}GiB < estimated mirror footprint "
+ f"{(mirror_footprint_bytes + margin_bytes) / 2**30:.1f}GiB (estimate only; device kept)"
+ )
+ logger.warning("[tune-ddp] %s", notes[-1])
+ return DDPPlan(world, devices, batch_size // world, notes)
+
+
+def _move_tensor(t: torch.Tensor, device: torch.device) -> torch.Tensor:
+ """Move a tensor to ``device`` (seam for tests on single-device hosts)."""
+ return t.to(device)
+
+
+def resolve_tune_ddp_plan_(quantizer, block, fp_inputs, fp_outputs, home, world=None, log: bool = True):
+ """Resolve the engaged DDP plan for one block -- shared by quantizer + composer.
+
+ Single source of truth for engagement so the collection pass (composer)
+ and the tune (quantizer) resolve identically: both check the same eligibility
+ (policy world, accelerator home, no LFQ, list pools, not
+ on the torchrun lane), price the same mirror
+ footprint, apply the same device selection and the same
+ power-of-two gate. The resolved plan is cached on the quantizer's parallel
+ state (engine-owned);
+ the tune-side caller re-resolves post-wrap (exact wrapper pricing, fresh
+ VRAM), so pass ``log=False`` from the pre-wrap collection call to keep the
+ engagement/decline lines one-per-block. ``fp_outputs`` may be None when the
+ reference outputs are not collected yet (composer entry) -- the tune-side
+ caller re-checks the outputs are a list before engaging.
+
+ Returns the DDPPlan (world=1 => not engaged).
+ """
+ from auto_round.utils.distributed import is_distributed
+
+ _st = parallel_state(quantizer)
+ cached = _st.plan if _st is not None else None
+ if cached is not None:
+ return cached
+ if world is None:
+ world = _quantizer_policy(quantizer).world
+ # the pow2 rule is iters-conditional: the exchange collectives need a
+ # power-of-two rank count, and they run only when the tune loop does
+ _iters = int(getattr(quantizer, "iters", 0) or 0)
+ home = torch.device(home) if not isinstance(home, torch.device) else home
+ if home.type in _SUPPORTED_ACCEL_TYPES and home.index is None:
+ try:
+ _mod = getattr(torch, home.type, None)
+ _cur = getattr(_mod, "current_device", None)
+ home = torch.device(home.type, int(_cur()) if callable(_cur) else 0)
+ except Exception: # backend module init can assert on builds without it
+ home = torch.device(home.type, 0)
+ decline = []
+ # NOTE: engagement itself is not iters-gated -- the DDP world shards
+ # the sharded no-grad collection passes at iters=0 exactly as at
+ # iters>0 (the tune loop has zero iterations there); only the pow2
+ # rule is iters-conditional (see _iters below)
+ if world > 1 and home.type not in _SUPPORTED_ACCEL_TYPES:
+ decline.append(f"home device {home} is not a supported accelerator ({'/'.join(_SUPPORTED_ACCEL_TYPES)})")
+ # eligibility constraints are DECLARED by the quantizer
+ # (BaseQuantizer.parallel_constraints); the resolver consumes the
+ # declaration
+ _constraints_fn = getattr(quantizer, "parallel_constraints", None)
+ _constraints = _constraints_fn() if callable(_constraints_fn) else {}
+ _lfq_active = bool(_constraints.get("enable_lfq"))
+ _home_span_devs = {p.device for p in block.parameters() if p.device.type == home.type}
+ eligible = (
+ world > 1
+ and home.type in _SUPPORTED_ACCEL_TYPES
+ and not _lfq_active
+ and isinstance(fp_inputs, list)
+ and (fp_outputs is None or isinstance(fp_outputs, list))
+ and not is_distributed()
+ and len(_home_span_devs) <= 1
+ )
+ if world > 1 and _lfq_active:
+ decline.append("enable_lfq")
+ if world > 1 and not isinstance(fp_inputs, list):
+ decline.append("non-list calibration inputs (diffusion-style pools)")
+ if world > 1 and isinstance(fp_inputs, list) and fp_outputs is not None and not isinstance(fp_outputs, list):
+ decline.append("non-list reference outputs (diffusion-style pools)")
+ # Parallel tuning tunes whole-block mirrors: every replica, including the
+ # home copy, must sit on ONE device for the tune. A block whose weights
+ # span several devices of the home's accelerator type (multi-device auto
+ # placement) is a different topology (pipeline-style stages), so fail
+ # visibly: the placement itself must be single-device when parallel
+ # tuning runs. CPU-resident tensors are legal alongside the
+ # home device (pinned subtrees such as ngram embedding tables stay on CPU
+ # by design).
+ if world > 1 and len(_home_span_devs) > 1:
+ decline.append(
+ f"block spans {len(_home_span_devs)} {home.type.upper()} devices; parallel tuning requires "
+ "single-device block placement (run with no --device_map or a single-device "
+ "--device_map; multi-device/auto maps may shard a block across GPUs -- or run "
+ "with --data_parallel off)"
+ )
+ if world > 1 and is_distributed():
+ decline.append("multi-process torchrun lane is active (--data_parallel is single-process)")
+
+ plan = DDPPlan(1, [home], len(fp_inputs) if isinstance(fp_inputs, list) else 0)
+ if eligible:
+ nsamples = len(fp_inputs)
+ batch_size = getattr(getattr(quantizer, "calibration_context", None), "batch_size", nsamples)
+ global_batch_size = min(nsamples, batch_size * getattr(quantizer, "gradient_accumulate_steps", 1))
+ mirror_bytes = sum(pp.numel() * pp.element_size() for pp in block.parameters()) + sum(
+ pp.numel() * pp.element_size()
+ for _m in block.modules()
+ if hasattr(_m, "orig_layer")
+ for pp in _m.params.values()
+ )
+ # charge the per-forward activation working set on top of the mirror:
+ # the existing estimator walks the block's real shapes at the
+ # per-forward batch (the serial micro-batch size -- replicas chunk
+ # their shards to it), so an activation-heavy block cannot silently
+ # pass a weights-only budget
+ try:
+ from auto_round.utils.device import estimate_tuning_block_mem
+
+ _per_fwd = min(int(batch_size), max(1, global_batch_size // max(1, world)))
+ (
+ _layer_mem,
+ _act_mem,
+ _io_mem,
+ _add_mem,
+ _pd_act,
+ _pd_exp,
+ ) = estimate_tuning_block_mem(block, fp_inputs, _per_fwd)
+ # ``_act_mem`` already sums the per-layer ``output_memory``
+ # (grad-doubled) with the MoE ratio applied -- the dict totals
+ # are already included. The two per-device maps
+ # (``_pd_act``/``_pd_exp``) split the same activation charge by
+ # weight home for multi-GPU placement planning; a mirror is
+ # single-device by contract, so the aggregate charge is the
+ # right price here.
+ _act_bytes = int((_act_mem + _io_mem + _add_mem) * 2**30)
+ mirror_bytes += _act_bytes
+ except Exception as e: # pragma: no cover - estimator is best-effort
+ # a failed estimate is a degraded state: the budget silently
+ # drops back to weights-only pricing (an activation-heavy block
+ # can then pass a plan it does not fit)
+ logger.warning("[tune-ddp] activation pricing skipped (estimator failed: %s)", e)
+ free = _accel_free_bytes_map(home.type)
+ _n_devs = _accel_device_count(home.type)
+ plan = resolve_ddp_plan(
+ world,
+ home,
+ global_batch_size,
+ visible_devices=list(range(_n_devs)) if _n_devs else None,
+ vram_free_bytes=free,
+ mirror_footprint_bytes=mirror_bytes,
+ pow2_floor=_iters > 0,
+ )
+ if plan.world > 1:
+ # F5: the final peak-VRAM summary only reports devices the
+ # memory monitor samples, and its callers pass the device-map
+ # list -- replicas engaged outside the map would stay invisible.
+ # Register the plan devices once the lane engages (idempotent,
+ # per block; best-effort: the monitor must never gate tuning)
+ try:
+ from auto_round.utils.device import memory_monitor
+
+ memory_monitor.register_extra_devices(plan.devices)
+ except Exception: # pragma: no cover - diagnostics must not gate tuning
+ pass
+ if log:
+ global _ENGAGED_LOGGED_SIG
+ sig = (plan.world, tuple(str(d) for d in plan.devices), plan.shard_size)
+ _changed = sig != _ENGAGED_LOGGED_SIG
+ else:
+ _changed = False
+ if plan.enabled and _iters > 0 and plan.world & (plan.world - 1) != 0:
+ decline.append(
+ f"resolved world {plan.world} is not a power of two (the gradient exchange needs "
+ "one when the tune loop runs)"
+ )
+ plan = DDPPlan(1, [home], nsamples)
+ elif plan.enabled and log and _changed:
+ _mode = _quantizer_policy(quantizer).batch_mode
+ if _mode == "per_replica":
+ # each replica draws the full requested batch (clamped to its
+ # pool shard): the effective global batch scales with the world
+ _per_draw = max(1, min(global_batch_size, nsamples // max(plan.world, 1)))
+ _eff_batch = _per_draw * plan.world
+ else:
+ _eff_batch = global_batch_size
+ logger.info(
+ "[tune-ddp] engaged: world=%d, batch_size per device=%d, batch=%s, "
+ "effective global batch=%d, devices=%s",
+ plan.world,
+ plan.shard_size,
+ _mode,
+ _eff_batch,
+ [str(d) for d in plan.devices],
+ )
+ _ENGAGED_LOGGED_SIG = sig
+ if plan.enabled and plan.world > 1:
+ # parallel lanes multiply the static shape specializations of the
+ # shared compiled quant functions (per-layer v shapes x shard /
+ # micro-batch chunk shapes across the replicas' searches), so the
+ # dynamo cache limit must scale with the engaged world or the lane
+ # silently degrades to eager mid-block once the limit is hit.
+ # Monotone raise-only, so the
+ # per-block resolve is idempotent and a serial run keeps the
+ # plain env default.
+ try:
+ from auto_round import envs as _cache_envs
+ from auto_round.utils.device import _bump_dynamo_cache_limit
+
+ _bump_dynamo_cache_limit(int(_cache_envs.AR_DYNAMO_CACHE_SIZE_LIMIT) * plan.world)
+ except Exception: # pragma: no cover - best effort; the tune continues
+ pass
+ if world > 1 and not plan.enabled:
+ reasons = decline + [n for n in plan.notes if n]
+ if reasons:
+ # A requested parallel world is a hard requirement: continuing
+ # on the serial path would silently invalidate the requested
+ # configuration and any measurement against it. `auto`
+ # resolves to the largest power of two fitting the visible devices
+ # at entry and the VRAM check is advisory, so reaching this raise
+ # means parallel tuning is genuinely ineligible for this run
+ # (an engagement gate or fewer visible devices than the world).
+ raise RuntimeError(
+ f"parallel tuning with world={world} is ineligible: "
+ f"{'; '.join(reasons)}. Fix the blocking condition listed above, or run "
+ "with --data_parallel off to tune serial deliberately."
+ )
+ parallel_state(quantizer, create=True).plan = plan
+ return plan
+
+
+def _relocate_params(module: torch.nn.Module, device: torch.device) -> None:
+ """Move ALL state that ``nn.Module.to()`` cannot see onto ``device``.
+
+ Wrapper modules keep three kinds of non-registered state that a mirrored
+ block would otherwise leave on the home device:
+ - tunable tensors in the plain ``params`` dict (``v``/``min_scale``/...)
+ -- recreated as fresh leaf Parameters (grad state resets, correct for a
+ fresh mirror);
+ - plain tensor attributes (``weight_min``/``weight_max`` anchors, cached
+ imatrix slices, ...) -- moved in place;
+ - device-typed attributes (``device``/``output_device``/``tuning_device``)
+ -- repointed so wrapper forward staging targets the mirror.
+ """
+ for _n, m in module.named_modules():
+ params = getattr(m, "params", None)
+ if isinstance(params, dict):
+ for key, val in params.items():
+ if isinstance(val, torch.nn.Parameter):
+ # move IN PLACE: the dict entry and the registered
+ # _parameters entry are the SAME object by construction
+ # (wrapper _init_params). Replacing the object here broke
+ # that aliasing -- optimizers collected the dict's dead
+ # clone while the forward/backward used the registered
+ # Parameter, so mirror grads stayed None, sync_grads
+ # silently skipped the allreduce, and the home stepped on
+ # its own shard's gradient only (quality regressed,
+ # monotonically in world size).
+ val.data = _move_tensor(val.detach(), device)
+ for key, val in list(m.__dict__.items()):
+ if isinstance(val, torch.Tensor) and val.device != device and val.device.type != "meta":
+ m.__dict__[key] = _move_tensor(val, device)
+ elif isinstance(val, torch.device):
+ m.__dict__[key] = device
+ elif (
+ key in ("tuning_device", "device", "output_device")
+ and isinstance(val, str)
+ and _parse_device_token(val).type == device.type
+ ):
+ # wrapper staging targets (plain strings -- invisible to the
+ # torch.device branch above); same accelerator family only
+ m.__dict__[key] = str(device)
+
+
+def _param_grad_buffers(params_by_device: list[list[torch.nn.Parameter]]) -> list[torch.Tensor | None]:
+ """Flatten each replica's gradients into one contiguous fp32 buffer."""
+ bufs: list[torch.Tensor | None] = []
+ for params in params_by_device:
+ parts = [p.grad.detach().reshape(-1) for p in params if p.grad is not None]
+ if not parts:
+ bufs.append(None)
+ continue
+ buf = torch.cat(parts) if len(parts) > 1 else parts[0].clone()
+ bufs.append(buf.to(torch.float32) if buf.dtype != torch.float32 else buf)
+ return bufs
+
+
+def _write_back_grads(buf: torch.Tensor | None, params: list[torch.nn.Parameter]) -> None:
+ """Scatter an averaged flat buffer back into ``param.grad`` tensors."""
+ if buf is None:
+ return
+ offset = 0
+ for p in params:
+ if p.grad is None:
+ continue
+ n = p.grad.numel()
+ p.grad.copy_(buf[offset : offset + n].view_as(p.grad))
+ offset += n
+
+
+def _transport_segment(seg: torch.Tensor, dev: torch.device) -> torch.Tensor:
+ """Move a peer's segment onto ``dev`` (fp32 wire, lossless).
+
+ The wire hop uses non_blocking=True: the allreduce cost is
+ payload-independent across dtypes (host-blocking-bound), and cross-device
+ copy_ is stream-ordered on both endpoints, so dropping the host block
+ lets the exchange chain execute back-to-back on the GPUs without races.
+ """
+ return seg.to(dev, non_blocking=True)
+
+
+def halving_doubling_allreduce(buffers: list[torch.Tensor], scale: float = 1.0) -> None:
+ """In-place all-reduce across device-resident buffers (single process).
+
+ Chunked recursive halving-doubling: the flat space is split into W
+ chunks; the reduce-scatter phase leaves rank r owning reduced chunk r
+ (each step exchanges half of the working block with the partner rank);
+ the all-gather phase re-exchanges owned chunks until every rank holds
+ the fully reduced buffer. Per-rank traffic is 2*(W-1)/W*bytes, same as a
+ ring. Cross-device ``to()`` copies use P2P when available. ``scale`` is
+ applied at the end (pass ``1/world`` to average). Requires a
+ power-of-two world (the resolver guarantees it).
+ """
+ world = len(buffers)
+ if world < 2:
+ if buffers:
+ buffers[0].mul_(scale)
+ return
+ if world & (world - 1):
+ raise ValueError(f"halving_doubling_allreduce needs a power-of-two world, got {world}")
+
+ numel = buffers[0].numel()
+ chunk = (numel + world - 1) // world
+
+ _reduce_scatter_halving(buffers)
+ _allgather_doubling(buffers)
+
+ for buf in buffers:
+ buf.mul_(scale)
+
+
+def _reduce_scatter_halving(buffers: list[torch.Tensor]) -> None:
+ """Recursive-halving reduce-scatter across device-resident buffers.
+
+ The flat space is split into W chunks; the working block per rank halves
+ each step (each rank keeps one half, adding the partner's copy of
+ it). Rank r ends up owning reduced chunk r; the other
+ chunks hold partial garbage. Accumulation stays fp32 on the receiver.
+ Requires a power-of-two world.
+ """
+ world = len(buffers)
+ numel = buffers[0].numel()
+ chunk = (numel + world - 1) // world
+ length = world # chunks in each rank's working block
+ while length > 1:
+ half = length // 2
+ for rank in range(world):
+ base = rank - (rank % length) # aligned working-block start
+ mid, hi = base + half, base + length
+ if rank % length < half:
+ partner = rank + half # keep [base, mid)
+ seg = buffers[partner][chunk * base : chunk * mid]
+ seg = _transport_segment(seg, buffers[rank].device)
+ buffers[rank][chunk * base : chunk * mid].add_(seg)
+ else:
+ partner = rank - half # keep [mid, hi)
+ seg = buffers[partner][chunk * mid : chunk * hi]
+ seg = _transport_segment(seg, buffers[rank].device)
+ buffers[rank][chunk * mid : chunk * hi].add_(seg)
+ length = half
+
+
+def _allgather_doubling(buffers: list[torch.Tensor]) -> None:
+ """Recursive-doubling all-gather of per-rank owned chunks.
+
+ Mirrors the reduce-scatter block structure: the working block per rank
+ doubles each step; ranks exchange the chunks the partner owns (copies,
+ no adds). Every rank ends holding every chunk -- bitwise identical
+ (same-device fp32 copies for gradient buffers, plain int8 copies for
+ the sign buffers).
+ """
+ world = len(buffers)
+ numel = buffers[0].numel()
+ chunk = (numel + world - 1) // world
+ length = 2
+ while length <= world:
+ half = length // 2
+ for rank in range(world):
+ base = rank - (rank % length)
+ mid, hi = base + half, base + length
+ if rank % length < half:
+ partner = rank + half
+ src = buffers[partner][chunk * mid : chunk * hi]
+ dst = buffers[rank][chunk * mid : chunk * hi]
+ dst.copy_(_transport_segment(src, dst.device))
+ else:
+ partner = rank - half
+ src = buffers[partner][chunk * base : chunk * mid]
+ dst = buffers[rank][chunk * base : chunk * mid]
+ dst.copy_(_transport_segment(src, dst.device))
+ length *= 2
+
+
+_ENGAGED_LOGGED_SIG = None # module-level init for the global read in resolve_tune_ddp_plan_
+
+
+# Calibration statistics folded from the mirror copies into the home after a
+# sharded collection pass. The engine owns act_max (the composer's act_max
+# hooks); a block quantizer declares what its own hooks write in
+# ``BaseQuantizer.calibration_stats``, and a pass merges the union (see
+# ``calibration_merge_stats``). Associative reductions reproduce the serial
+# totals up to fp32 summation order.
+ENGINE_CALIBRATION_STATS = MappingProxyType({"act_max": "max"}) # element-wise running max
+STAT_REDUCTIONS = ("sum", "max")
+
+
+def calibration_merge_stats(quantizer, quantizer_hooks) -> dict[str, str] | None:
+ """Shard-merge map for a calibration pass carrying ``quantizer_hooks``.
+
+ The engine's act_max plus the quantizer's declared ``calibration_stats``.
+ ``None`` when the quantizer registered hooks without a declaration: the
+ engine cannot fold statistics it does not know, so the pass must stay
+ serial (a sharded pass would keep only the home shard's samples).
+ """
+ declared = getattr(quantizer, "calibration_stats", None)
+ if quantizer_hooks and declared is None:
+ return None
+ stats = dict(ENGINE_CALIBRATION_STATS)
+ for attr, how in (declared or {}).items():
+ if how not in STAT_REDUCTIONS:
+ raise ValueError(
+ f"{type(quantizer).__name__}.calibration_stats[{attr!r}] = {how!r}: "
+ f"supported shard reductions are {STAT_REDUCTIONS}"
+ )
+ stats[attr] = how
+ return stats
+
+
+def _merge_mirror_stats(home: torch.nn.Module, mirrors: list[torch.nn.Module], stats) -> None:
+ """Fold hook-written statistics from mirror copies back into the home.
+
+ Each mirror forwarded a disjoint sample shard; its hooks accumulated
+ into mirror-local module attrs which die with the mirror. ``stats`` maps
+ each attribute to its reduction ("sum" | "max"); both are associative, so
+ the merged home totals equal the serial pass's totals up to fp32
+ summation order (~1e-7 relative) -- far below any quantization-relevant
+ scale. Python-scalar statistics (a row count) fold with the same
+ semantics, so they always co-travel with their tensor sibling.
+
+ Each mirror value is folded exactly once and then removed from the
+ mirror: pool mirrors live across all collection passes of a block, so a
+ later pass's merge would otherwise add the same shard values to the home
+ again (a hook pass followed by any other sharded pass doubled the
+ mirrors' share).
+ """
+ home_mods = dict(home.named_modules())
+ for mirror in mirrors:
+ if mirror is None:
+ continue
+ for name, m_mod in mirror.named_modules():
+ h_mod = home_mods.get(name)
+ if h_mod is None:
+ continue
+ for attr, how in stats.items():
+ m_val = getattr(m_mod, attr, None)
+ if m_val is None:
+ continue
+ h_val = getattr(h_mod, attr, None)
+ if not torch.is_tensor(m_val):
+ # scalar stats (imatrix_cnt is a python int): fold with the
+ # same copy-or-sum semantics so they always co-travel with
+ # their tensor sibling
+ if not isinstance(m_val, (int, float)) or isinstance(m_val, bool):
+ continue
+ if h_val is None:
+ setattr(h_mod, attr, m_val)
+ else:
+ setattr(h_mod, attr, h_val + m_val)
+ elif h_val is None:
+ setattr(h_mod, attr, m_val.to(h_mod.weight.device if hasattr(h_mod, "weight") else m_val.device))
+ else:
+ m_val = m_val.to(h_val.device, h_val.dtype)
+ if how == "sum":
+ setattr(h_mod, attr, h_val + m_val)
+ else: # max
+ setattr(h_mod, attr, torch.max(h_val, m_val))
+ delattr(m_mod, attr) # folded: a later pass must not add it again
+
+
+_MODULE_INTERNALS = frozenset(vars(torch.nn.Module()))
+
+
+def _stat_signature(val):
+ """Change signature of a module attribute a hook may accumulate into:
+ tensors by identity + in-place version, containers holding tensors by
+ identity + length; other kinds (config scalars, strings, bookkeeping
+ such as hook-handle lists) are not tracked."""
+ if torch.is_tensor(val):
+ return (id(val), val._version)
+ if isinstance(val, (list, dict)):
+ items = val.values() if isinstance(val, dict) else val
+ if any(torch.is_tensor(v) for v in items):
+ return (id(val), len(val))
+ return None
+
+
+def _mirror_attr_snapshot(mirrors) -> list[dict]:
+ return [
+ {
+ (name, attr): _stat_signature(val)
+ for name, mod in mirror.named_modules()
+ for attr, val in vars(mod).items()
+ if attr not in _MODULE_INTERNALS
+ }
+ for mirror in mirrors
+ ]
+
+
+def _warn_undeclared_mirror_writes(mirrors, snapshots, merge_stats) -> None:
+ """Safety net for the ``calibration_stats`` contract.
+
+ A tensor/container attribute that changed on a mirror module during a
+ sharded pass but is not in the merge map dies with the mirror, so the
+ home keeps only its own shard's values. Warn once per attribute name.
+ """
+ for mirror, snap in zip(mirrors, snapshots):
+ for name, mod in mirror.named_modules():
+ for attr, val in vars(mod).items():
+ if attr in _MODULE_INTERNALS or attr in merge_stats:
+ continue
+ sig = _stat_signature(val)
+ if sig is not None and snap.get((name, attr)) != sig:
+ logger.warning_once(
+ "[tune-ddp] '%s' changed on mirror modules during a sharded collection pass but is not a "
+ "declared calibration statistic: if a calibration hook accumulates it, the mirrors' share "
+ "is dropped (the home keeps only its own shard). Declare its reduction in the block "
+ "quantizer's calibration_stats, or run with --data_parallel off.",
+ attr,
+ )
+
+
+def distribute_pool(pool: list[torch.Tensor], devices: list[torch.device]) -> None:
+ """Scatter a per-sample calibration pool across ``devices`` (in place).
+
+ Device r owns its contiguous ceil/floor slice -- the same boundaries the
+ DDP tune shards and the sharded collection use (contiguous_shard_bounds;
+ a remainder lands on an earlier replica) -- so shard-local reads
+ (ref_r build, _select_batch for shard r, per-replica collections) never
+ cross devices. Pieces already on their target device are untouched
+ (idempotent; a pool produced by the previous block's sharded collection
+ on the same group costs nothing). Pools smaller than the world are left
+ alone (serial consumers handle them via move-on-demand cats).
+ """
+ n = len(pool)
+ world = len(devices)
+ if world < 2 or n < world:
+ return
+ bounds = contiguous_shard_bounds(n, world)
+ for r, dev in enumerate(devices):
+ dev = torch.device(dev)
+ for i in range(bounds[r], bounds[r + 1]):
+ if pool[i].device != dev:
+ pool[i] = pool[i].to(dev)
+
+
+def contiguous_shard_bounds(n: int, world: int) -> list[int]:
+ """Contiguous ceil/floor split bounds over ``n`` items for ``world`` shards.
+
+ The first ``n % world`` shards take one extra item (remainder on the
+ EARLIER shards). This exact idiom is a cross-component invariant: pool
+ placement, collection shards, tune shards and warm-up windows must all
+ slice identically or pools and shards silently misalign -- every site
+ must call this helper for the bounds.
+ """
+ sizes = [n // world + (1 if r < n % world else 0) for r in range(world)]
+ bounds = [0]
+ for sz in sizes:
+ bounds.append(bounds[-1] + sz)
+ return bounds
+
+
+class MirrorPool:
+ """Persistent-per-block ephemeral mirrors, reused across collection phases.
+
+ Builds the same per-device replicas ``sharded_nograd_forward`` builds
+ per call (home device keeps the original block; every other device gets a
+ relocated deepcopy), but ONCE per block: the collection forwards reuse
+ the pool; after a zero-shot RTN/OptRTN quantize the mirrors are synced to
+ the quantized home block so the cascade reuses them; the tune ranks tune
+ the mirrors (spmd). ``release()`` drops the references; the home block
+ stays canonical throughout.
+ """
+
+ def __init__(self, block, devices):
+ import copy as _copy
+
+ home_dev = _block_device(block)
+ self.block = block
+ self.devices = list(devices)
+ self.reps: list[torch.nn.Module] = []
+ self._owned: list[torch.nn.Module] = [] # deepcopies this pool must free
+ for dev in self.devices:
+ if dev == home_dev:
+ self.reps.append(block)
+ else:
+ m = _copy.deepcopy(block).to(dev)
+ _relocate_params(m, dev)
+ self.reps.append(m)
+ self._owned.append(m)
+
+ @property
+ def world(self) -> int:
+ return len(self.reps)
+
+ def mirror_layer(self, rep_idx: int, name: str) -> torch.nn.Module:
+ for n, m in self.reps[rep_idx].named_modules():
+ if n == name:
+ return m
+ raise KeyError(name)
+
+ def sync_hooks_from(self, block) -> None:
+ """Mirror the home block's CURRENT forward hooks onto every replica.
+
+ A fresh-deepcopy pass inherits the hook registry at copy time; a
+ persistent pool must re-sync instead, or it would carry hooks removed
+ from the home block (stale stats writes) and miss hooks registered
+ after the pool was built (missing shard statistics). Hook callables
+ are shared by identity -- the same behavior deepcopy provides.
+ """
+ home_hooks = {}
+ for n, m in block.named_modules():
+ fwd = getattr(m, "_forward_hooks", None)
+ pre = getattr(m, "_forward_pre_hooks", None)
+ if fwd or pre:
+ fwd_wk = getattr(m, "_forward_hooks_with_kwargs", None)
+ pre_wk = getattr(m, "_forward_pre_hooks_with_kwargs", None)
+ home_hooks[n] = (dict(fwd or {}), dict(pre or {}), dict(fwd_wk or {}), dict(pre_wk or {}))
+ for rep in self.reps:
+ if rep is block:
+ continue
+ for _n, m in rep.named_modules():
+ m._forward_hooks.clear()
+ m._forward_pre_hooks.clear()
+ if hasattr(m, "_forward_hooks_with_kwargs"):
+ m._forward_hooks_with_kwargs.clear()
+ if hasattr(m, "_forward_pre_hooks_with_kwargs"):
+ m._forward_pre_hooks_with_kwargs.clear()
+ for n, (fwd, pre, fwd_wk, pre_wk) in home_hooks.items():
+ try:
+ m = rep.get_submodule(n)
+ except AttributeError:
+ continue
+ for k, h in fwd.items():
+ m.register_forward_hook(h, with_kwargs=bool(fwd_wk.get(k, False)))
+ for k, h in pre.items():
+ m.register_forward_pre_hook(h, with_kwargs=bool(pre_wk.get(k, False)))
+
+ def release(self) -> None:
+ self._owned.clear()
+ self.reps = []
+
+
+def sharded_nograd_forward(
+ runner,
+ block,
+ inputs,
+ input_others: dict,
+ out_device: torch.device,
+ devices: list[torch.device],
+ sample_count: int | None = None,
+ merge_stats: Mapping[str, str] | None = None,
+ max_devices: int = 0,
+ pool: MirrorPool | None = None,
+):
+ """Parallelize a no-grad collection forward across ``devices``.
+
+ ``merge_stats`` (attr -> "sum" | "max") folds the hook-written statistics
+ from the mirrors into the home after the pass; ``None`` merges nothing.
+
+ ``max_devices > 0`` truncates the shard set to the first K devices
+ (shards grow accordingly): forward hooks force dynamo graph breaks,
+ leaving the compiled runner as python-bound eager sections that
+ GIL-convoy beyond a handful of threads -- hook-carrying passes are
+ capped by the caller while hookless passes shard wide.
+
+ The collection passes (reference outputs, quantized-output cascade) are
+ plain forwards over the whole sample pool on one GPU while the mirrors
+ idle. Here the pool is split into equal disjoint shards; an ephemeral
+ copy of the block on each device forwards its shard in a parallel thread;
+ per-shard outputs are parked on ``out_device`` and concatenated in order.
+ Bit-identical to the serial pass: rows are sample-independent and the
+ module copies carry identical weights. Falls back to the serial runner
+ call when the pool is not divisible or fewer than two devices are given.
+ Mirrors are dropped (freed) afterwards. Returned pieces stay on the
+ device that computed them (distributed pool); consumers either read
+ shard-locally or use device-safe cats.
+ """
+ world = len(devices)
+ n = sample_count if sample_count is not None else (len(inputs) if isinstance(inputs, list) else 0)
+ if max_devices and 0 < max_devices < world:
+ logger.info(
+ "[tune-ddp] sharded collect: capping concurrent shards at %d of %d device(s) (hook pass)",
+ max_devices,
+ world,
+ )
+ devices = list(devices)[:max_devices]
+ world = max_devices
+ # ceil/floor contiguous split (same tolerance the search seam applies):
+ # any sample count shards -- a remainder lands on an earlier replica and
+ # the wall is the largest slice; nothing is dropped or run serially
+ world = max(1, min(world, n))
+ if len(devices) < 2 or world < 2:
+ return runner(block, inputs, input_others, cache_device=out_device)
+ devices = list(devices)[:world]
+ bounds = contiguous_shard_bounds(n, world)
+ shards = [list(range(bounds[r], bounds[r + 1])) for r in range(world)]
+ # the input pool typically arrives pooled on the primary device; without
+ # re-placement every shard pulls its pieces cross-device from that ONE
+ # source, which dominates the shard-forward time. Distribute first
+ # (idempotent, same layout the tune engagement
+ # enforces later); serial consumers afterwards re-gather to home.
+ # Tensor pools only: the helper's contract also admits opaque pools.
+ if (
+ isinstance(inputs, list)
+ and inputs
+ and all(isinstance(t, torch.Tensor) for t in inputs)
+ and (sample_count is None or sample_count == len(inputs))
+ ):
+ distribute_pool(inputs, devices)
+ import time as _time
+
+ _t_mirrors = _time.perf_counter()
+ mirrors: list[torch.nn.Module] = [] # the actual deepcopies used this pass
+ if pool is not None and pool.world >= len(devices):
+ # capped passes REUSE the full-width pool: take the first K replicas
+ # and leave the rest idle (a fresh per-pass deepcopy of the block,
+ # plus a second full-width rebuild once the pool was withheld, would
+ # cost far more). The unused replicas idle;
+ # their stats stay stale for this pass and the home fold below only
+ # merges the mirrors that actually ran.
+ pool.sync_hooks_from(block)
+ reps = pool.reps[: len(devices)]
+ mirrors = [m for m in reps if m is not block]
+ pool_owned = True
+ else:
+ home_dev = _block_device(block)
+ reps: list[torch.nn.Module] = []
+ for dev in devices:
+ if dev == home_dev:
+ reps.append(block)
+ else:
+ m = copy.deepcopy(block).to(dev)
+ _relocate_params(m, dev)
+ reps.append(m)
+ mirrors.append(m)
+ pool_owned = False
+ _t_setup_ms = (_time.perf_counter() - _t_mirrors) * 1000
+ parts: list = [None] * world
+ fwd_walls = [0.0] * world # per-thread stores at distinct indices: race-free
+ evs: list = [None] * world
+
+ def _run(r):
+ rep = reps[r]
+ dev_r = _block_device(rep)
+ t_r = _time.perf_counter()
+ ev = None
+ if dev_r.type == "cuda":
+ ev = (torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True))
+ ev[0].record()
+ # outputs stay ON the replica that computed them: the per-sample list
+ # returned below is a distributed pool (device r owns shard r's rows)
+ if dev_r.type == "cuda":
+ with torch.cuda.device(dev_r):
+ parts[r] = runner(rep, inputs, input_others, shards[r], cache_device=dev_r)
+ else:
+ parts[r] = runner(rep, inputs, input_others, shards[r], cache_device=dev_r)
+ if ev is not None:
+ ev[1].record()
+ evs[r] = ev
+ fwd_walls[r] = (_time.perf_counter() - t_r) * 1000
+
+ # spawn path: collection passes run twice per block, so the per-call
+ # thread spawn cost is irrelevant here
+ _attr_snaps = _mirror_attr_snapshot(mirrors) if merge_stats is not None else None
+ run_threaded_spawn([lambda r=r: _run(r) for r in range(world)])
+ # per-shard GPU times: sync the participating devices (their outputs are
+ # consumed right after anyway -- the first downstream read syncs too)
+ fwd_gpu = [0.0] * world
+ for r, dev in enumerate(devices):
+ if evs[r] is not None:
+ try:
+ torch.cuda.synchronize(dev)
+ fwd_gpu[r] = evs[r][0].elapsed_time(evs[r][1])
+ except RuntimeError as e: # a broken pair stays non-fatal to the pass
+ logger.warning_once("[perf] sharded-collect GPU wall unavailable for a pair (%s)", e)
+ fwd_gpu[r] = float("nan")
+ _t_split = _time.perf_counter()
+ # Return the SERIAL structure: a per-sample list ([1, S, H] pieces from
+ # split_outputs) -- downstream consumers cat per-sample refs along dim 0
+ # (tune loss) and iterate the list for the
+ # cascade inputs, so a tensor here changes every per-sample shape.
+ pieces: list[torch.Tensor] = []
+ split_outputs = getattr(runner, "split_outputs", None)
+ for part in parts:
+ for piece in split_outputs(part) if split_outputs else torch.split(part, 1, dim=0):
+ # clone: torch.split/split_outputs return VIEWS into the batched
+ # shard base -- a view-holding pool pins the whole [n, S, H] base
+ # on its device for as long as any piece lives, and that base is
+ # large at tuning batch sizes. Standalone pieces free the base at
+ # pass end.
+ pieces.append(piece.clone())
+ # threaded shard calls each stamped a PARTIAL last_output_dict (their own
+ # shard's rows); the serial text path leaves it unset so callers fall back
+ # to the returned list -- clear the residue to keep that contract.
+ runner.last_output_dict = None
+ _t_merge = _time.perf_counter()
+ if merge_stats is not None:
+ _merge_mirror_stats(block, mirrors, merge_stats)
+ _warn_undeclared_mirror_writes(mirrors, _attr_snaps, merge_stats)
+ if not pool_owned:
+ mirrors.clear() # drop mirror refs; the caching allocator reclaims them
+ reps.clear() # pool-owned reps stay with the pool (later passes reuse them)
+ # steady-state breakdown for the collection passes: setup = ephemeral
+ # mirror deepcopy/relocate (per pass!), fwd = threaded shard forwards
+ # (wall includes enqueue, gpu is the device-side chain incl. the
+ # per-thread output parking), split/merge = host assembly. Logged for
+ # EVERY pass so first-pass warm-up is separable from steady state.
+ _walls = sorted(fwd_walls)
+ _gpus = sorted(fwd_gpu)
+ logger.debug(
+ "[tune-ddp] sharded collect breakdown: world=%d n=%d setup=%.0fms "
+ "fwd_wall[min/med/max]=%.0f/%.0f/%.0fms fwd_gpu[min/med/max]=%.0f/%.0f/%.0fms "
+ "merge=%.0fms",
+ world,
+ n,
+ _t_setup_ms,
+ _walls[0],
+ _walls[len(_walls) // 2],
+ _walls[-1],
+ _gpus[0],
+ _gpus[len(_gpus) // 2],
+ _gpus[-1],
+ (_time.perf_counter() - _t_merge) * 1000,
+ )
+ return pieces
+
+
+def run_threaded_spawn(fns: Sequence) -> None:
+ """Run one callable per item in parallel SPAWNED threads (legacy path).
+
+ Exceptions propagate: the first failure (by thread index) is re-raised
+ in the joining thread -- a swallowed worker failure would otherwise
+ leave missing shard losses/grads and corrupt the step silently.
+ """
+ errors: dict = {}
+
+ def _guarded(idx, fn):
+ try:
+ fn()
+ except BaseException as exc: # noqa: BLE001 - re-raised below
+ errors[idx] = exc
+
+ threads = [threading.Thread(target=_guarded, args=(i, fn)) for i, fn in enumerate(fns)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ if errors:
+ raise errors[min(errors)]
+
+
+def _block_device(block) -> torch.device:
+ p = next(block.parameters(), None)
+ return p.device if p is not None else torch.device("cpu")
diff --git a/auto_round/algorithms/parallel/mirror_sync.py b/auto_round/algorithms/parallel/mirror_sync.py
new file mode 100644
index 000000000..3db729f9d
--- /dev/null
+++ b/auto_round/algorithms/parallel/mirror_sync.py
@@ -0,0 +1,154 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Resident mirror-pool synchronization for the data-parallel engine.
+
+The collection passes leave a per-block MirrorPool (one resident block copy
+per replica device). This module keeps those mirrors consistent with the HOME
+block at phase boundaries: device barriers, home search statistics pushed to
+the mirrors before the tune ranks wrap them, and the quantized home state pushed to
+the mirrors after a zero-shot RTN/OptRTN quantize so the Step-6 cascade can
+still run sharded on them."""
+
+import torch
+
+from auto_round.algorithms.parallel.data_parallel import parallel_state
+from auto_round.logger import logger
+from auto_round.utils import check_to_quantized
+
+
+def sync_mirrors_to_home(quantizer, block) -> bool:
+ """Make the block's resident mirrors exactly-home after a home-only quantize.
+
+ Zero-shot RTN/OptRTN quantize the HOME block through main's own search
+ lane (which already spreads the searches over idle devices). The
+ collection mirrors still serve the sharded Step-6 cascade, so push the
+ quantized home state (weight + result attrs) onto every replica, release
+ the home search stats and mark the pool consumed. Returns False (nothing
+ done) when no pool aligned with this block and plan is resident.
+ """
+ from auto_round.algorithms.parallel.data_parallel import MirrorPool
+
+ _st = parallel_state(quantizer)
+ plan = getattr(_st, "plan", None) if _st is not None else None
+ pool = getattr(_st, "pool", None) if _st is not None else None
+ if not (
+ plan is not None
+ and isinstance(pool, MirrorPool)
+ and pool.world == plan.world
+ and pool.block is block
+ and list(pool.devices) == list(plan.devices)
+ ):
+ return False
+ targets = [(n, m) for n, m in block.named_modules() if hasattr(m, "global_name") and check_to_quantized(m)]
+ _t_bar = _sync_pool_devices(pool.devices)
+ _sync_quantized_to_mirrors(pool, block, targets)
+ for _n, m in targets:
+ for _attr in ("imatrix", "imatrix_cnt"):
+ if hasattr(m, _attr):
+ delattr(m, _attr)
+ _st.pool_used = True
+ _t_bar += _sync_pool_devices(pool.devices)
+ logger.debug("[mirror-sync] quantized home state synced to %d mirrors (bar=%.2fs)", pool.world - 1, _t_bar)
+ return True
+
+
+_POOL_SEARCH_STATS = ("imatrix", "imatrix_cnt", "act_max", "weight_global_scale")
+
+
+def _sync_pool_devices(devices) -> float:
+ """Synchronize every distinct CUDA pool device; returns elapsed seconds.
+
+ Called at phase boundaries: cross-device copies issued in one
+ phase (stats distribution, result placement, mirror sync) complete before
+ the next phase touches the same tensors, and a CUDA fault surfaces at
+ the boundary of the phase that enqueued it instead of at an unrelated
+ later call.
+ """
+ import time as _ptime
+
+ _t0 = _ptime.perf_counter()
+ for d in dict.fromkeys(devices):
+ if d.type == "cuda":
+ torch.cuda.synchronize(d)
+ return _ptime.perf_counter() - _t0
+
+
+def _distribute_search_stats(pool, block, targets):
+ """Copy the home layers' collected stats onto every mirror.
+
+ The pool mirrors collected their own sample-shard statistics during the
+ collection passes; the merged totals live on the home layers (folded by
+ ``_merge_mirror_stats``). Tune ranks wrapping the mirrors must see the FULL
+ stats, so each mirror layer is overwritten with the home totals first.
+ """
+ for name, m in targets:
+ stats = {a: getattr(m, a) for a in _POOL_SEARCH_STATS if hasattr(m, a)}
+ if not stats:
+ continue
+ for r in range(pool.world):
+ rep = pool.reps[r]
+ if rep is block:
+ continue
+ lm = pool.mirror_layer(r, name)
+ dev = next(rep.parameters(), None)
+ dev = dev.device if dev is not None else torch.device("cpu")
+ for a, v in stats.items():
+ setattr(lm, a, v.to(dev) if isinstance(v, torch.Tensor) else v)
+
+
+_POOL_RESULT_ATTRS = ("scale", "zp", "q_scale_thresh", "data_type")
+
+
+def _sync_quantized_to_mirrors(pool, block, targets):
+ """Propagate every target's quantized weight + result attrs home -> mirrors.
+
+ The quantize ran on the HOME block only, so without this sync each mirror
+ would still hold the FP block -- and the cascade shards SAMPLES across
+ replicas. One weight copy per (target, non-home replica) makes every
+ mirror carry the exact home state (the cascade contract). Weight-only iters=0
+ contract: activation-quantization WrapperWALayer targets are outside this
+ lane (structural mirror/home parity is not guaranteed for them).
+ """
+ # per-MIRROR threads: each replica's copies are independent, so a serial
+ # loop pays the sum of all mirrors' copies while threading collapses the
+ # wall to the slowest single mirror
+ from auto_round.algorithms.parallel.data_parallel import run_threaded_spawn
+
+ def _sync_mirror(r):
+ rep = pool.reps[r]
+ if rep is block:
+ return
+ _p0 = next(rep.parameters(), None)
+ dev = _p0.device if _p0 is not None else torch.device("cpu")
+
+ def _copy_targets():
+ for name, _m in targets:
+ home_layer = block.get_submodule(name)
+ w = home_layer.weight.data
+ attrs = {a: getattr(home_layer, a) for a in _POOL_RESULT_ATTRS if hasattr(home_layer, a)}
+ lm = pool.mirror_layer(r, name)
+ lm.weight.data.copy_(w.to(dev, non_blocking=True))
+ for a, v in attrs.items():
+ setattr(lm, a, v.to(dev) if isinstance(v, torch.Tensor) else v)
+
+ if dev.type == "cuda":
+ with torch.cuda.device(dev):
+ try:
+ _copy_targets()
+ finally:
+ torch.cuda.synchronize(dev)
+ else:
+ _copy_targets()
+
+ run_threaded_spawn([lambda r=r: _sync_mirror(r) for r in range(pool.world)])
diff --git a/auto_round/algorithms/parallel/spmd.py b/auto_round/algorithms/parallel/spmd.py
new file mode 100644
index 000000000..926e6a040
--- /dev/null
+++ b/auto_round/algorithms/parallel/spmd.py
@@ -0,0 +1,432 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Thread ranks: a DDP-style wrapper for the per-block tuning loop.
+
+The engine runs the algorithm's UNCHANGED ``quantize_block`` once per rank,
+one thread per device, like ``torch.nn.parallel.DistributedDataParallel``
+runs an unchanged training loop once per process:
+
+- module: each rank tunes its own replica of the block (rank 0 = the home
+ block, the others = resident mirrors);
+- dataloader: ``make_index_sampler`` hands each rank a sampler over its
+ contiguous shard of the (device-distributed) calibration pool;
+- gradients: ``sync_gradients()`` from ``setup_ddp_if_needed_`` averages
+ the gradients across the ranks (DDP semantics);
+- loss: ``all_reduce_loss`` gives every rank the same global loss, so
+ best-iteration selection and early stopping agree on every rank.
+
+The algorithm uses only those three hooks, the same ones the torchrun path
+provides. Rank 0's result is the block result; the mirrors are discarded.
+
+The first block of each shape runs each rank's first phase (wrap, optimizer
+build and first forward/backward, up to its first collective) serially in
+rank order: first-time ``torch.compile`` / triton autotuning is not
+thread-safe. Later blocks of the same shape on the same devices find their
+kernels compiled and run the ranks concurrently from the start.
+"""
+
+from __future__ import annotations
+
+import copy
+import dataclasses
+import logging
+import random
+import threading
+from collections.abc import Callable
+from typing import Any
+
+import torch
+
+from auto_round.algorithms.parallel.data_parallel import (
+ MirrorPool,
+ _param_grad_buffers,
+ _quantizer_policy,
+ _write_back_grads,
+ contiguous_shard_bounds,
+ distribute_pool,
+ halving_doubling_allreduce,
+ parallel_state,
+ resolve_tune_ddp_plan_,
+)
+from auto_round.algorithms.parallel.mirror_sync import (
+ _POOL_SEARCH_STATS,
+ _distribute_search_stats,
+ _sync_pool_devices,
+)
+from auto_round.logger import logger
+
+_tls = threading.local()
+# (block shape, devices) signatures whose rank kernels are compiled already
+_WARM_SIGNATURES: set = set()
+_WARM_LOCK = threading.Lock()
+
+
+def block_signature(block: torch.nn.Module, devices) -> tuple:
+ """What decides whether a block's rank kernels are compiled already."""
+ shapes = tuple((n, tuple(p.shape), p.dtype) for n, p in block.named_parameters())
+ return (type(block).__name__, shapes, tuple(str(d) for d in devices))
+
+
+class RankAborted(RuntimeError):
+ """Raised in a rank whose peer failed (the group was aborted)."""
+
+
+class RankGroup:
+ """In-process collective group for one block's thread ranks."""
+
+ def __init__(self, devices: list[torch.device], serial_first_phase: bool = True) -> None:
+ self.devices = [torch.device(d) for d in devices]
+ self.world = len(self.devices)
+ self._barrier = threading.Barrier(self.world)
+ self._slots: list[Any] = [None] * self.world
+ self._result: Any = None
+ self._turn = threading.Condition()
+ # without the serial first phase every rank starts at once
+ self.concurrent = not serial_first_phase
+ self._next_turn = 0 if serial_first_phase else self.world
+ self._released = [False] * self.world
+ self._broken = False
+ # one shuffle seed per rank, drawn in rank order from the run's RNG
+ self.seeds = [random.getrandbits(63) for _ in range(self.world)]
+
+ # ── first-phase turnstile ────────────────────────────────────────────
+ def wait_turn(self, rank: int) -> None:
+ with self._turn:
+ self._turn.wait_for(lambda: self._broken or self._next_turn >= rank)
+ if self._broken:
+ raise RankAborted("rank group aborted before this rank started")
+
+ def release_turn(self, rank: int) -> None:
+ with self._turn:
+ if not self._released[rank]:
+ self._released[rank] = True
+ self._next_turn = max(self._next_turn, rank + 1)
+ self._turn.notify_all()
+
+ def abort(self) -> None:
+ with self._turn:
+ self._broken = True
+ self._turn.notify_all()
+ self._barrier.abort()
+
+ # ── collectives ──────────────────────────────────────────────────────
+ def _wait(self) -> None:
+ try:
+ self._barrier.wait()
+ except threading.BrokenBarrierError as e:
+ raise RankAborted("a peer rank failed") from e
+
+ def exchange(self, rank: int, value: Any, reduce_fn: Callable[[list], Any]) -> Any:
+ """Post ``value``; rank 0 reduces every rank's value; all get the result."""
+ self.release_turn(rank)
+ self._slots[rank] = value
+ self._wait()
+ if rank == 0:
+ try:
+ self._result = reduce_fn(list(self._slots))
+ except BaseException:
+ self.abort()
+ raise
+ self._wait()
+ return self._result
+
+ def all_reduce_grads(self, rank: int, module: torch.nn.Module) -> None:
+ """Average the gradients of ``module``'s parameters across the ranks."""
+ params = [p for p in module.parameters() if p.grad is not None]
+
+ def _reduce(per_rank):
+ counts = {len(ps) for ps in per_rank}
+ if len(counts) != 1:
+ raise RuntimeError(f"ranks hold different gradient sets: {[len(ps) for ps in per_rank]}")
+ bufs = _param_grad_buffers(per_rank)
+ if any(b is None for b in bufs):
+ return
+ halving_doubling_allreduce(bufs, scale=1.0 / self.world)
+ for buf, ps in zip(bufs, per_rank):
+ _write_back_grads(buf, ps)
+ return
+
+ self.exchange(rank, params, _reduce)
+
+ def all_reduce_mean(self, rank: int, value: float) -> float:
+ """Mean of a python scalar across the ranks (identical on every rank)."""
+ return self.exchange(rank, float(value), lambda vals: sum(vals) / len(vals))
+
+ def all_reduce_weighted_mean(self, rank: int, value: float, weight: float) -> float:
+ """sum(value * weight) / sum(weight) across the ranks (identical on every rank)."""
+
+ def _reduce(pairs):
+ total = sum(w for _v, w in pairs)
+ return sum(v * w for v, w in pairs) / total if total else sum(v for v, _w in pairs) / len(pairs)
+
+ return self.exchange(rank, (float(value), float(weight)), _reduce)
+
+
+class RankContext:
+ """What one rank thread knows about itself."""
+
+ def __init__(self, group: RankGroup, rank: int) -> None:
+ self.group = group
+ self.rank = rank
+
+ @property
+ def world(self) -> int:
+ return self.group.world
+
+ @property
+ def device(self) -> torch.device:
+ return self.group.devices[self.rank]
+
+ def index_sampler(self, nsamples: int, batch_size: int):
+ """Sampler over this rank's contiguous shard of the calibration pool."""
+ from auto_round.compressors.utils import IndexSampler
+
+ bounds = contiguous_shard_bounds(nsamples, self.world)
+ lo, hi = bounds[self.rank], bounds[self.rank + 1]
+ return IndexSampler(
+ hi - lo,
+ min(batch_size, hi - lo),
+ indices=list(range(lo, hi)),
+ rng=random.Random(self.group.seeds[self.rank]),
+ )
+
+ def sync_gradients(self, module: torch.nn.Module) -> None:
+ self.group.all_reduce_grads(self.rank, module)
+
+ def all_reduce_mean(self, value: float) -> float:
+ return self.group.all_reduce_mean(self.rank, value)
+
+ def all_reduce_weighted_mean(self, value: float, weight: float) -> float:
+ return self.group.all_reduce_weighted_mean(self.rank, value, weight)
+
+
+def run_rank_sharded_wrap_searches(rank: RankContext, wrappers: list) -> bool:
+ """Shard a block's deferred wrap-time searches across the ranks.
+
+ Each rank searches the wrappers it owns (index % world) on its own
+ replica, the searched init scales are exchanged, and every rank
+ finalizes all its wrappers with them. Returns False (the caller runs
+ every search locally) when the ranks are in their serial first phase:
+ a collective there would release the compile turnstile early.
+ """
+ if not rank.group.concurrent:
+ return False
+ from auto_round.algorithms.quantization import search_dispatch
+
+ world, me = rank.world, rank.rank
+ own = [w for i, w in enumerate(wrappers) if i % world == me]
+ summary = search_dispatch.SearchSummary()
+ if own and not search_dispatch.run_batched_wrap_search(own, summary=summary):
+ for w in own:
+ with summary.timed(rank.device) as st:
+ w._run_deferred_search_now()
+ st["singletons"] += 1
+ st["modules"] += 1
+ summary.finish()
+ mine = {i: w.init_scale for i, w in enumerate(wrappers) if i % world == me}
+
+ def _merge(per_rank):
+ merged = {}
+ for part, _summary in per_rank:
+ merged.update(part)
+ return merged, [s for _part, s in per_rank]
+
+ # the block's summary line: every rank's share, logged once
+ scales, summaries = rank.group.exchange(me, (mine, summary), _merge)
+ if me == 0:
+ search_dispatch.SearchSummary.merged(summaries).log("[batched-search] wrap search:")
+ for i, w in enumerate(wrappers):
+ if i % world != me:
+ val = scales[i]
+ w.finalize_batched_search(val.to(rank.device) if torch.is_tensor(val) else val)
+ return True
+
+
+def current_rank() -> RankContext | None:
+ """The calling thread's rank, or None outside a thread-rank run."""
+ return getattr(_tls, "ctx", None)
+
+
+class _NonZeroRankFilter(logging.Filter):
+ """Rank threads other than rank 0 log warnings and errors only."""
+
+ def filter(self, record: logging.LogRecord) -> bool:
+ ctx = current_rank()
+ return ctx is None or ctx.rank == 0 or record.levelno >= logging.WARNING
+
+
+logger.addFilter(_NonZeroRankFilter())
+
+
+def rank_quantizer(quantizer, world: int, split_batch: bool):
+ """Shallow per-rank copy of the quantizer.
+
+ The copy shares the model and contexts (read-only during the tune) and
+ gets a private calibration context whose batch size is the rank's share
+ of the global batch (``split``) or the full batch (``per_replica``).
+ The loss is staged on the rank's own device.
+ """
+ rq = copy.copy(quantizer)
+ run_ctx = quantizer._BaseAlgorithm__run_ctx
+ cal = copy.copy(run_ctx.calibration_context)
+ if split_batch:
+ cal.batch_size = max(1, int(cal.batch_size) // world)
+ rq._BaseAlgorithm__run_ctx = dataclasses.replace(run_ctx, calibration_context=cal)
+ rq._loss_device = None
+ rq._card_0_in_high_risk = False
+ return rq
+
+
+def _decline(reason: str) -> None:
+ logger.info("[tune-ranks] declining: %s; serial path", reason)
+
+
+def quantize_block_on_ranks(
+ quantizer, block, fp_inputs, input_others, fp_outputs, q_inputs, block_ctx, input_ids=None
+) -> tuple[bool, Any]:
+ """Run ``quantizer.quantize_block`` once per data-parallel rank.
+
+ Returns ``(engaged, rank-0 result)``; ``engaged=False`` means the caller
+ runs the serial call. Engagement needs a tuning quantizer (iters > 0),
+ an engaged DDP plan, list pools whose size the world divides, and a
+ per-rank batch that fits its shard.
+ """
+ from auto_round.utils.device_manager import device_manager
+
+ if int(getattr(quantizer, "iters", 0) or 0) <= 0:
+ return False, None
+ enable_q = q_inputs is not None and getattr(quantizer, "enable_quanted_input", True)
+ active_inputs = q_inputs if enable_q else fp_inputs
+ if not isinstance(active_inputs, list) or not isinstance(fp_outputs, list):
+ return False, None
+
+ _st = parallel_state(quantizer, create=True)
+ # an engaged plan cached by this block's collection pass is reused:
+ # re-resolving would price a second set of mirrors next to the resident
+ # pool and warn about VRAM the pool itself occupies. A declined or
+ # missing plan resolves here, where its reasons are logged.
+ plan = _st.plan
+ if plan is None or not plan.enabled:
+ _st.plan = None
+ plan = resolve_tune_ddp_plan_(quantizer, block, active_inputs, fp_outputs, device_manager.device)
+ _st.plan = None # block-scoped: the next block resolves its own
+ if not plan.enabled:
+ return False, None
+ world = plan.world
+ nsamples = len(active_inputs)
+ policy = _quantizer_policy(quantizer)
+ split = str(getattr(policy, "batch_mode", "split") or "split") != "per_replica"
+ batch_size = int(quantizer.calibration_context.batch_size)
+ acc = int(getattr(quantizer, "gradient_accumulate_steps", 1) or 1)
+ if nsamples % world:
+ _decline(f"{nsamples} samples do not split evenly over {world} ranks")
+ return False, None
+ if split and batch_size % world:
+ _decline(f"batch size {batch_size} does not split evenly over {world} ranks")
+ return False, None
+ local_batch = (batch_size // world if split else batch_size) * acc
+ if local_batch > nsamples // world:
+ _decline(f"per-rank batch {local_batch} exceeds the per-rank shard {nsamples // world}")
+ return False, None
+
+ devices = list(plan.devices)
+ # data: each device owns its contiguous shard; the rank samplers read
+ # only their own shard (make_index_sampler)
+ distribute_pool(active_inputs, devices)
+ distribute_pool(fp_outputs, devices)
+
+ # module: rank 0 tunes the home block, the others tune mirrors (the
+ # collection pool's resident mirrors when aligned, else fresh copies)
+ pool = getattr(_st, "pool", None)
+ if not (isinstance(pool, MirrorPool) and pool.block is block and list(pool.devices) == devices):
+ pool = MirrorPool(block, devices)
+ else:
+ targets = [(n, m) for n, m in block.named_modules() if any(hasattr(m, a) for a in _POOL_SEARCH_STATS)]
+ if targets:
+ _distribute_search_stats(pool, block, targets)
+ pool.sync_hooks_from(block)
+ _sync_pool_devices(devices)
+ # the mirrors end the block in their own tuned state: nothing reuses them
+ _st.pool = None
+
+ signature = block_signature(block, devices)
+ with _WARM_LOCK:
+ warm = signature in _WARM_SIGNATURES
+ group = RankGroup(devices, serial_first_phase=not warm)
+ rank_qs = [rank_quantizer(quantizer, world, split) for _ in range(world)]
+ logger.info(
+ "[tune-ranks] engaged: world=%d, batch per rank=%d (%s), first phase %s, devices=%s",
+ world,
+ rank_qs[0].calibration_context.batch_size,
+ "split" if split else "per_replica",
+ "concurrent" if warm else "serial (compile)",
+ [str(d) for d in devices],
+ )
+
+ def _rank_fn(r):
+ return lambda: rank_qs[r].quantize_block(
+ pool.reps[r], fp_inputs, input_others, fp_outputs, q_inputs, block_ctx, input_ids=input_ids
+ )
+
+ try:
+ results = run_ranks([_rank_fn(r) for r in range(world)], devices, group)
+ with _WARM_LOCK:
+ _WARM_SIGNATURES.add(signature)
+ finally:
+ pool.release()
+ _st.plan = None
+ return True, results[0]
+
+
+def run_ranks(rank_fns: list[Callable[[], Any]], devices: list[torch.device], group: RankGroup) -> list[Any]:
+ """Run one callable per rank; rank 0 runs in the calling thread.
+
+ Every rank sees its own device as ``device_manager.device`` and as the
+ current CUDA device. A failure in any rank aborts the group (peers raise
+ RankAborted at their next collective) and the first real error is
+ re-raised here.
+ """
+ from auto_round.algorithms.parallel.tune_parallel import _device_scope
+ from auto_round.utils.device_manager import device_manager
+
+ world = len(rank_fns)
+ results: list[Any] = [None] * world
+ errors: list[BaseException | None] = [None] * world
+
+ def _run(r: int) -> None:
+ _tls.ctx = RankContext(group, r)
+ try:
+ with device_manager.thread_device_scope(devices[r]), _device_scope(devices[r]):
+ group.wait_turn(r)
+ results[r] = rank_fns[r]()
+ except BaseException as e: # noqa: BLE001 - re-raised by the caller below
+ errors[r] = e
+ group.abort()
+ finally:
+ group.release_turn(r)
+ _tls.ctx = None
+
+ threads = [threading.Thread(target=_run, args=(r,), name=f"ar-rank-{r}", daemon=True) for r in range(1, world)]
+ for t in threads:
+ t.start()
+ _run(0)
+ for t in threads:
+ t.join()
+ real = [e for e in errors if e is not None and not isinstance(e, RankAborted)]
+ if real:
+ raise real[0]
+ first = next((e for e in errors if e is not None), None)
+ if first is not None:
+ raise first
+ return results
diff --git a/auto_round/algorithms/parallel/tune_parallel.py b/auto_round/algorithms/parallel/tune_parallel.py
new file mode 100644
index 000000000..d4dd983df
--- /dev/null
+++ b/auto_round/algorithms/parallel/tune_parallel.py
@@ -0,0 +1,213 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Composer-owned context for the sharded calibration (collection) forwards.
+
+``TuneParallelContext.for_collection`` resolves the block's devices; the
+no-grad collection forwards then shard across the block's mirror pool
+(:func:`data_parallel.sharded_nograd_forward`). ``devices is None`` keeps the
+serial single-GPU collection. The tune itself runs on thread ranks
+(:mod:`spmd`).
+"""
+
+import time as _ptime
+
+import torch
+
+from auto_round.algorithms.parallel.data_parallel import (
+ ENGINE_CALIBRATION_STATS,
+ _quantizer_policy,
+ distribute_pool,
+ resolve_tune_ddp_plan_,
+ sharded_nograd_forward,
+)
+from auto_round.logger import logger
+
+
+class _null_scope:
+ """``with``-scope that is a no-op (CPU devices cannot enter torch.cuda.device)."""
+
+ def __enter__(self):
+ return None
+
+ def __exit__(self, *exc):
+ return False
+
+
+def _device_scope(dev: torch.device):
+ """``torch.cuda.device(dev)`` for cuda devices, no-op otherwise."""
+ if dev.type == "cuda":
+ return torch.cuda.device(dev)
+ return _null_scope()
+
+
+class TuneParallelContext:
+ """Collection-phase parallel context; ``devices is None`` means serial."""
+
+ def __init__(self) -> None:
+ self.devices: list[torch.device] | None = None
+ # collect-forward concurrency cap (from the run's ParallelPolicy;
+ # 0 = uncapped) -- see collect_forward(hook_pass=True)
+ self.collect_forward_cap: int = 0
+ # per-block mirror pool (None until the first sharded collection
+ # pass; reused by the collection forwards, the iters=0 cascade and
+ # the tune ranks)
+ self.pool = None
+
+ @classmethod
+ def for_collection(cls, composer, block, fp_inputs) -> "TuneParallelContext":
+ """Mirror devices for sharding the no-grad collection forwards.
+
+ Uses the SAME engagement resolver as the tune
+ (:func:`data_parallel.resolve_tune_ddp_plan_`, cached on the quantizer)
+ so the collection gates stay identical to what the tune later engages
+ -- plus two collection-specific gates: the runner must use the
+ plain tensor output layout, and the block class must not be a
+ multi-output registry entry (sharding clears ``last_output_dict``,
+ which those blocks need to feed the next block). ``devices is None``
+ keeps the serial single-GPU collection.
+ """
+ ctx = cls()
+ if not isinstance(fp_inputs, list) or not fp_inputs:
+ return ctx
+ runner_cfg = getattr(composer.block_forward, "output_config", None)
+ if runner_cfg and list(runner_cfg) != ["hidden_states"]:
+ return ctx # multi-output layouts need the serial last_output_dict path
+ from auto_round.algorithms.block_runner import _BLOCK_OUTPUT_REGISTRY
+
+ if len(_BLOCK_OUTPUT_REGISTRY.get(type(block).__name__, ["hidden_states"])) != 1:
+ return ctx # registry multi-output blocks (e.g. GlmMoeDsa) need serial collection
+ try:
+ home = next(block.parameters(), torch.empty(0)).device
+ except Exception as e: # pragma: no cover - defensive
+ logger.warning("[tune-ddp] collection sharding declined: cannot resolve the block home device (%s)", e)
+ return ctx
+ try:
+ plan = resolve_tune_ddp_plan_(composer.block_quantizer, block, fp_inputs, None, home, log=False)
+ except RuntimeError:
+ # the resolver raises when a requested world is ineligible
+ # (requirement semantics) -- let it propagate: silently continuing
+ # with serial collection would invalidate the request
+ raise
+ except Exception as e: # pragma: no cover - resolver reachability on odd hosts
+ logger.warning("[tune-ddp] collection sharding declined: resolver unreachable (%s)", e)
+ return ctx
+ ctx.devices = plan.devices if plan.world > 1 else None
+ ctx.collect_forward_cap = _quantizer_policy(composer.block_quantizer).collect_forward_cap
+ return ctx
+
+ def distribute_pools(self, fp_inputs, q_inputs) -> None:
+ """Distribute the calibration pool so each DDP device owns its sample
+ shard (matching the tune shards); shard-local reads stay device-local;
+ serial consumers use device-safe cats."""
+ if not self.devices:
+ return
+ if isinstance(fp_inputs, list) and fp_inputs:
+ distribute_pool(fp_inputs, self.devices)
+ if isinstance(q_inputs, list) and q_inputs:
+ distribute_pool(q_inputs, self.devices)
+
+ def collect_forward(
+ self,
+ block_forward,
+ block,
+ inputs,
+ input_others,
+ out_dev=None,
+ allow_shard: bool = True,
+ hook_pass: bool = False,
+ merge_stats=ENGINE_CALIBRATION_STATS,
+ ):
+ """Collection forward: sharded across DDP mirrors when eligible.
+
+ ``out_dev`` optionally overrides the runner's output cache device.
+ ``allow_shard=False`` forces the serial path.
+
+ ``merge_stats`` maps the statistics the pass's hooks write to their
+ shard reduction (``calibration_merge_stats``): hook writes land on the
+ mirror copies and are folded back into the home after the pass.
+ ``None`` means the block quantizer's hooks write statistics it did
+ not declare: the pass stays serial, since sharding would silently
+ keep only the home shard's samples.
+
+ ``hook_pass=True`` optionally caps the concurrent shards via the
+ policy's ``collect_forward_cap`` (from AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES,
+ default 0 = no cap): forward
+ hooks force dynamo graph breaks, leaving the compiled runner as
+ python-bound eager sections that can GIL-convoy under many threads.
+ """
+ if self.devices is None or not allow_shard:
+ return block_forward(block, inputs, input_others, cache_device=out_dev)
+ if merge_stats is None:
+ logger.warning_once(
+ "[tune-ddp] the block quantizer registers calibration hooks without declaring calibration_stats; "
+ "those calibration passes run serially (declare each statistic's shard reduction to shard them)"
+ )
+ return block_forward(block, inputs, input_others, cache_device=out_dev)
+ # The pool is ALWAYS built at the full device width and reused by
+ # every sharding-eligible pass; a hook pass capped below the world
+ # only limits WHICH pool replicas run -- sharded_nograd_forward slices
+ # the replica subset in place.
+ world = len(self.devices)
+ cap = int(self.collect_forward_cap or 0)
+ n = len(inputs) if isinstance(inputs, (list, tuple)) else 0
+ pool = None
+ if world >= 2 and isinstance(block, torch.nn.Module):
+ # ceil/floor split: any n >= world shards (a remainder lands on
+ # an earlier replica); n < world runs below the pool's width and
+ # stays serial
+ if n >= world:
+ if self.pool is None:
+ from auto_round.algorithms.parallel.data_parallel import MirrorPool
+
+ _t0 = _ptime.perf_counter()
+ self.pool = MirrorPool(block, self.devices)
+ logger.debug(
+ "[tune-ddp] pool build (first sharded pass): world=%d build=%.2fs "
+ "(per-device deepcopy + relocate)",
+ world,
+ _ptime.perf_counter() - _t0,
+ )
+ pool = self.pool
+ return sharded_nograd_forward(
+ block_forward,
+ block,
+ inputs,
+ input_others,
+ out_dev,
+ self.devices,
+ merge_stats=merge_stats,
+ max_devices=cap if hook_pass else 0,
+ pool=pool,
+ )
+
+ def ensure_pool(self, block) -> None:
+ """Build the per-block mirror pool if absent (block-wide, not tied to
+ any single pass's forward eligibility): the searches and the tune
+ consume it even when a collection pass fell back serial."""
+ if self.pool is None and self.devices and len(self.devices) >= 2:
+ from auto_round.algorithms.parallel.data_parallel import MirrorPool
+
+ _t0 = _ptime.perf_counter()
+ self.pool = MirrorPool(block, self.devices)
+ logger.debug(
+ "[tune-ddp] pool build: world=%d build=%.2fs (per-device deepcopy + relocate)",
+ len(self.devices),
+ _ptime.perf_counter() - _t0,
+ )
+
+ def release_pool(self) -> None:
+ """Drop the per-block mirror pool (clears the ctx-side references only)."""
+ if self.pool is not None:
+ self.pool.release()
+ self.pool = None
diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.py
index 7832cf2e5..de51df13f 100644
--- a/auto_round/algorithms/quantization/base.py
+++ b/auto_round/algorithms/quantization/base.py
@@ -44,8 +44,18 @@ class BaseQuantizer(BaseAlgorithm):
- :meth:`quantize_block` – **must override**: quantize a single block
- :meth:`quantize_layer_outside_block` – quantize layers outside blocks
- :meth:`finalize_run` – model-level teardown (once after all blocks)
+ A quantizer whose calibration hooks write statistics on modules declares
+ them in :attr:`calibration_stats`.
"""
+ #: Statistics this quantizer's calibration hooks write on modules, mapped
+ #: to their shard reduction ("sum" | "max"). Under --data_parallel
+ #: the hook-carrying calibration forwards shard across the block mirrors
+ #: and the engine folds these attributes back into the home block.
+ #: ``None`` = undeclared: if the quantizer registers calibration hooks,
+ #: those passes stay serial. A quantizer without hooks needs no declaration.
+ calibration_stats: dict[str, str] | None = None
+
def __init__(self, config: QuantizationConfig) -> None:
super().__init__(config)
# Whether to feed quantized-block outputs as inputs to the next block.
@@ -142,6 +152,17 @@ class BaseQuantizer(BaseAlgorithm):
return is_quantized
# ── Abstract quantization interface ───────────────────────────────────────
+ def parallel_constraints(self) -> dict:
+ """Declare eligibility facts for the single-process data-parallel lane.
+
+ The parallel engine consumes the declaration; algorithm internals stay
+ quantizer internals: ``enable_lfq`` reports LFQ layers whose handling is
+ single-device. A gradient scaler needs no declaration: every rank keeps
+ its own, and the gradients are averaged before ``scaler.step``, so the
+ ranks see the same inf/nan checks and keep the same scale.
+ """
+ return {"enable_lfq": bool(getattr(self, "enable_lfq", False))}
+
def quantize_block(
self,
block: "torch.nn.Module",
@@ -328,8 +349,14 @@ class BaseQuantizer(BaseAlgorithm):
Bulk pulls are a capacity optimization only: when the user explicitly
parked memory on the host (``low_gpu_mem_usage`` or
``calibration_data_device="cpu"``) the pool stays where that policy
- put it, however much room the target device has.
+ put it, however much room the target device has. Under data-parallel
+ thread ranks the engine already placed each rank's shard on its device,
+ so the pool stays as it is.
"""
+ from auto_round.algorithms.parallel.spmd import current_rank
+
+ if current_rank() is not None:
+ return pool
cc = self.compress_context
if (
getattr(cc, "low_gpu_mem_usage", False)
diff --git a/auto_round/algorithms/quantization/rtn/batched_search.py b/auto_round/algorithms/quantization/rtn/batched_search.py
index 666d3b40c..250423052 100644
--- a/auto_round/algorithms/quantization/rtn/batched_search.py
+++ b/auto_round/algorithms/quantization/rtn/batched_search.py
@@ -32,6 +32,7 @@ from typing import Any
import torch
from auto_round.algorithms.quantization.search_dispatch import (
+ SearchSummary,
_fn_key,
_plan_capacity_chunks,
_search_fn_stackable,
@@ -43,7 +44,6 @@ from auto_round.algorithms.quantization.search_dispatch import (
)
from auto_round.logger import logger
from auto_round.utils import set_module
-from auto_round.utils.perf_counters import perf_counters_enabled
from auto_round.wrapper import WrapperLinear
@@ -313,8 +313,9 @@ def run_batched_rtn_search(
for worker, _home, chunk in planned:
buckets.setdefault(str(worker), []).append(chunk)
- # per-worker execution stats for the wrap-grammar summary line
- stats: OrderedDict[str, dict] = OrderedDict()
+ # per-worker calls and search time for the summary line
+ summary = SearchSummary()
+ summary.offloaded = sum(1 for worker, home, _chunk in planned if worker != home)
def _worker_of(chunk: list[dict[str, Any]]) -> str:
for wk, cs in buckets.items():
@@ -346,7 +347,7 @@ def run_batched_rtn_search(
@torch.no_grad()
def _run_chunk(chunk: list[dict[str, Any]], worker: str, threaded: bool = False) -> None:
- st = stats.setdefault(str(worker), {"wall": 0.0, "modules": 0, "batches": 0, "singletons": 0})
+ st = summary.device(worker)
st["modules"] += len(chunk)
if len(chunk) > 1:
st["batches"] += 1
@@ -450,16 +451,9 @@ def run_batched_rtn_search(
set_module(model, e["name"], layer)
def _run_chunks(wk: str, cs: list[list[dict[str, Any]]]) -> None:
- import time as _ptime
-
- _t0 = _ptime.perf_counter()
- try:
+ with summary.timed(wk):
for c in cs:
_run_chunk_guarded(c, _worker_of(c), threaded=True)
- finally:
- stats.setdefault(str(wk), {"wall": 0.0, "modules": 0, "batches": 0, "singletons": 0})["wall"] += (
- _ptime.perf_counter() - _t0
- )
if len(buckets) > 1:
keyed = OrderedDict()
@@ -470,31 +464,8 @@ def run_batched_rtn_search(
for wk, cs in buckets.items():
_run_chunks(str(wk), cs)
- if stats and perf_counters_enabled():
- # one timing line, same grammar and gating as the wrap-search lane:
- # calls are chunks (each chunk is one stacked or per-module search
- # call), so the counts cannot read as "one call per module"
- from auto_round.utils.pool_placement import _short_device_key
-
- _n_mod = sum(st["modules"] for st in stats.values())
- _n_batch = sum(st["batches"] for st in stats.values())
- _n_single = sum(st["singletons"] for st in stats.values())
- _n_offloaded = sum(1 for worker, home, _chunk in planned if worker != home)
- _walls = sum(st.get("wall", 0.0) for st in stats.values())
- per_device = (
- "{" + ", ".join(f"'{_short_device_key(dev)}': {st.get('wall', 0.0):.2f}" for dev, st in stats.items()) + "}"
- )
- logger.info(
- "[rtn-batch] %d modules, %d batch + %d singleton search calls"
- + (f" ({_n_offloaded} chunk(s) offloaded to idle devices)" if _n_offloaded else "")
- + ", %.2fs device-wall (%.2fs summed) walls %s",
- _n_mod,
- _n_batch,
- _n_single,
- max((st.get("wall", 0.0) for st in stats.values()), default=0.0),
- _walls,
- per_device,
- )
+ if summary.devices:
+ summary.finish().log("[rtn-batch]")
def _relocate_tensor_kwargs(kwargs: dict[str, Any], device: str) -> dict[str, Any]:
diff --git a/auto_round/algorithms/quantization/rtn/quantizer.py b/auto_round/algorithms/quantization/rtn/quantizer.py
index 6eaf3e3d9..2e0e1275e 100644
--- a/auto_round/algorithms/quantization/rtn/quantizer.py
+++ b/auto_round/algorithms/quantization/rtn/quantizer.py
@@ -83,6 +83,11 @@ class RTNQuantizer(BaseQuantizer):
@register_pipeline_member(OptimizedRTNConfig)
class OptimizedRTNQuantizer(RTNQuantizer):
+ # imatrix_cnt (a python int row count) must co-travel with imatrix: a cold
+ # expert can collect zero routed rows on the home shard and receive only
+ # the merged tensor, and the normalize step (imatrix /= imatrix_cnt) needs both
+ calibration_stats = {"imatrix": "sum", "imatrix_cnt": "sum"}
+
def __init__(self, config: RTNConfig) -> None:
super().__init__(config)
diff --git a/auto_round/algorithms/quantization/search_dispatch.py b/auto_round/algorithms/quantization/search_dispatch.py
index 3aa9c2013..26727a1fd 100644
--- a/auto_round/algorithms/quantization/search_dispatch.py
+++ b/auto_round/algorithms/quantization/search_dispatch.py
@@ -184,6 +184,12 @@ def pick_search_worker_devices(
"""
if cross_device_search_disabled():
return [home_device] if home_device is not None else []
+ from auto_round.algorithms.parallel.spmd import current_rank
+
+ if current_rank() is not None:
+ # a data-parallel rank owns only its device: the "idle" ones are the
+ # other ranks' devices, busy with their own searches and tune steps
+ return [home_device] if home_device is not None else []
home_type = str(home_device).split(":")[0] if home_device else ""
if home_type == "hpu" and is_hpu_lazy_mode():
return [home_device] if home_device is not None else []
@@ -223,6 +229,70 @@ def log_engaged_once(label: str) -> None:
logger.info("[batched-search] %s: running weight-local searches with one worker per device", label)
+class SearchSummary:
+ """Calls and per-device search time of one search phase, logged as one line.
+
+ ``walls`` sums the time each device spent searching -- stacked and
+ per-module calls alike; ``device-wall`` is the elapsed time of the phase.
+ Summaries of concurrent phases (data-parallel ranks) merge into one line.
+ """
+
+ def __init__(self) -> None:
+ self._t0 = time.perf_counter()
+ self.elapsed = 0.0
+ self.offloaded = 0
+ self.devices: OrderedDict[str, dict] = OrderedDict()
+
+ def device(self, key) -> dict:
+ return self.devices.setdefault(str(key), {"wall": 0.0, "modules": 0, "batches": 0, "singletons": 0})
+
+ @contextlib.contextmanager
+ def timed(self, key) -> Iterator[dict]:
+ """Charge the enclosed search time to device ``key``; yields its counters."""
+ st = self.device(key)
+ t0 = time.perf_counter()
+ try:
+ yield st
+ finally:
+ st["wall"] += time.perf_counter() - t0
+
+ def finish(self) -> "SearchSummary":
+ self.elapsed = time.perf_counter() - self._t0
+ return self
+
+ @classmethod
+ def merged(cls, parts: "list[SearchSummary]") -> "SearchSummary":
+ """One summary for concurrent parts: device counters add up, the elapsed time is the slowest part's."""
+ out = cls()
+ out.elapsed = max((p.elapsed for p in parts), default=0.0)
+ for p in parts:
+ out.offloaded += p.offloaded
+ for key, st in p.devices.items():
+ dst = out.device(key)
+ for k, v in st.items():
+ dst[k] += v
+ return out
+
+ def log(self, label: str) -> None:
+ if not perf_counters_enabled():
+ return
+ from auto_round.utils.pool_placement import _short_device_key
+
+ sts = list(self.devices.values())
+ logger.info(
+ "%s %d modules, %d batch + %d singleton search calls"
+ + (f" ({self.offloaded} chunk(s) offloaded to idle devices)" if self.offloaded else "")
+ + ", %.2fs device-wall (%.2fs summed) walls %s",
+ label,
+ sum(st["modules"] for st in sts),
+ sum(st["batches"] for st in sts),
+ sum(st["singletons"] for st in sts),
+ self.elapsed,
+ sum(st["wall"] for st in sts),
+ "{" + ", ".join(f"'{_short_device_key(k)}': {st['wall']:.2f}" for k, st in self.devices.items()) + "}",
+ )
+
+
def batched_search_disabled() -> bool:
"""Kill switch for the batched search machinery (stacking + per-device workers)."""
return bool(envs.AR_DISABLE_BATCHED_SEARCH)
@@ -433,7 +503,9 @@ def _plan_capacity_chunks(entries: list) -> list:
return plans
-def run_batched_wrap_search(deferred_wrappers: list[WrapperLinear], max_batch: int | None = None) -> bool:
+def run_batched_wrap_search(
+ deferred_wrappers: list[WrapperLinear], max_batch: int | None = None, summary: "SearchSummary | None" = None
+) -> bool:
"""Run deferred weight-local wrap searches on stacked same-shape batches.
Wrappers stage ``(weight_reshape, data_type, bits, imatrix, q_scale_thresh,
@@ -456,11 +528,16 @@ def run_batched_wrap_search(deferred_wrappers: list[WrapperLinear], max_batch: i
Returns True when the inputs were consumed; False when batching is disabled
by AR_DISABLE_BATCHED_SEARCH (the caller then runs the searches per module).
+ A caller-supplied ``summary`` collects the calls and times instead of the
+ summary line (a data-parallel rank searches a share of the block).
"""
if batched_search_disabled():
return False
if not deferred_wrappers:
return False
+ _log_summary = summary is None
+ if summary is None:
+ summary = SearchSummary()
device_groups = OrderedDict()
for w in deferred_wrappers:
@@ -511,7 +588,7 @@ def run_batched_wrap_search(deferred_wrappers: list[WrapperLinear], max_batch: i
for worker, home, chunk in planned:
buckets.setdefault(worker, []).append((home, chunk))
- stats = {wk: {"modules": 0, "batches": 0, "singletons": 0} for wk in buckets}
+ summary.offloaded += sum(1 for worker, home, _chunk in planned if worker != home)
@torch.no_grad()
def _run_one(wrapper: WrapperLinear) -> None:
@@ -575,7 +652,9 @@ def run_batched_wrap_search(deferred_wrappers: list[WrapperLinear], max_batch: i
dump_oom_tensor_census_("wrap search")
for w in chunk:
w._run_deferred_search_now()
- stats[worker_key]["singletons"] += len(chunk)
+ st = summary.device(worker_key)
+ st["singletons"] += len(chunk)
+ st["modules"] += len(chunk)
return
if worker_key != home_key:
if torch.is_tensor(results):
@@ -584,21 +663,23 @@ def run_batched_wrap_search(deferred_wrappers: list[WrapperLinear], max_batch: i
results = [r.to(home_key) if torch.is_tensor(r) else r for r in results]
for w, res in zip(chunk, results):
w.finalize_batched_search(res)
- stats[worker_key]["batches"] += 1
- stats[worker_key]["modules"] += len(chunk)
+ st = summary.device(worker_key)
+ st["batches"] += 1
+ st["modules"] += len(chunk)
@torch.no_grad()
def _execute(worker_key: str, items: list[tuple[str, list[WrapperLinear]]]) -> None:
- _t0 = time.perf_counter()
- for home_key, chunk in items:
- _run_chunk_stacked(home_key, chunk, worker_key)
- stats[worker_key]["wall"] = time.perf_counter() - _t0
+ with summary.timed(worker_key):
+ for home_key, chunk in items:
+ _run_chunk_stacked(home_key, chunk, worker_key)
# singleton groups and non-stackable searches run per module on the
# calling thread, each on its own weight's device
- for _dev, w in singletons:
- _run_one(w)
- _n_single = len(singletons)
+ for dev, w in singletons:
+ with summary.timed(dev) as st:
+ _run_one(w)
+ st["singletons"] += 1
+ st["modules"] += 1
if len(buckets) > 1:
pairs = list(buckets.items())
@@ -607,27 +688,9 @@ def run_batched_wrap_search(deferred_wrappers: list[WrapperLinear], max_batch: i
elif buckets:
wk, items = next(iter(buckets.items()))
_execute(wk, items)
- _t_total = sum(st.get("wall", 0.0) for st in stats.values())
- _n_mod = sum(st["modules"] for st in stats.values()) + _n_single
- _n_batch = sum(st["batches"] for st in stats.values())
- from auto_round.utils.pool_placement import _short_device_key
-
- per_device = (
- "{" + ", ".join(f"'{_short_device_key(dev)}': {st.get('wall', 0.0):.2f}" for dev, st in stats.items()) + "}"
- )
- _n_offloaded = sum(1 for worker, home, _chunk in planned if worker != home)
- if perf_counters_enabled():
- logger.info(
- "[batched-search] wrap search: %d modules, %d batch + %d singleton search calls"
- + (f" ({_n_offloaded} chunk(s) offloaded to idle devices)" if _n_offloaded else "")
- + ", %.2fs device-wall (%.2fs summed) walls %s",
- _n_mod,
- _n_batch,
- _n_single,
- max(st.get("wall", 0.0) for st in stats.values()) if stats else 0.0,
- _t_total,
- per_device,
- )
+ summary.finish()
+ if _log_summary:
+ summary.log("[batched-search] wrap search:")
return True
@@ -714,7 +777,8 @@ def run_batched_neuqi_anchors(deferred_wrappers: list) -> bool:
for worker, _home, chunk in planned:
buckets.setdefault(worker, []).append(chunk)
- stats = {wk: {"modules": 0, "batches": 0, "singletons": 0} for wk in buckets}
+ summary = SearchSummary()
+ summary.offloaded = sum(1 for worker, home, _chunk in planned if worker != home)
@torch.no_grad()
def _run_anchor_stack(chunk: list, worker_key: str):
@@ -741,7 +805,11 @@ def run_batched_neuqi_anchors(deferred_wrappers: list) -> bool:
@torch.no_grad()
def _execute(worker_key: str, chunk_list: list) -> None:
- _t0 = time.perf_counter()
+ with summary.timed(worker_key) as st:
+ _execute_chunks(worker_key, chunk_list, st)
+
+ @torch.no_grad()
+ def _execute_chunks(worker_key: str, chunk_list: list, st: dict) -> None:
for chunk in chunk_list:
staged = [_staged(w) for w in chunk]
homes = [str(s[0].device) for s in staged]
@@ -769,18 +837,21 @@ def run_batched_neuqi_anchors(deferred_wrappers: list) -> bool:
dump_oom_tensor_census_("neuqi anchor")
for w in chunk:
w._run_deferred_neuqi_anchor_now()
- stats[worker_key]["singletons"] += len(chunk) # counted like the wrap lane's fallbacks
+ st["singletons"] += len(chunk) # counted like the wrap lane's fallbacks
+ st["modules"] += len(chunk)
continue
wmin = wmin.reshape(len(chunk), -1)
wmax = wmax.reshape(len(chunk), -1)
for w, s, mn, mx in zip(chunk, staged, wmin, wmax):
w._apply_neuqi_anchor(mn.to(s[0].device), mx.to(s[0].device))
- stats[worker_key]["batches"] += 1
- stats[worker_key]["modules"] += len(chunk)
- stats[worker_key]["wall"] = time.perf_counter() - _t0
+ st["batches"] += 1
+ st["modules"] += len(chunk)
for w in singletons:
- w._run_deferred_neuqi_anchor_now()
+ with summary.timed(str(_staged(w)[0].device)) as st:
+ w._run_deferred_neuqi_anchor_now()
+ st["singletons"] += 1
+ st["modules"] += 1
if len(buckets) > 1:
pairs = list(buckets.items())
@@ -790,26 +861,5 @@ def run_batched_neuqi_anchors(deferred_wrappers: list) -> bool:
wk, items = next(iter(buckets.items()))
_execute(wk, items)
- _t_total = sum(st.get("wall", 0.0) for st in stats.values())
- _n_single = len(singletons) + sum(st["singletons"] for st in stats.values())
- _n_mod = sum(st["modules"] for st in stats.values()) + _n_single
- _n_batch = sum(st["batches"] for st in stats.values())
- _n_offloaded = sum(1 for worker, home, _chunk in planned if worker != home)
- from auto_round.utils.pool_placement import _short_device_key
-
- per_device = (
- "{" + ", ".join(f"'{_short_device_key(dev)}': {st.get('wall', 0.0):.2f}" for dev, st in stats.items()) + "}"
- )
- if perf_counters_enabled():
- logger.info(
- "[batched-search] neuqi anchors: %d modules, %d batch + %d singleton search calls"
- + (f" ({_n_offloaded} chunk(s) offloaded to idle devices)" if _n_offloaded else "")
- + ", %.2fs device-wall (%.2fs summed) walls %s",
- _n_mod,
- _n_batch,
- _n_single,
- max(st.get("wall", 0.0) for st in stats.values()) if stats else 0.0,
- _t_total,
- per_device,
- )
+ summary.finish().log("[batched-search] neuqi anchors:")
return True
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.py
index d2a3a8070..e150bbab0 100644
--- a/auto_round/algorithms/quantization/sign_round/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py
@@ -95,12 +95,11 @@ from auto_round.utils import (
)
from auto_round.utils.device import clear_memory_if_reached_threshold
from auto_round.utils.device_manager import device_manager
-from auto_round.utils.distributed import setup_ddp_if_needed_
-from auto_round.wrapper import (
- WrapperLinear,
- unwrapper_layer,
- wrapper_block,
-)
+from auto_round.utils.distributed import all_reduce_loss, make_index_sampler, setup_ddp_if_needed_
+from auto_round.wrapper import WrapperLinear, unwrapper_layer, wrapper_block
+
+if TYPE_CHECKING:
+ from auto_round.algorithms.composer import BlockContext
@register_pipeline_member(SignRoundConfig)
@@ -426,7 +425,10 @@ class SignRoundQuantizer(LFQMixin, BaseQuantizer):
best_loss = torch.finfo(torch.float).max
num_elm = 1
mse_reduction = "mean"
- if self.gradient_accumulate_steps != 1:
+ # a masked loss is a SUM over the valid elements, normalized by the
+ # valid-token count below (num_elm) -- also at accumulation == 1, where
+ # a mean over all elements divided again by that count normalizes twice
+ if self.gradient_accumulate_steps != 1 or valid_token_mask:
mse_reduction = "sum"
mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)
scaler = self._get_scaler() # pylint: disable=assignment-from-none
@@ -444,7 +446,7 @@ class SignRoundQuantizer(LFQMixin, BaseQuantizer):
num_elm = sum(active_inputs[i.item()].numel() for i in whole_indices)
block, sync_gradients = setup_ddp_if_needed_(self, block, device_manager.device_list)
- index_sampler = IndexSampler(nsamples, global_batch_size)
+ index_sampler = make_index_sampler(nsamples, global_batch_size)
block_fwd = self.block_forward
# When low_gpu_mem_usage is enabled, active_inputs / fp_outputs are intentionally
@@ -652,6 +654,7 @@ class SignRoundQuantizer(LFQMixin, BaseQuantizer):
# clear memory to avoid OOM due to memory fragmentation
clear_memory_if_reached_threshold(threshold=0.8, device_list=device_manager.device_list)
+ total_loss = all_reduce_loss(total_loss, num_elm)
if i == 0:
init_loss = total_loss
current_lr = optimizer.param_groups[0]["lr"]
diff --git a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
index c833a6542..2289e7b16 100644
--- a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
@@ -490,6 +490,8 @@ class SignRoundDQWrapperLinear(WrapperLinear):
class SignRoundV2Quantizer(SignRoundQuantizer):
"""SignRound variant using the open algorithm-extension path in the new architecture."""
+ calibration_stats = {"imatrix": "sum"}
+
def __init__(self, config: SignRoundConfig) -> None:
super().__init__(config)
self._use_outlier_suppressed_loss = False
diff --git a/auto_round/cli/main.py b/auto_round/cli/main.py
index 7984a58bd..96fdc9011 100644
--- a/auto_round/cli/main.py
+++ b/auto_round/cli/main.py
@@ -446,6 +446,45 @@ def tune(args):
device_str, use_auto_mapping = get_device_and_parallelism(args.device_map)
+ _parallel = str(getattr(args, "data_parallel", "off") or "off").strip().lower()
+ _parallel_policy = None
+ if _parallel != "off":
+ import torch
+
+ from auto_round import envs as _envs
+ from auto_round.algorithms.parallel.data_parallel import (
+ ParallelPolicy,
+ resolve_parallel_world,
+ )
+
+ _n = torch.cuda.device_count()
+ _iters_cli = int(getattr(args, "iters", 0) or 0)
+ _world = resolve_parallel_world(_parallel, _n, iters=_iters_cli)
+ # early eligibility: a multi-device --device_map cannot host the
+ # data-parallel lane at iters > 0 (blocks shard across GPUs in a
+ # pipeline-style placement) -- stop here, before the model loads,
+ # with the same message the runtime resolver raises at block 0
+ from auto_round.algorithms.parallel.data_parallel import check_parallel_device_map_compat_
+
+ check_parallel_device_map_compat_(_world, args.device_map, _iters_cli)
+ _parallel_policy = ParallelPolicy(
+ world=_world,
+ source="auto" if _parallel == "auto" else "explicit",
+ collect_forward_cap=int(getattr(_envs, "AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES", 0) or 0),
+ batch_mode=str(getattr(args, "parallel_batch", "split") or "split").strip().lower(),
+ )
+ if _parallel == "auto" and _world < _n:
+ # only `auto` adds information here (the reduction from the
+ # visible count); an explicit N either validates silently or
+ # raises, and the engagement line reports the resolved plan
+ logger.info(
+ "[tune-ddp] --data_parallel auto -> world=%d (largest power of two <= %d visible "
+ "devices; %d unused)",
+ _world,
+ _n,
+ _n - _world,
+ )
+
if args.enable_torch_compile is False:
logger.info("`torch.compile` is explicitly disabled with `--disable_torch_compile`.")
@@ -575,6 +614,11 @@ def tune(args):
),
)
+ if _parallel_policy is not None:
+ # the resolved --data_parallel policy is set on the compress
+ # context; the data-parallel engine reads it from there
+ autoround.compress_context.parallel_policy = _parallel_policy
+
model, folders = autoround.quantize_and_save( # pylint: disable=no-member
args.output_dir,
format=getattr(args, "_api_format", args.format),
diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.py
index 60c29e799..e8ae636f9 100644
--- a/auto_round/cli/parser.py
+++ b/auto_round/cli/parser.py
@@ -154,6 +154,28 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Argu
rt.add_argument(
"--device_map", "--device", "--devices", default="0", type=str, help="Device mapping used for quantization."
)
+ rt.add_argument(
+ "--data_parallel",
+ default="off",
+ type=str,
+ help="Single-process data-parallel SignRound tuning: 'off' (default), 'auto' (all visible CUDA "
+ "devices at --iters 0, otherwise the largest power of two that fits them), or an explicit "
+ "world size N (at least 2, at most the visible device count; a power of two when "
+ "--iters > 0). Replicas hold full block mirrors on the plan "
+ "devices and draw disjoint calibration shards; free-VRAM estimates are advisory.",
+ )
+ rt.add_argument(
+ "--parallel_batch",
+ default="split",
+ type=str,
+ choices=["split", "per_replica"],
+ help="Batch semantics for --data_parallel tuning. 'split' (default): the requested "
+ "--batch_size is the GLOBAL per-iteration batch and every replica draws its world-split "
+ "share (serial-equivalent updates). 'per_replica': every replica draws the FULL "
+ "--batch_size from its own calibration shard each iteration, so the effective global "
+ "batch is batch_size x world (torchrun-like); the per-replica draw clamps to the pool shard "
+ "when it is smaller.",
+ )
rt.add_argument("--dataset", default=None, type=str, help="Calibration dataset or local dataset path.")
rt.add_argument("--seed", default=42, type=int, help="Random seed for reproducibility.")
rt.add_argument(
diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.py
index c87ddb34e..e3fe0dc3b 100644
--- a/auto_round/compressors/orchestrator.py
+++ b/auto_round/compressors/orchestrator.py
@@ -416,6 +416,12 @@ class CompressionOrchestrator(BaseOrchestrator):
# folded into the end-of-block phases line as its first item so
# the line's total covers the whole block (true-wall comparable)
block_name_or_names = n if nblocks == 1 else names
+ # tick(key) closes the running span and opens `key`, so every
+ # tick sits right BEFORE the region it names. The chain opens
+ # here: "prep" (materialize + dtype convert + dispatch_block
+ # device placement + BlockContext) costs seconds per block on
+ # large MoE models, and the total covers the true per-block wall.
+ _perf.tick("prep")
# ── Infrastructure: materialize, dtype convert, device placement ──
materialize_model_(m)
@@ -453,8 +459,9 @@ class CompressionOrchestrator(BaseOrchestrator):
# primary cache device when it fits (today's behavior, zero peer
# traffic), or sharded across free GPUs when it does not. Placement is
# pure memory behavior -- chunk values are bit-identical.
- self._attach_pool_placement(m, input_ids, q_input, input_others)
_perf.tick("attach")
+ self._attach_pool_placement(m, input_ids, q_input, input_others)
+ _perf.tick("compress")
# ── Run block pipeline (calibration → quantization → collection) ──
new_q_input, reference_output = self.alg_composer.compress_block(
@@ -465,7 +472,7 @@ class CompressionOrchestrator(BaseOrchestrator):
q_inputs=q_input,
input_ids=token_ids,
)
- _perf.tick("compress")
+ _perf.tick("swap")
# ── Infrastructure: memory management ─────────────────────────────
# Mirrors the original q_input-swap + end-of-loop clear_memory semantics:
@@ -483,7 +490,7 @@ class CompressionOrchestrator(BaseOrchestrator):
clear_memory(input_ids if input_ids is not next_input_ids else None)
q_input = new_q_input
- _perf.tick("swap")
+ _perf.tick("sweep")
# Belt-and-braces: no module may carry staged batched-search inputs
# (weight reshape + imatrix copies) across the block boundary;
@@ -492,10 +499,10 @@ class CompressionOrchestrator(BaseOrchestrator):
for _mn, _mm in m.named_modules():
if getattr(_mm, "_deferred_search_inputs", None) is not None:
_mm._deferred_search_inputs = None
- _perf.tick("sweep")
+ _perf.tick("hooks")
if len(device_manager.device_list) > 1 and not self.model_context.is_diffusion:
accelerate.hooks.remove_hook_from_submodules(m)
- _perf.tick("hooks")
+ _perf.tick("pack")
# ── Infrastructure: immediate_pack / shard write ──────────────────
# Pack BEFORE draining the block off the GPUs: pack_layer resolves
@@ -521,14 +528,14 @@ class CompressionOrchestrator(BaseOrchestrator):
if module_name is None:
continue
_immediate_pack(module_name, _layer_cfg)
- _perf.tick("pack")
+ _perf.tick("mv")
mv_module_from_gpu(m)
- _perf.tick("mv")
- clear_memory(device_list=device_manager.device_list)
_perf.tick("clear")
- memory_monitor.log_summary()
+ clear_memory(device_list=device_manager.device_list)
_perf.tick("monitor")
+ memory_monitor.log_summary()
+ _perf.tick("write")
input_ids = next_input_ids
@@ -583,13 +590,13 @@ class CompressionOrchestrator(BaseOrchestrator):
_bg_resume.dispatch(
lambda _n=n, _q=_resume_q, _i=_resume_i: resume_state.mark_block_done(_n, _q, _i)
)
- _perf.tick("write")
if _perf.enabled:
_perf.stop("write")
# sweep/hooks fold into the printed pack span, monitor into the
# printed write span; they were bad-perf-investigation probes that
# read 0.00 in every validated run since
_parts = [
+ f"prep={_perf.get('prep'):.2f}s",
f"attach={_perf.get('attach'):.2f}s",
f"compress={_perf.get('compress'):.2f}s",
f"swap={_perf.get('swap'):.2f}s",
@@ -681,6 +688,13 @@ class CompressionOrchestrator(BaseOrchestrator):
# full-model materialization and linear CPU RAM growth.
logger.info("Zero-shot mode (no calibration data needed): using blockwise quantization.")
+ # an explicit --data_parallel N cannot engage here (no
+ # calibration forward for the data-parallel lane to shard): one INFO
+ # line, no refusal -- weights-only RTN has nothing to parallelize
+ from auto_round.algorithms.parallel.data_parallel import log_zero_shot_parallel_noop_
+
+ log_zero_shot_parallel_noop_(getattr(self.compress_context, "parallel_policy", None))
+
tied_weights_keys = getattr(self.model, "_tied_weights_keys", [])
if tied_weights_keys is None:
tied_weights_keys = []
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index d94a64d37..78ac1233a 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -185,6 +185,64 @@ def block_forward(
input_others[param_name] = val
positional_inputs = ()
+ # DDP-mirror device audit (last line of defense): every tensor kwarg must
+ # sit on the same accelerator device as the block's parameters. A stale
+ # kwarg (e.g. an entry recorded on the primary GPU while the replica
+ # lives elsewhere) silently overrides the staged first-arg inside this
+ # function and crashes cross-device inside the block forward -- serial
+ # runs with a single shared device stay unaffected. Move the stragglers
+ # onto the block's device AND name them (fail-visible).
+ try:
+ _block_dev = next(block.parameters()).device
+ except StopIteration: # pragma: no cover - param-less blocks
+ _block_dev = device
+
+ def _audit(value, prefix, bad):
+ if torch.is_tensor(value):
+ if value.device.type in ("cuda", "xpu", "hpu") and value.device != _block_dev:
+ bad.append(f"{prefix}:{value.device}")
+ elif isinstance(value, (list, tuple)):
+ for _i, _v in enumerate(value):
+ _audit(_v, f"{prefix}[{_i}]", bad)
+ elif isinstance(value, dict):
+ for _k, _v in value.items():
+ _audit(_v, f"{prefix}.{_k}", bad)
+
+ def _relocate(value):
+ # Mirror _audit's condition exactly: only accelerator tensors that
+ # mismatch the block device move. A wholesale to_device() here would
+ # also migrate host tensors nested in lists/dicts onto the accelerator
+ # (semantic change + VRAM), which the audit never flagged.
+ if torch.is_tensor(value):
+ if value.device.type in ("cuda", "xpu", "hpu") and value.device != _block_dev:
+ return value.to(_block_dev)
+ return value
+ if isinstance(value, list):
+ return [_relocate(_v) for _v in value]
+ if isinstance(value, tuple):
+ return tuple(_relocate(_v) for _v in value)
+ if isinstance(value, dict):
+ return {k: _relocate(v) for k, v in value.items()}
+ return value
+
+ _bad = []
+ for _k, _v in input_others.items():
+ _audit(_v, str(_k), _bad)
+ if _bad:
+ logger.warning(
+ "[block-forward] device sweep: moving %d stale kwarg tensor(s) onto %s: %s%s",
+ len(_bad),
+ _block_dev,
+ ", ".join(_bad[:8]),
+ " ..." if len(_bad) > 8 else "",
+ )
+ for _k in list(input_others.keys()):
+ if torch.is_tensor(input_others[_k]) and input_others[_k].device.type in ("cuda", "xpu", "hpu"):
+ if input_others[_k].device != _block_dev:
+ input_others[_k] = input_others[_k].to(_block_dev)
+ elif isinstance(input_others[_k], (list, tuple, dict)):
+ input_others[_k] = _relocate(input_others[_k])
+
if amp:
with autocast(device_type=str(device).split(":")[0], dtype=amp_dtype): # pragma: no cover
output = block(**input_others)
@@ -452,19 +510,35 @@ class IndexSampler:
nsamples (int): Total number of samples.
batch_size (int): Number of indices to return in each batch.
index (int): Current position in the index list.
- indices (List[int]): Shuffled list of indices.
+ indices (list[int]): Shuffled list of indices.
"""
- def __init__(self, nsamples: int, batch_size: int) -> None:
+ def __init__(
+ self,
+ nsamples: int,
+ batch_size: int,
+ indices: "list[int] | None" = None,
+ rng: "random.Random | None" = None,
+ ) -> None:
"""Initializes the sampler.
Args:
nsamples (int): Total number of samples (must be >= batch_size).
batch_size (int): Number of indices per batch.
+ indices (list[int], optional): Explicit index pool to shuffle
+ (defaults to ``range(nsamples)``). Used by shard-constrained
+ DDP sampling; ``nsamples`` must equal ``len(indices)``.
+ rng (random.Random, optional): Private shuffle generator (defaults
+ to the global ``random`` module). Data-parallel rank threads
+ pass their own so concurrent reshuffles stay reproducible.
Raises:
ValueError: If batch_size is not in the range (0, nsamples].
"""
+ if indices is None:
+ indices = list(range(nsamples))
+ elif len(indices) != nsamples:
+ raise ValueError(f"indices pool ({len(indices)}) does not match nsamples ({nsamples})")
if batch_size <= 0 or batch_size > nsamples:
raise ValueError("batch_size must be > 0 and <= nsamples")
@@ -472,8 +546,9 @@ class IndexSampler:
self.batch_size: int = batch_size
self.index: int = 0
- self.indices: list[int] = list(range(nsamples))
- random.shuffle(self.indices)
+ self._rng = rng if rng is not None else random
+ self.indices: list[int] = list(indices)
+ self._rng.shuffle(self.indices)
def next_batch(self) -> list[int]:
"""Returns the next batch of shuffled indices.
@@ -485,7 +560,7 @@ class IndexSampler:
list[int]: A list of size `batch_size` containing sample indices.
"""
if self.index + self.batch_size > self.nsamples:
- random.shuffle(self.indices)
+ self._rng.shuffle(self.indices)
self.index = 0
batch = self.indices[self.index : self.index + self.batch_size]
diff --git a/auto_round/context/compress.py b/auto_round/context/compress.py
index c6f35b74b..09eb35c2b 100644
--- a/auto_round/context/compress.py
+++ b/auto_round/context/compress.py
@@ -66,6 +66,9 @@ class CompressContext(BaseContext):
self.static_attention_dtype = static_attention_dtype
self.static_kv_granularity = static_kv_granularity
self.static_attention_granularity = static_attention_granularity
+ # Resolved --data_parallel policy (ParallelPolicy or None = off);
+ # set once at the CLI entry, read by the data-parallel engine
+ self.parallel_policy = kwargs.pop("parallel_policy", None)
def clear_memory(self, tensor=None):
"""Clear GPU/CPU memory only when ``low_gpu_mem_usage`` is enabled."""
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 1b6744be0..fb679e89b 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import functools
import math
from collections.abc import Callable
from functools import lru_cache
@@ -199,7 +200,6 @@ def _int_search_with_thresh(search_scales: Any, q_scale_thresh: float) -> "Calla
per-call closure, so resolved callables compare equal across calls and can
safely key same-shape search batches.
"""
- import functools
def _clamped(
weight_reshape: torch.Tensor, bits: int, imatrix: "torch.Tensor | None", *, _thresh: float
@@ -372,10 +372,12 @@ def reshape_imatrix_for_weight(imatrix, weight_reshape: torch.Tensor, group_size
"""
if imatrix is None or not isinstance(imatrix, torch.Tensor):
return torch.ones_like(weight_reshape)
- imatrix = imatrix.reshape(1, -1)
+ # relocate the RAW column first: moving the expanded copy after the fact
+ # would pay a weight-sized cross-device transfer for sharded placements
+ imatrix = imatrix.reshape(1, -1).to(weight_reshape.device)
imatrix = reshape_pad_tensor_by_group_size(imatrix, group_size, val=1e-5)[0].view(1, -1)
imatrix = imatrix.expand(weight_reshape.numel() // imatrix.numel(), -1)
- return imatrix.reshape(weight_reshape.shape).to(weight_reshape.device)
+ return imatrix.reshape(weight_reshape.shape)
def compute_optimized_init_scale(
diff --git a/auto_round/envs.py b/auto_round/envs.py
index 595b17d1a..57f0f0398 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -45,6 +45,7 @@ if TYPE_CHECKING:
AR_SEARCH_BATCH_GB: float | None = None
AR_DISABLE_CROSS_DEVICE_SEARCH: bool = False
AR_ENABLE_LOG_MINMAX: bool = False
+ AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES: int = 0
def _get_optional_positive_int_env(name: str) -> int | None:
@@ -61,6 +62,20 @@ def _get_optional_positive_int_env(name: str) -> int | None:
return value
+def _get_non_negative_int_env(name: str, default: int) -> int:
+ """Read an env var that must be a non-negative integer; malformed values fail fast."""
+ raw = os.getenv(name)
+ if raw is None:
+ return default
+ try:
+ value = int(raw)
+ except ValueError as exc:
+ raise ValueError(f"{name} must be a non-negative integer, got {raw!r}") from exc
+ if value < 0:
+ raise ValueError(f"{name} must be a non-negative integer, got {raw!r}")
+ return value
+
+
def _get_optional_positive_float_env(name: str) -> float | None:
"""Read an optional env var that must be a positive float when set."""
raw = os.getenv(name)
@@ -172,6 +187,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
"AR_RESUME_DIR": lambda: os.getenv("AR_RESUME_DIR", None),
"AR_DISABLE_BATCHED_SEARCH": lambda: os.getenv("AR_DISABLE_BATCHED_SEARCH", "False").strip().lower()
in ("1", "true", "yes"),
+ # Emit [perf] phase-breakdown log lines (per-block load/tune/pack/write/clean/offload
+ # in the data-driven loop; the sharded collection breakdown for --data_parallel).
"AR_PERF_COUNTERS": lambda: os.getenv("AR_PERF_COUNTERS", "False").strip().lower() in ("1", "true", "yes"),
"AR_SEARCH_BATCH_GB": lambda: _get_optional_positive_float_env("AR_SEARCH_BATCH_GB"),
"AR_DISABLE_CROSS_DEVICE_SEARCH": lambda: os.getenv("AR_DISABLE_CROSS_DEVICE_SEARCH", "False").strip().lower()
@@ -216,9 +233,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Benchmarks (A100, W4G128 tuning fwd+bwd) show "auto" matches or beats a
# fixed 16 on every Qwen3-MoE preset (e.g. 35B-A3B: 108 ms vs 156 ms).
# <int > 0> - fixed group size. Fusing every active expert at once builds a working set
- # that grows with the expert count; past a point that costs peak memory on
- # GPU and cache locality on CPU (measured: fusing 64 experts halved CPU
- # calibration throughput, and doubled the GPU calibration peak).
+ # that grows with the expert count; past a point that costs peak
+ # memory on GPU and cache locality on CPU.
# 0 or <0 - disable chunking/tiling: "fuse everything" in one group (e.g. set -1 to turn it off).
# Results are identical for any value -- rows stay independent. A fixed count and "auto"
# are both torch.compile-friendly (constant fused shape -> no per-count recompile).
@@ -257,6 +273,14 @@ environment_variables: dict[str, Callable[[], Any]] = {
"AR_NEUQI_COARSE": lambda: int(v) if (v := os.getenv("AR_NEUQI_COARSE")) is not None else None,
"AR_NEUQI_FINE": lambda: int(v) if (v := os.getenv("AR_NEUQI_FINE")) is not None else None,
"AR_NEUQI_BACKEND": lambda: os.getenv("AR_NEUQI_BACKEND", "auto").lower(),
+ # Cap the number of devices a hook-carrying sharded collection forward runs
+ # on concurrently (hook passes keep compiled runners python-bound between
+ # graph sections, which can convoy on the GIL). Only lower this on hosts
+ # where hook-carrying passes convoy. Values above the engaged world are
+ # no-ops. Default 0 = no cap.
+ "AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES": lambda: _get_non_negative_int_env(
+ "AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES", 0
+ ),
}
diff --git a/auto_round/utils/device.py b/auto_round/utils/device.py
index 1381aeaf7..ac9e777c8 100644
--- a/auto_round/utils/device.py
+++ b/auto_round/utils/device.py
@@ -127,7 +127,12 @@ def _bump_dynamo_cache_limit(min_size: int | None = None):
min_size = envs.AR_DYNAMO_CACHE_SIZE_LIMIT
from torch._dynamo import config as _dynamo_config
- for attr in ("cache_size_limit", "accumulated_cache_size_limit", "recompile_limit"):
+ for attr in (
+ "cache_size_limit",
+ "accumulated_cache_size_limit",
+ "recompile_limit",
+ "accumulated_recompile_limit", # newer torch: process-wide cap (binds on MoE shape zoos)
+ ):
if hasattr(_dynamo_config, attr) and getattr(_dynamo_config, attr) < min_size:
setattr(_dynamo_config, attr, min_size)
except Exception: # pragma: no cover - best effort
@@ -1580,6 +1585,9 @@ class MemoryMonitor:
self.peak_ram = 0.0 # GB
self.peak_vram = {} # {device_id: peak_mb}
self.enabled = True
+ # devices registered for sampling BEYOND the caller's device_map list
+ # (e.g. DDP replica devices the parallel lane engaged outside the map)
+ self._extra_devices: set[str] = set()
@staticmethod
def _process_tree_rss() -> float:
@@ -1593,6 +1601,31 @@ class MemoryMonitor:
continue
return rss / 1024**3
+ def register_extra_devices(self, devices) -> None:
+ """Add accelerator devices to the monitored set beyond the device_map.
+
+ The final ``'peak_vram': {...}`` summary only reports devices the
+ ``update`` calls sample; callers pass the device-map list, so devices
+ engaged outside the map (single-process DDP replicas on all visible
+ GPUs while ``--device_map`` is single-device) would stay invisible.
+ Registered devices are merged into every subsequent ``update``/
+ ``update_hpu`` sample; ``reset`` clears the peaks but keeps the
+ registration (engagement re-registers per block).
+ """
+ if not hasattr(self, "_extra_devices"): # older singleton instance
+ self._extra_devices = set()
+ for d in devices or []:
+ key = str(d)
+ if key.split(":")[0] in ("cuda", "xpu", "hpu"):
+ self._extra_devices.add(key)
+
+ def _merged_device_list(self, device_list):
+ """The caller's device list plus any registered extra devices (deduped)."""
+ if not getattr(self, "_extra_devices", None):
+ return device_list
+ seen = [str(d) for d in device_list]
+ return list(device_list) + [d for d in sorted(self._extra_devices) if d not in seen]
+
def update(self, device_list=None):
"""Update current memory usage and track peaks."""
if not self.enabled:
@@ -1611,6 +1644,7 @@ class MemoryMonitor:
device_list = list(range(dev_mgr.device_count()))
else:
device_list = [0]
+ device_list = self._merged_device_list(device_list)
dev_mgr = get_current_device_manager()
for device in device_list:
@@ -1658,6 +1692,7 @@ class MemoryMonitor:
device_list = list(range(count)) if count > 0 else [0]
elif not isinstance(device_list, (list, tuple)):
device_list = [device_list]
+ device_list = self._merged_device_list(device_list)
for device in device_list:
if str(device) == "cpu":
continue
diff --git a/auto_round/utils/device_manager.py b/auto_round/utils/device_manager.py
index 9db19f703..ba43e9033 100644
--- a/auto_round/utils/device_manager.py
+++ b/auto_round/utils/device_manager.py
@@ -821,6 +821,9 @@ class DeviceManager:
@property
def device_list(self) -> list:
"""All concrete devices selected by the configured ``device_map``."""
+ scoped = self._thread_scope()
+ if scoped is not None:
+ return [scoped]
if self._device_list is None:
self.configure(self._device_map)
return self._device_list
@@ -828,10 +831,35 @@ class DeviceManager:
@property
def device(self) -> str:
"""The major (primary, non-CPU when possible) device string."""
+ scoped = self._thread_scope()
+ if scoped is not None:
+ return scoped
if self._major_device is None:
self.configure(self._device_map)
return self._major_device
+ def _thread_scope(self) -> str | None:
+ local = self.__dict__.get("_thread_local")
+ return getattr(local, "device", None) if local is not None else None
+
+ @contextlib.contextmanager
+ def thread_device_scope(self, device: str | torch.device):
+ """Pin :attr:`device` / :attr:`device_list` to ``device`` for the calling thread.
+
+ Used by the data-parallel engine: each rank thread runs the unchanged
+ per-block algorithm and must see its own replica device as the major
+ device. Other threads keep the configured devices.
+ """
+ import threading
+
+ local = self.__dict__.setdefault("_thread_local", threading.local())
+ prev = getattr(local, "device", None)
+ local.device = str(device)
+ try:
+ yield
+ finally:
+ local.device = prev
+
@device.setter
def device(self, value: str | torch.device) -> None:
"""Override the major device (e.g. an OOM fallback to ``"cpu"``)."""
diff --git a/auto_round/utils/distributed.py b/auto_round/utils/distributed.py
index 6a7936ce9..5879a8f2b 100644
--- a/auto_round/utils/distributed.py
+++ b/auto_round/utils/distributed.py
@@ -55,6 +55,14 @@ def setup_ddp_if_needed_(ar, block: torch.nn.Module, device_list: list[int]):
"""
import torch.distributed as dist
+ from auto_round.algorithms.parallel.spmd import current_rank
+
+ rank = current_rank()
+ if rank is not None:
+ # in-process thread ranks (--data_parallel): each rank tunes
+ # its own replica; the hook averages gradients across the ranks
+ return block, lambda: rank.sync_gradients(block)
+
# The process group can be created and destroyed by individual tests or
# by successive launcher invocations. Do not reuse a cached result from
# a previous process-group lifetime.
@@ -105,6 +113,40 @@ def _noop_sync():
pass
+def make_index_sampler(nsamples: int, batch_size: int):
+ """The tuning loop's sample sampler.
+
+ Serial and torchrun ranks (whose calibration data the caller already
+ partitioned) draw from all ``nsamples``. An in-process thread rank draws
+ from its own contiguous shard of the calibration pool.
+ """
+ from auto_round.algorithms.parallel.spmd import current_rank
+ from auto_round.compressors.utils import IndexSampler
+
+ rank = current_rank()
+ if rank is not None:
+ return rank.index_sampler(nsamples, batch_size)
+ return IndexSampler(nsamples, batch_size)
+
+
+def all_reduce_loss(loss: float, weight: float = 1.0) -> float:
+ """Global iteration loss across ranks; identity when not data-parallel.
+
+ ``loss`` is the rank's normalized loss and ``weight`` its normalizer
+ (element or valid-token count), so the result is the weighted mean
+ sum(loss * weight) / sum(weight) -- the serial value over the union of
+ the ranks' samples. Every rank must take the same best-iteration and
+ early-stop decisions (it holds the same parameters after each synced
+ step), so the decisions use the global loss.
+ """
+ from auto_round.algorithms.parallel.spmd import current_rank
+
+ rank = current_rank()
+ if rank is not None:
+ return rank.all_reduce_weighted_mean(loss, weight)
+ return loss
+
+
def _move_block_to_device(module: torch.nn.Module, device):
"""Move *module* to *device*, used before DDP wrapping."""
module.to(device)
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 331d84b1a..d154b604a 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -1063,10 +1063,15 @@ def wrapper_block(
for w in anchor_pending:
w._run_deferred_neuqi_anchor_now()
# stacked same-shape batches per (device, shape, config, search fn);
- # singletons and the kill-switch fall back to the identical per-module call
- if not search_dispatch.run_batched_wrap_search(deferred_wrappers):
- for w in deferred_wrappers:
- w._run_deferred_search_now()
+ # singletons and the kill-switch fall back to the identical per-module call.
+ # Data-parallel ranks shard the searches across the ranks instead.
+ from auto_round.algorithms.parallel.spmd import current_rank, run_rank_sharded_wrap_searches
+
+ rank = current_rank()
+ if not (rank is not None and run_rank_sharded_wrap_searches(rank, deferred_wrappers)):
+ if not search_dispatch.run_batched_wrap_search(deferred_wrappers):
+ for w in deferred_wrappers:
+ w._run_deferred_search_now()
quantized_layers = [n for (n, _m, _k), r in zip(work, results) if r == "q"]
unquantized_layers = [n for (n, _m, _k), r in zip(work, results) if r == "u"]
diff --git a/docs/environments.md b/docs/environments.md
index 66e5f1788..74db88e88 100644
--- a/docs/environments.md
+++ b/docs/environments.md
@@ -411,6 +411,16 @@ else:
4. **Performance Optimization**: Enable `AR_ENABLE_COMPILE_PACKING=1` if you have sufficient computational resources
5. **Custom Workspace**: Set `AR_WORK_SPACE` to a directory with sufficient disk space for model processing
+### AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES
+- **Description**: Caps the number of devices a hook-carrying sharded collection forward runs on concurrently (hook passes keep compiled runners python-bound between graph sections, which can convoy on the GIL). Only lower this on hosts where hook-carrying passes convoy; not reproduced on the validation rig up to world=8. Values above the engaged world are no-ops.
+- **Default**: `0` (no cap)
+- **Valid Values**: non-negative integer
+- **Usage**: Limit concurrent shard-forward threads on weak-CPU hosts.
+
+```bash
+AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES=2 python -m auto_round --model ... --data_parallel 8
+```
+
## Notes
- Environment variables are evaluated lazily, meaning they are only read when first accessed
diff --git a/docs/environments_CN.md b/docs/environments_CN.md
index 7a43b744a..1eb1f1d1d 100644
--- a/docs/environments_CN.md
+++ b/docs/environments_CN.md
@@ -411,6 +411,16 @@ else:
4. **性能优化**:如有足够算力,可启用 `AR_ENABLE_COMPILE_PACKING=1`
5. **自定义工作目录**:将 `AR_WORK_SPACE` 设置为磁盘空间充足的目录
+### AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES
+- **描述**:限制携带 hook 的分片采集前向并发使用的设备数(hook 前向会使编译后的执行器在图分段之间退回 Python 绑定,可能在 GIL 上形成串行瓶颈)。仅在 hook 前向出现串行瓶颈的主机上才需要调低;在验证机台上至 world=8 未复现。大于已启用 world 的值等效于不限制。
+- **默认值**:`0`(不限制)
+- **取值**:非负整数
+- **用法**:在弱 CPU 主机上限制并发分片前向线程数。
+
+```bash
+AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES=2 python -m auto_round --model ... --data_parallel 8
+```
+
## 注意事项
- 环境变量采用懒加载方式,仅在首次访问时读取
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 60bf7421a..58366832e 100755
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -936,6 +936,37 @@ CUDA_VISIBLE_DEVICES=0,1,2,3 auto-round --model "Qwen/Qwen3-0.6B" --scheme "W4A1
There are typically two scenarios that require multi-GPU tuning: one is the calibration phase mainly for lm-head quantization, and the other is quantizing extremely large models (e.g., models larger than 100 GB).
+#### Parallel block tuning across GPUs
+When a tuning block fits on one GPU, `--data_parallel` uses the other GPUs of the same process for
+data-parallel tuning: the block's tuning loop runs once per GPU (a rank, one thread each) on that GPU's copy of the
+block and its share of the calibration samples, and the ranks average their gradients every iteration. This is the
+same contract as the multi-process torchrun path (`sync_gradients()`), so the algorithm's loop itself is unchanged.
+
+~~~bash
+CUDA_VISIBLE_DEVICES=0,1,2,3 auto-round --model "Qwen/Qwen3-0.6B" --scheme "W4A16" --data_parallel auto
+~~~
+
+- `off` (default) keeps the serial loop. `N` pins the number of ranks: at least 2, at most the visible device count,
+ and a power of two when `--iters > 0` (the gradient all-reduce is a halving-doubling exchange); checked at startup.
+ `auto` takes all visible devices at `--iters 0` and the largest power of two that fits them at `--iters > 0`, and
+ announces the choice in the log.
+- The block must sit whole on one GPU: run without `--device_map` or with a single-device map. A block spread across
+ several GPUs stops with that reason; blocks larger than one GPU are not covered.
+- Every rank holds a full copy of the block plus its tuning state. Free-VRAM estimates only warn (naming the
+ device); an actual shortage surfaces as an out-of-memory error.
+- `--parallel_batch split` (default) keeps `--batch_size` as the global batch of an iteration; every rank draws
+ `batch_size / world` samples (the division must be exact). `--parallel_batch per_replica` lets every rank draw the
+ full `--batch_size`, so the global batch is `batch_size x world`.
+- A block whose sample count or batch does not split evenly over the ranks, or whose per-rank batch exceeds its
+ share of the samples, tunes serially and logs the reason. LFQ, dict-shaped (diffusion-style) calibration pools and
+ an active torchrun group are not supported.
+- The calibration forwards (reference outputs, the quantized-input chain, imatrix and activation statistics) are
+ sharded across the GPUs too, also at `--iters 0`. Quantizers declare the statistics their hooks write and how they
+ reduce (`calibration_stats`); a hook pass with undeclared statistics stays serial.
+- Each rank samples from its own share of the calibration pool, so per-iteration loss lines and tuned weights differ
+ from a serial run. Compare runs at the model level (accuracy, KL divergence against the base model).
+- To exclude devices, set the backend's visibility variable (e.g. `CUDA_VISIBLE_DEVICES`) before launch.
+
#### Enable multiple gpus calibration in lm_head quantization
For LM head tuning, AutoRound needs to cache the inputs to the lm-head, which requires the entire model to reside on
the GPU for efficient calibration. If there is no enough VRAM, some layers will fallback to RTN mode
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index 5514fa88d..1e708b5bc 100755
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -907,6 +907,34 @@ CUDA_VISIBLE_DEVICES=0,1,2,3 auto-round --model "Qwen/Qwen3-0.6B" --scheme "W4A1
通常有两种情况需要启用多 GPU 训练:一是主要针对 lm-head 量化的标定阶段,二是参数量极大(如显存占用超 100GB)的模型。
+#### 跨 GPU 并行 block 调优
+当单个 block 可以放入单张 GPU 时,可通过 `--data_parallel` 在同一进程中利用其他 GPU 做数据并行调优:
+block 的调优循环在每张 GPU 上各运行一次(每张 GPU 一个 rank,各占一个线程),使用该 GPU 上的 block 副本和它分到的
+那部分标定样本,各 rank 在每次迭代中对梯度求平均。这与多进程 torchrun 路径的约定(`sync_gradients()`)相同,因此算法
+本身的调优循环不需要修改。
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1,2,3 auto-round --model "Qwen/Qwen3-0.6B" --scheme "W4A16" --data_parallel auto
+```
+
+- `off`(默认)保持串行循环。`N` 指定 rank 数:至少为 2,不超过可见设备数;`--iters > 0` 时必须为 2 的幂
+ (梯度 all-reduce 采用 halving-doubling 交换),启动时校验。`auto` 在 `--iters 0` 时使用全部可见设备,在
+ `--iters > 0` 时取不超过可见设备数的最大 2 的幂,并在日志中说明所选数量。
+- block 必须完整放在一张 GPU 上:不指定 `--device_map`,或使用单设备的 map。跨多张 GPU 放置的 block 会带着原因
+ 停止;超过单张 GPU 容量的 block 不在支持范围内。
+- 每个 rank 持有完整的 block 副本及其调优状态。空闲显存估算只会给出告警(指明设备),真正的显存不足会以 OOM
+ 错误的形式出现。
+- `--parallel_batch split`(默认)将 `--batch_size` 作为每次迭代的全局 batch,每个 rank 抽取 `batch_size / world`
+ 个样本(必须整除)。`--parallel_batch per_replica` 让每个 rank 抽取完整的 `--batch_size`,全局 batch 为
+ `batch_size x world`。
+- 若某个 block 的样本数或 batch 不能被 rank 数整除,或每个 rank 的 batch 超过它分到的样本数,该 block 改为串行
+ 调优并在日志中给出原因。不支持 LFQ、字典形式(diffusion 类)的标定池,以及已激活的 torchrun 进程组。
+- 标定前向(参考输出、量化输入链、imatrix 与激活统计)同样在各 GPU 间分片执行,`--iters 0` 时也是如此。量化器通过
+ `calibration_stats` 声明其 hook 写入的统计量及归约方式;带有未声明统计量的 hook 前向保持串行。
+- 每个 rank 从自己那部分标定池中抽样,因此每次迭代的 loss 日志和调优后的权重与串行运行不同。请在模型层面
+ (精度、相对基础模型的 KL 散度)比较不同运行。
+- 如需排除设备,请在启动前设置对应后端的可见性环境变量(如 `CUDA_VISIBLE_DEVICES`)。
+
#### lm_head 量化中开启多 GPU 标定
量化 lm-head 时,AutoRound 需要缓存其输入数据以进行高效的标定,这要求**整个模型驻留在 GPU 显存中** ;若 GPU 显存不足,部分层会退回至 Pure RTN 模式。
diff --git a/test/unit/common/compressors/test_reference_output.py b/test/unit/common/compressors/test_reference_output.py
index b55630aa9..6f8acdecd 100644
--- a/test/unit/common/compressors/test_reference_output.py
+++ b/test/unit/common/compressors/test_reference_output.py
@@ -34,8 +34,8 @@ def _patch_composer(composer, fp_inputs):
composer.block_forward = mock_forward
composer.block_quantizer.quantize_block = MagicMock()
composer.block_quantizer.enable_quanted_input = False
- composer._get_fp_act_hooks = MagicMock(return_value=[])
- composer._get_q_act_hooks = MagicMock(return_value=[])
+ composer._get_fp_act_hooks = MagicMock(return_value=([], {"act_max": "max"}))
+ composer._get_q_act_hooks = MagicMock(return_value=([], {"act_max": "max"}))
composer.preprocessors = []
return mock_forward
@@ -127,8 +127,8 @@ def test_weight_changing_preprocessor_recomputes_seeded_reference():
composer.block_forward = BlockForwardRunner(batch_size=1, amp=False, enable_torch_compile=False)
composer.block_quantizer.quantize_block = MagicMock()
composer.block_quantizer.enable_quanted_input = False
- composer._get_fp_act_hooks = MagicMock(return_value=[])
- composer._get_q_act_hooks = MagicMock(return_value=[])
+ composer._get_fp_act_hooks = MagicMock(return_value=([], None))
+ composer._get_q_act_hooks = MagicMock(return_value=([], None))
composer.preprocessors = [_WeightChangingPreprocessor()]
_, reference_next_input = composer.compress_block(
diff --git a/test/unit/test_cpu/algorithms/test_qad_warm_start.py b/test/unit/test_cpu/algorithms/test_qad_warm_start.py
index 18bc46ead..48ab754bc 100644
--- a/test/unit/test_cpu/algorithms/test_qad_warm_start.py
+++ b/test/unit/test_cpu/algorithms/test_qad_warm_start.py
@@ -128,7 +128,7 @@ def test_warmup_propagates_best_quantized_output(monkeypatch, wrapper_cls, outpu
expected_outputs = []
calls = []
- def forward(module, samples, others):
+ def forward(module, samples, others, cache_device=None):
calls.append(module[0])
result = [module(sample) for sample in samples]
forward.last_output_dict = {"hidden_states": result} if output_dict else None
@@ -163,8 +163,8 @@ def test_warmup_propagates_best_quantized_output(monkeypatch, wrapper_cls, outpu
composer.preprocessors = []
composer._rotation_members = []
composer.scheme = None
- composer._get_fp_act_hooks = lambda block: []
- composer._get_q_act_hooks = lambda block: []
+ composer._get_fp_act_hooks = lambda block: ([], None)
+ composer._get_q_act_hooks = lambda block: ([], None)
monkeypatch.setattr("auto_round.algorithms.composer.clear_memory", lambda: None)
ctx = BlockContext(model=block, block_names=["0"], block_name="0", block_index=0)
quantized, reference = composer.compress_block(block, inputs, {}, ctx)
diff --git a/test/unit/test_cpu/quantization/test_ddp_core.py b/test/unit/test_cpu/quantization/test_ddp_core.py
new file mode 100644
index 000000000..b7123ab95
--- /dev/null
+++ b/test/unit/test_cpu/quantization/test_ddp_core.py
@@ -0,0 +1,2220 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""CPU-tier tests for the ported single-process DDP tuning surface.
+
+Covers the device-agnostic contract: shard-constrained samplers, plan
+resolution with injected free-VRAM data, transport encoding math, pool
+distribution layout, gather-on-CPU paths, env defaults, and the
+block-runner device-safe concatenation. Mirror/replica behavior on real
+CUDA devices lives in the CUDA tier (test_ddp_mirror.py + e2e).
+"""
+
+import logging
+
+import pytest
+import torch
+
+from auto_round.algorithms.block_runner import _cat_device_safe
+from auto_round.algorithms.parallel.data_parallel import (
+ _transport_segment,
+ distribute_pool,
+ parallel_state,
+ resolve_ddp_plan,
+ resolve_parallel_world,
+)
+
+
+def _with_policy(q, world=2):
+ """Attach a ParallelPolicy-bearing compress_context to a fake quantizer
+ (the policy rides on the compress context; env vars are gone)."""
+ from types import SimpleNamespace
+
+ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy
+
+ q.compress_context = SimpleNamespace(parallel_policy=ParallelPolicy(world=world))
+ return q
+
+
+class TestResolveDDPPlan:
+ def _plan(self, world, free, footprint=100):
+ return resolve_ddp_plan(
+ world,
+ torch.device("cuda", 0), # device OBJECTS only -- no CUDA runtime touched
+ 8,
+ visible_devices=[0, 1, 2, 3],
+ vram_free_bytes={torch.device("cuda", i): free[i] for i in range(4)},
+ mirror_footprint_bytes=footprint,
+ margin_bytes=0, # toy byte values in these tests
+ )
+
+ def test_full_world_when_vram_fits(self):
+ plan = self._plan(4, [1000, 1000, 1000, 1000])
+ assert plan.enabled and plan.world == 4
+ assert plan.shard_size == 2
+
+ def test_device_without_mirror_fit_is_kept_with_warning_note(self):
+ plan = self._plan(4, [1000, 10, 1000, 1000]) # cuda:1 estimate too small
+ assert plan.enabled
+ assert torch.device("cuda", 1) in plan.devices
+ assert plan.world == 4 # the VRAM check is advisory: the requested world stays
+ assert any("low free VRAM on" in n and "device kept" in n for n in plan.notes)
+
+ def test_all_devices_low_vram_still_engages_full_world(self):
+ plan = self._plan(4, [5, 5, 5, 5], footprint=100)
+ assert plan.enabled and plan.world == 4
+ assert sum(1 for n in plan.notes if "low free VRAM on" in n) == 3 # mirrors only, not home
+
+ def test_batch_smaller_than_world_demotes_to_pow2(self):
+ from auto_round.algorithms.parallel.data_parallel import resolve_ddp_plan as _rdp
+
+ def _resolve(batch, world):
+ return _rdp(
+ world,
+ torch.device("cuda", 0),
+ batch,
+ visible_devices=[0, 1, 2, 3],
+ vram_free_bytes={torch.device("cuda", i): 1 << 40 for i in range(4)},
+ mirror_footprint_bytes=1,
+ margin_bytes=0,
+ )
+
+ # batch 1 cannot shard: serial (world collapses to 1; the reason
+ # goes to the log)
+ p = _resolve(1, 4)
+ assert p.world == 1
+ # batch 3, world 4 -> halved to 2 (3 >= 2, one replica gets 2 samples)
+ p = _resolve(3, 4)
+ assert p.world == 2 and any("world set to 2" in str(n) for n in p.notes)
+ # batch 6, world 4 -> no demotion (6 >= 4)
+ p = _resolve(6, 4)
+ assert p.world == 4
+ # batch divisible by world stays
+ p = _resolve(8, 4)
+ assert p.world == 4
+
+ def test_batch_demotion_is_exact_when_pow2_floor_off(self):
+ from auto_round.algorithms.parallel.data_parallel import resolve_ddp_plan as _rdp
+
+ def _resolve(batch, world):
+ return _rdp(
+ world,
+ torch.device("cuda", 0),
+ batch,
+ visible_devices=[0, 1, 2, 3],
+ vram_free_bytes={torch.device("cuda", i): 1 << 40 for i in range(4)},
+ mirror_footprint_bytes=1,
+ margin_bytes=0,
+ pow2_floor=False,
+ )
+
+ # iters=0 worlds carry no pow2 constraint: the demotion is exact
+ p = _resolve(3, 4)
+ assert p.world == 3 and any("world set to 3" in str(n) for n in p.notes)
+ p = _resolve(6, 3)
+ assert p.world == 3 # no demotion
+ p = _resolve(1, 4)
+ assert p.world == 1 # serial: a replica needs at least one sample
+
+ def test_non_power_of_two_request_resolves_but_caller_gates(self):
+ # resolve itself is policy-free: a non-pow2 world with enough
+ # visible devices resolves enabled; the engagement resolver and the
+ # CLI policy entry enforce the pow2 rule only for iters > 0 (the
+ # gradient exchange needs it; iters=0 runs have no exchange)
+ plan = resolve_ddp_plan(
+ 3,
+ torch.device("cuda", 0),
+ 12,
+ visible_devices=[0, 1, 2],
+ vram_free_bytes={torch.device("cuda", i): 1 << 30 for i in range(3)},
+ mirror_footprint_bytes=1,
+ margin_bytes=0,
+ )
+ assert plan.world == 3 and plan.enabled
+
+
+class TestResolveParallelWorld:
+ """Policy entry: off|auto|N resolution (the CLI delegates here)."""
+
+ def test_off_returns_one(self):
+ assert resolve_parallel_world("off", 8) == 1
+ assert resolve_parallel_world(None, 8) == 1
+ assert resolve_parallel_world("", 8) == 1
+
+ def test_auto_takes_largest_pow2_fitting_visible_at_iters_above_zero(self):
+ assert resolve_parallel_world("auto", 8, iters=10) == 8
+ assert resolve_parallel_world("auto", 6, iters=10) == 4
+ assert resolve_parallel_world("auto", 5, iters=10) == 4
+
+ def test_auto_takes_all_visible_at_iters_zero(self):
+ # no gradient exchange runs at iters=0, so every visible device works
+ assert resolve_parallel_world("auto", 6, iters=0) == 6
+ assert resolve_parallel_world("auto", 8, iters=0) == 8
+
+ def test_auto_needs_two_visible(self):
+ with pytest.raises(RuntimeError, match="at least 2 visible CUDA devices"):
+ resolve_parallel_world("auto", 1)
+
+ def test_explicit_valid_pow2(self):
+ assert resolve_parallel_world("2", 4) == 2
+ assert resolve_parallel_world("4", 4) == 4
+
+ def test_explicit_non_pow2_rejected_at_iters_above_zero(self):
+ with pytest.raises(RuntimeError, match="power of two"):
+ resolve_parallel_world("6", 8, iters=10)
+
+ def test_explicit_non_pow2_allowed_at_iters_zero(self):
+ assert resolve_parallel_world("6", 8, iters=0) == 6
+ assert resolve_parallel_world("3", 3, iters=0) == 3
+
+ def test_explicit_below_two_rejected(self):
+ with pytest.raises(RuntimeError, match="at least 2"):
+ resolve_parallel_world("1", 8)
+
+ def test_explicit_over_visible_fails_loudly(self):
+ with pytest.raises(RuntimeError, match="requires at least 8 visible CUDA devices, got 4"):
+ resolve_parallel_world("8", 4)
+
+ def test_garbage_rejected(self):
+ with pytest.raises(RuntimeError, match=r"accepts off\|auto\|N"):
+ resolve_parallel_world("yes", 8)
+
+
+class TestVisibleCountShortfall:
+ def test_visible_fewer_than_requested_world_reports_shortfall(self):
+ plan = resolve_ddp_plan(
+ 4,
+ torch.device("cuda", 0),
+ 8,
+ visible_devices=[0, 1], # home + one other: fewer than the requested world
+ vram_free_bytes=None,
+ mirror_footprint_bytes=None,
+ )
+ assert not plan.enabled
+ assert any("fewer than the requested world" in n for n in plan.notes)
+
+
+class TestTransportMath:
+ def test_fp32_passthrough(self):
+ t = torch.randn(16)
+ out = _transport_segment(t, t.device)
+ assert out is t
+
+ def test_cross_device_copy_is_dtype_preserving(self):
+ t = torch.randn(16) if not torch.cuda.is_available() else torch.randn(16, device="cuda:0")
+ out = _transport_segment(t, torch.device("cpu"))
+ assert out.dtype == t.dtype and out.shape == t.shape
+
+
+class TestDistributePool:
+ def test_plan_layout_covers_pool_exactly(self):
+ # distribute_pool gives device r the contiguous range [r*shard,(r+1)*shard);
+ # on uniform CPU devices that is a no-op, so pin the layout arithmetic
+ # through the plan the pool follows
+ from auto_round.algorithms.parallel.data_parallel import resolve_ddp_plan
+
+ plan = resolve_ddp_plan(
+ 4,
+ torch.device("cuda", 0),
+ 8,
+ visible_devices=[0, 1, 2, 3], # avoids torch.cuda.device_count() on CUDA-less hosts
+ vram_free_bytes={torch.device("cuda", i): 1 << 40 for i in range(4)},
+ mirror_footprint_bytes=1,
+ margin_bytes=0,
+ )
+ assert plan.world == 4
+ assert plan.shard_size * plan.world == 8
+ # device objects in the plan are cuda:N (unusable on CUDA-less hosts);
+ # exercise the scatter loop with the degenerate all-cpu device list
+ distribute_pool([torch.zeros(2) for _ in range(8)], [torch.device("cpu")] * 4)
+ # indivisible / too-small pools are left alone by contract (serial cats handle them)
+ distribute_pool([torch.zeros(2)], [torch.device("cpu")] * 4)
+
+
+class TestSingleDevicePlacement:
+ """Parallel tuning tunes whole-block mirrors: a block whose weights span
+ several CUDA devices fails the resolver (validation, never a gather)."""
+
+ @staticmethod
+ def _param(dev, n=8):
+ from types import SimpleNamespace
+
+ return SimpleNamespace(device=torch.device(dev), numel=lambda: n, element_size=lambda: 4)
+
+ @staticmethod
+ def _quantizer():
+ from types import SimpleNamespace
+
+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)
+ q._get_scaler = lambda: None
+ return q
+
+ def test_block_spanning_two_cuda_devices_declines(self, monkeypatch):
+ from types import SimpleNamespace
+
+ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_
+
+ block = SimpleNamespace(parameters=lambda: iter([self._param("cuda:0"), self._param("cuda:1")]))
+ with pytest.raises(RuntimeError, match="spans 2 CUDA devices"):
+ resolve_tune_ddp_plan_(_with_policy(self._quantizer()), block, [torch.zeros(1)], None, "cuda:0")
+
+ def test_non_cuda_accelerator_home_is_eligible(self, monkeypatch):
+ """cuda/xpu/hpu homes are eligible; a fake xpu home with explicit
+ devices resolves a plan on CPU (no xpu runtime touched)."""
+ from auto_round.algorithms.parallel import data_parallel as dp
+ from auto_round.algorithms.parallel.data_parallel import resolve_ddp_plan
+
+ free_map = dp._accel_free_bytes_map("xpu") # None on a box without xpu
+ block = SimpleNamespace(
+ parameters=lambda: iter(
+ [SimpleNamespace(device=torch.device("xpu", 0), numel=lambda: 8, element_size=lambda: 4)]
+ ),
+ modules=lambda: iter([]),
+ )
+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)
+ q._get_scaler = lambda: None
+ # plan resolution itself works for an xpu home with visible devices
+ plan = resolve_ddp_plan(
+ 2,
+ torch.device("xpu", 0),
+ 2,
+ visible_devices=[0, 1],
+ vram_free_bytes=free_map,
+ mirror_footprint_bytes=1,
+ )
+ assert plan.enabled and plan.world == 2
+ assert all(d.type == "xpu" for d in plan.devices)
+ # and the full resolver reports (decline, without raising) only on
+ # real blockers:
+ # an xpu home on a box without xpu resolves without the CUDA-only error
+ try:
+ resolved = dp.resolve_tune_ddp_plan_(_with_policy(q), block, [torch.zeros(1)], None, "xpu:0")
+ except RuntimeError as e:
+ assert "not a supported accelerator" not in str(e)
+ resolved = None
+ if resolved is not None:
+ assert resolved.world >= 1
+
+ def test_cpu_resident_weights_pass_span_rule(self, monkeypatch):
+ """CPU-pinned subtrees alongside the CUDA home are legal placement."""
+ from types import SimpleNamespace
+
+ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_
+
+ block = SimpleNamespace(
+ parameters=lambda: iter([self._param("cpu"), self._param("cuda:0")]),
+ modules=list,
+ )
+ # the span rule passes; the resolve then proceeds (it may decline
+ # later on the free-VRAM probe, which is CPU-safe) -- the assertion
+ # checks the decline list for the placement reason
+ try:
+ resolve_tune_ddp_plan_(_with_policy(self._quantizer()), block, [torch.zeros(1)], None, "cuda:0")
+ except RuntimeError as e:
+ assert "spans" not in str(e)
+
+ def test_iters0_engages_non_pow2_world(self, monkeypatch):
+ """A world=3 plan engages when the tune loop is empty (iters=0):
+ no gradient exchange runs, so the halving-doubling pow2 rule lifts."""
+ from auto_round.algorithms.parallel import data_parallel as dp
+
+ q = SimpleNamespace(iters=0, gradient_accumulate_steps=1, enable_lfq=False)
+ q._get_scaler = lambda: None
+ _with_policy(q, world=3)
+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 3)
+ monkeypatch.setattr(
+ dp,
+ "_accel_free_bytes_map",
+ lambda _t: {torch.device("cuda", i): 1 << 40 for i in range(3)},
+ )
+ block = SimpleNamespace(
+ parameters=lambda: iter(
+ [SimpleNamespace(device=torch.device("cuda", 0), numel=lambda: 8, element_size=lambda: 4)]
+ ),
+ modules=lambda: iter([]),
+ )
+ plan = dp.resolve_tune_ddp_plan_(q, block, [torch.zeros(1)] * 3, None, "cuda:0", log=False)
+ assert plan.enabled and plan.world == 3
+
+ def test_iters_above_zero_declines_non_pow2_world(self, monkeypatch):
+ """The same world=3 with a live tune loop raises: the gradient
+ exchange (recursive halving-doubling) needs a power-of-two world."""
+ from auto_round.algorithms.parallel import data_parallel as dp
+
+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)
+ q._get_scaler = lambda: None
+ _with_policy(q, world=3)
+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 3)
+ monkeypatch.setattr(
+ dp,
+ "_accel_free_bytes_map",
+ lambda _t: {torch.device("cuda", i): 1 << 40 for i in range(3)},
+ )
+ block = SimpleNamespace(
+ parameters=lambda: iter(
+ [SimpleNamespace(device=torch.device("cuda", 0), numel=lambda: 8, element_size=lambda: 4)]
+ ),
+ modules=lambda: iter([]),
+ )
+ with pytest.raises(RuntimeError, match="power of two"):
+ dp.resolve_tune_ddp_plan_(q, block, [torch.zeros(1)] * 3, None, "cuda:0", log=False)
+
+ def test_ddp_engagement_raises_dynamo_cache_limit_by_world(self, monkeypatch):
+ """Engaging a parallel plan bumps the dynamo cache limit to base x world."""
+ from auto_round.algorithms.parallel import data_parallel as dp
+ from auto_round.utils import device as _dev
+
+ calls = []
+ monkeypatch.setattr(_dev, "_bump_dynamo_cache_limit", lambda n=None: calls.append(n))
+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)
+ q._get_scaler = lambda: None
+ _with_policy(q)
+ # fake a two-device visible set (this host has at most one CUDA device)
+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 2)
+ monkeypatch.setattr(
+ dp, "_accel_free_bytes_map", lambda _t: {torch.device("cuda", 0): 1 << 40, torch.device("cuda", 1): 1 << 40}
+ )
+ block = SimpleNamespace(
+ parameters=lambda: iter(
+ [SimpleNamespace(device=torch.device("cuda", 0), numel=lambda: 8, element_size=lambda: 4)]
+ ),
+ modules=lambda: iter([]),
+ )
+ plan = dp.resolve_tune_ddp_plan_(q, block, [torch.zeros(1)] * 2, None, "cuda:0", log=False)
+ assert plan.world == 2
+ assert calls and calls[-1] == 16 * 2 # default env base 16 x engaged world
+
+ def test_accumulation_engages(self, monkeypatch):
+ """gradient_accumulate_steps > 1 no longer gates the lane."""
+ from auto_round.algorithms.parallel import data_parallel as dp
+ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_
+
+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=2, enable_lfq=False)
+ q._get_scaler = lambda: None
+ _with_policy(q)
+ # fake a two-device visible set (this host has at most one CUDA device)
+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 2)
+ monkeypatch.setattr(
+ dp, "_accel_free_bytes_map", lambda _t: {torch.device("cuda", 0): 1 << 40, torch.device("cuda", 1): 1 << 40}
+ )
+ block = SimpleNamespace(
+ parameters=lambda: iter(
+ [SimpleNamespace(device=torch.device("cuda", 0), numel=lambda: 8, element_size=lambda: 4)]
+ ),
+ modules=lambda: iter([]),
+ )
+ plan = resolve_tune_ddp_plan_(q, block, [torch.zeros(1)] * 2, None, "cuda:0", log=False)
+ # engages with a requested world; the accumulation gate is gone
+ assert plan.world == 2
+ assert not any("accumulate" in str(n) for n in plan.notes)
+
+ def test_accumulated_grad_sign_matches_serial(self):
+ """Two replicas, sum-reduced shard losses; the full-value exchange
+ leaves each replica with the averaged gradient -- a positive rescale
+ of the serial accumulated (summed) gradient, so sign-SGD updates and
+ momentum directions match serial."""
+ from auto_round.algorithms.parallel import data_parallel as dp
+
+ world = 2
+ # replicas hold the SAME parameters (mirrors); each accumulates the
+ # sum-reduced gradient over its own data shard. Shard grads derived
+ # analytically: grad of (x[shard]**2).sum() lives on the shard slice.
+ x = torch.randn(16, requires_grad=False)
+ grad0 = torch.zeros(16)
+ grad0[:8] = 2 * x[:8]
+ grad1 = torch.zeros(16)
+ grad1[8:] = 2 * x[8:]
+ bufs = [grad0.clone(), grad1.clone()]
+ dp.halving_doubling_allreduce(bufs, scale=1.0 / world)
+ serial_accumulated = 2 * x # serial: grads summed over both shards
+ for buf in bufs:
+ # exchanged buffer x world == serial accumulated gradient
+ assert torch.allclose(buf * world, serial_accumulated, atol=1e-6)
+
+
+class TestCatDeviceSafe:
+ def test_same_device_is_plain_cat(self):
+ parts = [torch.zeros(2, 3), torch.ones(2, 3)]
+ out = _cat_device_safe(parts, dim=0)
+ assert torch.equal(out, torch.cat(parts, dim=0))
+
+ def test_empty_selection_raises(self):
+ with pytest.raises(ValueError):
+ _cat_device_safe([], dim=0)
+
+
+class TestPolicyDefaults:
+ """The policy rides on the compress context; a quantizer without one is
+ serial (off), and the resolver reads the policy's world."""
+
+ def test_quantizer_without_context_defaults_to_off(self):
+ from types import SimpleNamespace
+
+ from auto_round.algorithms.parallel.data_parallel import _quantizer_policy
+
+ assert _quantizer_policy(SimpleNamespace()).world == 1
+
+ def test_policy_world_round_trip(self):
+ from types import SimpleNamespace
+
+ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy, _quantizer_policy
+
+ q = _with_policy(SimpleNamespace(), world=4)
+ assert _quantizer_policy(q).world == 4
+ assert isinstance(_quantizer_policy(q), ParallelPolicy)
+
+ def test_declared_lfq_constraint_declines(self):
+ """Eligibility constraints are declared by the quantizer
+ (parallel_constraints); LFQ is a real blocker."""
+ from types import SimpleNamespace
+
+ import torch
+
+ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_
+
+ q = _with_policy(SimpleNamespace(iters=10, gradient_accumulate_steps=1))
+ q.parallel_constraints = lambda: {"enable_lfq": True}
+ block = torch.nn.Sequential(torch.nn.Linear(4, 4))
+ with pytest.raises(RuntimeError, match="enable_lfq"):
+ resolve_tune_ddp_plan_(q, block, [torch.zeros(1)], None, "cpu")
+
+ def test_base_quantizer_declares_constraints(self):
+ """BaseQuantizer.parallel_constraints reports LFQ without the resolver
+ introspecting quantizer internals; a grad scaler is no constraint."""
+ from auto_round.algorithms.quantization.base import BaseQuantizer
+
+ q = BaseQuantizer.__new__(BaseQuantizer)
+ assert q.parallel_constraints() == {"enable_lfq": False}
+ q.enable_lfq = True
+ assert q.parallel_constraints()["enable_lfq"] is True
+
+
+class TestTupleKwargSlicing:
+ """transformers-v5 rope arrives as position_embeddings=(cos, sin) with a
+ per-sample batch dim; shard/batch forwards smaller than the cached batch
+ must slice tuple-of-tensors kwargs elementwise or crash in rope."""
+
+ def _runner(self, n=8, batch_size=4):
+ from auto_round.algorithms.block_runner import BlockForwardRunner
+
+ inputs = [torch.randn(1, 6, 4) for _ in range(n)]
+ return (
+ BlockForwardRunner(
+ batch_dim=0, batch_size=batch_size, device=torch.device("cpu"), cache_device="cpu", amp=False
+ ),
+ inputs,
+ )
+
+ def test_tuple_kwargs_sliced_to_batch_indices(self):
+ runner, inputs = self._runner()
+ cos = torch.randn(8, 6, 2)
+ sin = torch.randn(8, 6, 2)
+ _, others = runner.select_batch(inputs, {"position_embeddings": (cos, sin)}, [3, 5])
+ assert isinstance(others["position_embeddings"], tuple)
+ assert others["position_embeddings"][0].shape == (2, 6, 2)
+ assert others["position_embeddings"][1].shape == (2, 6, 2)
+ assert torch.equal(others["position_embeddings"][0][0], cos[3])
+ assert torch.equal(others["position_embeddings"][0][1], cos[5])
+
+ def test_broadcast_tuple_elements_pass_through(self):
+ runner, inputs = self._runner()
+ table = torch.randn(1, 6, 2) # broadcast-shaped: not per-sample
+ _, others = runner.select_batch(inputs, {"position_embeddings": (table, table)}, [3, 5])
+ assert others["position_embeddings"][0] is table # unsliced, broadcastable
+
+ def test_mixed_and_non_tensor_tuples_untouched(self):
+ runner, inputs = self._runner()
+ val = (torch.randn(8, 6, 2), "flag")
+ _, others = runner.select_batch(inputs, {"weird": val}, [3, 5])
+ assert others["weird"] is val # mixed tuple treated as opaque
+
+
+class TestSharedCacheShardPick:
+ """shared_cache_keys kwargs arrive as ONE ENTRY PER BATCH (list[n_pool/batch]).
+ A sub-batch draw (DDP shard) must pick the owning batch's entry and slice its
+ within-batch rows; full-batch draws keep the legacy pick exactly."""
+
+ def _runner(self, batch_size=8):
+ from auto_round.algorithms.block_runner import BlockForwardRunner
+
+ inputs = [torch.randn(1, 6, 4) for _ in range(128)]
+ return (
+ BlockForwardRunner(
+ batch_dim=0,
+ batch_size=batch_size,
+ device=torch.device("cpu"),
+ cache_device="cpu",
+ amp=False,
+ shared_cache_keys=("position_embeddings",),
+ ),
+ inputs,
+ )
+
+ def _pe(self):
+ # 16 batch entries, each an (cos, sin) tuple of [8, S, D] per-sample tensors
+ return [(torch.full((8, 6, 2), float(b)), torch.full((8, 6, 2), -float(b))) for b in range(16)]
+
+ def test_tensor_and_list_indices_pick_identically(self):
+ # ``indices`` normally ride on the inputs' device (a CUDA tensor on GPU
+ # runs); the host-int hoist must yield the same picks as a plain list
+ runner, inputs = self._runner()
+ pe = self._pe()
+ t_in, t_other = runner.select_batch(inputs, {"position_embeddings": pe}, torch.tensor([32, 33]))
+ l_in, l_other = runner.select_batch(inputs, {"position_embeddings": pe}, [32, 33])
+ assert torch.equal(t_in, l_in)
+ assert torch.equal(t_other["position_embeddings"][0], l_other["position_embeddings"][0])
+ assert torch.equal(t_other["position_embeddings"][1], l_other["position_embeddings"][1])
+
+ def test_shard_picks_owning_batch_and_rows(self):
+ runner, inputs = self._runner()
+ pe = self._pe()
+ _, others = runner.select_batch(inputs, {"position_embeddings": pe}, [32, 33])
+ cos, sin = others["position_embeddings"]
+ assert cos.shape == (2, 6, 2) and sin.shape == (2, 6, 2)
+ assert torch.all(cos == 4.0) and torch.all(sin == -4.0) # entry 4, rows 0-1
+
+ def test_shard_wrapping_rows_across_batch_boundary_fails_safe(self):
+ runner, inputs = self._runner()
+ pe = self._pe()
+ _, others = runner.select_batch(inputs, {"position_embeddings": pe}, [7, 8])
+ # batch 0 entry (indices[0]=7), rows 7 and 0 -- within-entry slice
+ cos, _ = others["position_embeddings"]
+ assert cos.shape == (2, 6, 2) and torch.all(cos == 0.0)
+
+ def test_full_batch_draw_keeps_legacy_pick(self):
+ runner, inputs = self._runner()
+ pe = self._pe()
+ _, others = runner.select_batch(inputs, {"position_embeddings": pe}, list(range(8, 16)))
+ # legacy: multi-index full-batch -> val[0], untouched (batch 0's entry, 8 rows)
+ cos, _ = others["position_embeddings"]
+ assert isinstance(cos, torch.Tensor) and cos.shape[0] == 8 and torch.all(cos == 0.0)
+
+ def test_single_index_sub_batch_draw_per_batch_list(self):
+ runner, inputs = self._runner()
+ pe = self._pe()
+ _, others = runner.select_batch(inputs, {"position_embeddings": pe}, [5])
+ # per-batch list: sample 5 -> batch 0 entry, row 5
+ cos, _ = others["position_embeddings"]
+ assert cos.shape == (1, 6, 2) and torch.all(cos == 0.0)
+
+ def test_per_sample_list_sub_batch_draw(self):
+ runner, inputs = self._runner()
+ # per-sample layout: 128 entries, each a [1, 6, 2] tensor
+ ps = [torch.full((1, 6, 2), float(i)) for i in range(128)]
+ _, others = runner.select_batch(inputs, {"position_embeddings": ps}, [32, 33])
+ assert others["position_embeddings"].shape == (2, 6, 2)
+ assert torch.all(others["position_embeddings"][0] == 32.0)
+
+ @pytest.mark.parametrize("rank_batch", [1, 2])
+ def test_rank_runner_with_its_batch_share(self, rank_batch):
+ # a thread rank tunes with its share of the batch (8 // world), while
+ # the cache was collected 8 per entry: the collection batch comes from
+ # the cache, not from the rank runner's batch_size (regression: at 8
+ # ranks a 1-row draw got a whole 8-row entry)
+ runner, inputs = self._runner(batch_size=rank_batch)
+ pe = self._pe()
+ draw = [37, 38][:rank_batch]
+ _, others = runner.select_batch(inputs, {"position_embeddings": pe}, draw)
+ cos, sin = others["position_embeddings"]
+ assert cos.shape == (rank_batch, 6, 2) and torch.all(cos == 4.0) and torch.all(sin == -4.0)
+
+ def test_broadcast_entry_passes_through(self):
+ # [1, S, D] tables broadcast over the batch: never row-sliced (on CUDA an
+ # out-of-range index_select is a device-side assert, not an exception)
+ runner, inputs = self._runner(batch_size=1)
+ pe = [(torch.full((1, 6, 2), float(b)), torch.full((1, 6, 2), -float(b))) for b in range(16)]
+ _, others = runner.select_batch(inputs, {"position_embeddings": pe}, [37])
+ cos, _ = others["position_embeddings"]
+ assert cos.shape == (1, 6, 2) and torch.all(cos == 4.0)
+
+
+class TestLoggingGlobals:
+ def test_engaged_log_globals_initialized(self):
+ """Regression: the env-strip once dropped the module-level
+ ``_ENGAGED_LOGGED_SIG`` init, and the ``global`` read in
+ resolve_tune_ddp_plan_ then NameError'd -- but only on GPU runs,
+ because a CPU home declines before the sig block (CPU tests cannot
+ reach the engaged path)."""
+ from auto_round.algorithms.parallel import data_parallel as dp
+ from auto_round.algorithms.quantization.search_dispatch import _ENGAGED_LOGGED
+
+ assert dp._ENGAGED_LOGGED_SIG is None
+ # _coll_mirror_setup_logged was removed with its once-per-process
+ # INFO line; search_dispatch._ENGAGED_LOGGED remains the registry
+ assert isinstance(_ENGAGED_LOGGED, set)
+
+
+class TestActivationPricing:
+ """Mirror pricing charges the per-forward activation working set on top
+ of the weights (estimate_tuning_block_mem's SIX-value return contract).
+
+ Regression: the rebase onto main left the DDP seam unpacking the
+ estimator's pre-main FOUR-tuple, so every block logged ``activation
+ pricing skipped (too many values to unpack)`` at INFO and the plan
+ budget silently dropped to weights-only."""
+
+ @staticmethod
+ def _block():
+ layer = torch.nn.Linear(32, 32, bias=False)
+ layer.global_name = "m.blk.proj"
+ layer.to_quantized = True
+ layer.bits = 4
+ layer.act_bits = 16
+ return torch.nn.Sequential(layer)
+
+ @staticmethod
+ def _quantizer():
+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)
+ q._get_scaler = lambda: None
+ return _with_policy(q)
+
+ def test_pricing_succeeds_and_adds_bytes_to_the_budget(self, monkeypatch, caplog, _autoround_log_propagate):
+ """The resolver's pricing call runs (no skip warning) and its bytes
+ enter the mirror budget: cuda:1 fits the weights-only mirror (plus
+ the 2GiB margin) but misses once the activation charge (>= the flat
+ 1GiB ``additional_memory``) lands on top."""
+ from auto_round.algorithms.parallel import data_parallel as dp
+
+ block = self._block()
+ weight_bytes = sum(p.numel() * p.element_size() for p in block.parameters())
+ free = {
+ torch.device("cuda", 0): 1 << 40,
+ # weights + margin + 64MiB slack: passes weights-only pricing,
+ # fails once >=1GiB activation bytes are charged
+ torch.device("cuda", 1): weight_bytes + (2 << 30) + (64 << 20),
+ }
+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 2)
+ monkeypatch.setattr(dp, "_accel_free_bytes_map", lambda _t: free)
+ inputs = [torch.randn(1, 16, 32) for _ in range(2)]
+ with caplog.at_level("WARNING"):
+ plan = dp.resolve_tune_ddp_plan_(self._quantizer(), block, inputs, None, "cuda:0", log=False)
+ assert plan.enabled and plan.world == 2
+ assert not any("activation pricing skipped" in r.message for r in caplog.records)
+ assert any(
+ "low free VRAM on cuda:1" in n for n in plan.notes
+ ), "activation bytes never entered the budget (weights-only pricing)"
+
+ def test_estimator_failure_is_a_warning(self, monkeypatch, caplog, _autoround_log_propagate):
+ """A failed estimate is a degraded state (weights-only budget): the
+ skip line must be WARNING-level, not buried at INFO."""
+ from auto_round.algorithms.parallel import data_parallel as dp
+ from auto_round.utils import device as _dev
+
+ def _boom(*_a, **_k):
+ raise RuntimeError("estimator exploded")
+
+ monkeypatch.setattr(_dev, "estimate_tuning_block_mem", _boom)
+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 2)
+ monkeypatch.setattr(
+ dp, "_accel_free_bytes_map", lambda _t: {torch.device("cuda", i): 1 << 40 for i in range(2)}
+ )
+ inputs = [torch.randn(1, 16, 32) for _ in range(2)]
+ with caplog.at_level("WARNING"):
+ plan = dp.resolve_tune_ddp_plan_(self._quantizer(), self._block(), inputs, None, "cuda:0", log=False)
+ assert plan.enabled and plan.world == 2
+ assert any("activation pricing skipped" in r.message and r.levelno == logging.WARNING for r in caplog.records)
+
+
+class TestEarlyDeviceMapEligibility:
+ """F3: the multi-device-map ineligibility fires at policy resolution
+ (before model load) with the same message the runtime resolver raises at
+ block 0; the runtime check remains the second line of defense."""
+
+ def test_multi_device_map_with_live_loop_raises_before_load(self):
+ from auto_round.algorithms.parallel.data_parallel import check_parallel_device_map_compat_
+
+ with pytest.raises(RuntimeError, match="spans 2 CUDA devices; parallel tuning requires"):
+ check_parallel_device_map_compat_(4, {"a": "cuda:0", "b": "cuda:1"}, iters=10)
+
+ def test_message_lists_the_fixes(self):
+ from auto_round.algorithms.parallel.data_parallel import check_parallel_device_map_compat_
+
+ with pytest.raises(RuntimeError, match="--data_parallel off"):
+ check_parallel_device_map_compat_(4, {"a": "cuda:0", "b": "cuda:1"}, iters=10)
+
+ def test_single_device_map_passes(self):
+ from auto_round.algorithms.parallel.data_parallel import check_parallel_device_map_compat_
+
+ check_parallel_device_map_compat_(4, {"a": "cuda:0"}, iters=10) # one device
+ check_parallel_device_map_compat_(4, "0", iters=10) # CLI default
+ check_parallel_device_map_compat_(4, None, iters=10) # no map
+
+ def test_iters0_and_world1_are_not_cli_blocked(self):
+ from auto_round.algorithms.parallel.data_parallel import check_parallel_device_map_compat_
+
+ multi = {"a": "cuda:0", "b": "cuda:1"}
+ check_parallel_device_map_compat_(4, multi, iters=0) # no tune loop: runtime decides
+ check_parallel_device_map_compat_(1, multi, iters=10) # lane off
+
+ def test_cpu_entries_do_not_count(self):
+ from auto_round.algorithms.parallel.data_parallel import check_parallel_device_map_compat_
+
+ check_parallel_device_map_compat_(4, {"a": "cpu", "b": "cpu"}, iters=10)
+
+ def test_cli_policy_resolution_calls_the_early_check(self):
+ """Wiring: the CLI calls the early check right after resolving the
+ world, before the model loads."""
+ import inspect
+
+ from auto_round.cli import main as cli_main
+
+ src = inspect.getsource(cli_main)
+ assert "check_parallel_device_map_compat_" in src
+ assert src.index("check_parallel_device_map_compat_(") > src.index("resolve_parallel_world(")
+
+ def test_runtime_span_check_still_exists(self):
+ """Second line of defense: the resolver's per-block span check."""
+ from types import SimpleNamespace
+
+ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_
+
+ block = SimpleNamespace(
+ parameters=lambda: iter(
+ [
+ SimpleNamespace(device=torch.device("cuda:0"), numel=lambda: 8, element_size=lambda: 4),
+ SimpleNamespace(device=torch.device("cuda:1"), numel=lambda: 8, element_size=lambda: 4),
+ ]
+ )
+ )
+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)
+ q._get_scaler = lambda: None
+ with pytest.raises(RuntimeError, match="spans 2 CUDA devices"):
+ resolve_tune_ddp_plan_(_with_policy(q), block, [torch.zeros(1)], None, "cuda:0")
+
+
+class TestZeroShotParallelNotice:
+ """F4: an explicit --data_parallel N on the zero-shot RTN path
+ (no calibration forward) emits exactly ONE INFO line; auto/off stay
+ silent; nothing refuses to run."""
+
+ def test_explicit_request_logs_one_info_line(self, caplog, _autoround_log_propagate):
+ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy, log_zero_shot_parallel_noop_
+
+ with caplog.at_level("INFO"):
+ log_zero_shot_parallel_noop_(ParallelPolicy(world=4, source="explicit"))
+ infos = [r for r in caplog.records if "has no effect on this run" in r.message]
+ assert len(infos) == 1 and infos[0].levelno == logging.INFO
+ assert "--data_parallel 4" in infos[0].message
+ assert "zero-shot RTN needs no calibration forward" in infos[0].message
+ assert "quantizing serially" in infos[0].message
+
+ def test_auto_and_off_are_silent(self, caplog, _autoround_log_propagate):
+ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy, log_zero_shot_parallel_noop_
+
+ with caplog.at_level("INFO"):
+ log_zero_shot_parallel_noop_(ParallelPolicy(world=8, source="auto"))
+ log_zero_shot_parallel_noop_(ParallelPolicy(world=1, source="off"))
+ log_zero_shot_parallel_noop_(None)
+ assert not any("has no effect" in r.message for r in caplog.records)
+
+ def test_zero_shot_path_wires_the_notice(self):
+ """Wiring: _quantize_zero_shot consults the compress-context policy."""
+ import inspect
+
+ from auto_round.compressors import orchestrator
+
+ src = inspect.getsource(orchestrator.CompressionOrchestrator._quantize_zero_shot)
+ assert "log_zero_shot_parallel_noop_" in src
+
+
+class TestPeakVramReplicaDevices:
+ """F5: the final peak-VRAM summary must cover DDP replica devices engaged
+ outside the device map (the monitor's callers pass the device-map list)."""
+
+ def test_registered_devices_are_sampled_beyond_the_caller_list(self, monkeypatch, _memory_monitor_clean):
+ from auto_round.utils import device as dev_mod
+
+ class _FakeMgr:
+ type = "cuda"
+
+ @staticmethod
+ def is_available():
+ return True
+
+ @staticmethod
+ def memory_reserved(i):
+ return (i + 1) * 1024**3 # cuda:i reports (i+1) GiB reserved
+
+ mm = _memory_monitor_clean
+ mm.register_extra_devices([torch.device("cuda", 3)])
+ monkeypatch.setattr(dev_mod, "get_current_device_manager", lambda: _FakeMgr())
+ mm.update(["cuda:0"]) # the device-map list, WITHOUT the replica device
+ assert "0" in mm.peak_vram and mm.peak_vram["0"] == 1.0
+ assert "3" in mm.peak_vram and mm.peak_vram["3"] == 4.0 # replica now visible
+ assert "cuda:3" in mm.get_summary() or "'3': 4.0GB" in mm.get_summary()
+
+ def test_registration_survives_reset_and_cpu_entries_are_ignored(self, _memory_monitor_clean):
+ mm = _memory_monitor_clean
+ mm.register_extra_devices([torch.device("cuda", 2), torch.device("cpu"), None])
+ assert mm._extra_devices == {"cuda:2"}
+ mm.reset()
+ assert mm._extra_devices == {"cuda:2"} # peaks reset, registration kept
+
+ def test_engaging_plan_registers_its_devices(self, monkeypatch, _memory_monitor_clean):
+ from auto_round.algorithms.parallel import data_parallel as dp
+
+ mm = _memory_monitor_clean
+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 2)
+ monkeypatch.setattr(
+ dp, "_accel_free_bytes_map", lambda _t: {torch.device("cuda", i): 1 << 40 for i in range(2)}
+ )
+ block = SimpleNamespace(
+ parameters=lambda: iter(
+ [SimpleNamespace(device=torch.device("cuda", 0), numel=lambda: 8, element_size=lambda: 4)]
+ ),
+ modules=lambda: iter([]),
+ )
+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)
+ q._get_scaler = lambda: None
+ _with_policy(q)
+ plan = dp.resolve_tune_ddp_plan_(q, block, [torch.zeros(1)] * 2, None, "cuda:0", log=False)
+ assert plan.enabled
+ assert mm._extra_devices == {"cuda:0", "cuda:1"}
+
+ def test_serial_plan_registers_nothing(self, _memory_monitor_clean):
+ from auto_round.algorithms.parallel.data_parallel import resolve_ddp_plan
+
+ assert resolve_ddp_plan(1, torch.device("cuda", 0), 8).world == 1
+ assert _memory_monitor_clean._extra_devices == set()
+
+
+class TestPerfPrepSpan:
+ """F6: the per-block phases line must account for the region between the
+ reload mark and the first tick (materialize + dtype convert +
+ dispatch_block placement + pool-placement attach) -- ~8s/block on big
+ MoE blocks that tqdm counted but total= missed."""
+
+ def test_prep_span_opens_before_attach_and_prints(self):
+ import inspect
+
+ from auto_round.compressors import orchestrator
+
+ src = inspect.getsource(orchestrator.CompressionOrchestrator._quantize_blocks)
+ first_prep = src.index('_perf.tick("prep")')
+ first_attach = src.index('_perf.tick("attach")')
+ assert first_prep < first_attach, "the measured chain must open with the prep span"
+ assert "prep=" in src
+
+ def test_each_tick_opens_the_region_it_names(self):
+ """tick(key) closes the running span and opens ``key``: each tick must
+ sit right before the code it names (a tick placed after its region
+ shifts every printed label one phase late)."""
+ import inspect
+
+ from auto_round.compressors import orchestrator
+
+ src = inspect.getsource(orchestrator.CompressionOrchestrator._quantize_blocks)
+ chain = [
+ ('_perf.tick("prep")', "materialize_model_(m)"),
+ ('_perf.tick("attach")', "self._attach_pool_placement("),
+ ('_perf.tick("compress")', "self.alg_composer.compress_block("),
+ ('_perf.tick("swap")', "if q_input is not None:"),
+ ('_perf.tick("sweep")', "_mm._deferred_search_inputs = None"),
+ ('_perf.tick("hooks")', "accelerate.hooks.remove_hook_from_submodules(m)"),
+ ('_perf.tick("pack")', "if self.compress_context.is_immediate_packing:"),
+ ('_perf.tick("mv")', "mv_module_from_gpu(m)"),
+ ('_perf.tick("clear")', "clear_memory(device_list=device_manager.device_list)"),
+ ('_perf.tick("monitor")', "memory_monitor.log_summary()"),
+ ('_perf.tick("write")', "_bg_writer.dispatch(_bg_write)"),
+ ]
+ pos = 0
+ for tick, region in chain:
+ t = src.index(tick, pos)
+ r = src.index(region, t)
+ assert (
+ src.find("_perf.tick(", t + len(tick)) > r or src.find("_perf.tick(", t + len(tick)) == -1
+ ), f"{tick} must directly precede {region!r}"
+ pos = r
+
+
+class TestEngagedLineBatchMode:
+ """F7: the engaged line logs the batch mode and the effective global batch."""
+
+ def test_engaged_line_reports_batch_mode_and_effective_batch(self, monkeypatch, caplog, _autoround_log_propagate):
+ """F7: the engaged line logs the batch mode and the effective global
+ batch (per_replica scales it by the world)."""
+ from auto_round.algorithms.parallel import data_parallel as dp
+ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy
+
+ monkeypatch.setattr(dp, "_accel_device_count", lambda _t: 2)
+ monkeypatch.setattr(
+ dp, "_accel_free_bytes_map", lambda _t: {torch.device("cuda", i): 1 << 40 for i in range(2)}
+ )
+ block = SimpleNamespace(
+ parameters=lambda: iter(
+ [SimpleNamespace(device=torch.device("cuda", 0), numel=lambda: 8, element_size=lambda: 4)]
+ ),
+ modules=lambda: iter([]),
+ )
+
+ def _q(mode):
+ q = SimpleNamespace(
+ iters=10,
+ gradient_accumulate_steps=1,
+ enable_lfq=False,
+ calibration_context=SimpleNamespace(batch_size=8),
+ )
+ q._get_scaler = lambda: None
+ q.compress_context = SimpleNamespace(parallel_policy=ParallelPolicy(world=2, batch_mode=mode))
+ return q
+
+ with caplog.at_level("INFO"):
+ dp._ENGAGED_LOGGED_SIG = None # the engaged line dedups by signature
+ dp.resolve_tune_ddp_plan_(_q("split"), block, [torch.zeros(1)] * 32, None, "cuda:0", log=True)
+ dp._ENGAGED_LOGGED_SIG = None
+ dp.resolve_tune_ddp_plan_(_q("per_replica"), block, [torch.zeros(1)] * 32, None, "cuda:0", log=True)
+ engaged = [r.message for r in caplog.records if "[tune-ddp] engaged:" in r.message]
+ assert len(engaged) == 2
+ assert "batch=split" in engaged[0] and "effective global batch=8" in engaged[0]
+ assert "batch=per_replica" in engaged[1] and "effective global batch=16" in engaged[1]
+
+
+class TestSyncMirrorsToHome:
+ """iters=0 RTN/OptRTN under DDP: the quantizer runs main's code on the HOME
+ block and never touches the pool; the engine then syncs the quantized home
+ state onto the mirrors (sync_mirrors_to_home, called from the composer's
+ _settle_block_pool) so the Step-6 cascade still runs sharded."""
+
+ @staticmethod
+ def _block():
+ class _L(torch.nn.Linear):
+ def __init__(self, i):
+ super().__init__(4, 4, bias=False)
+ self.global_name = f"m.blk.l{i}"
+ self.to_quantized = True
+ self.bits = 4
+ self.sym = True
+ self.data_type = "int"
+ self.group_size = 4
+ self.act_bits = 16
+ self.imatrix = torch.rand(4) + 0.5
+ self.imatrix_cnt = 1
+
+ class _B(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.l0 = _L(0)
+ self.l1 = _L(1)
+
+ def forward(self, x):
+ return self.l0(x) + self.l1(x)
+
+ return _B()
+
+ @staticmethod
+ def _ddp_quantizer(quantizer_cls, block, with_pool=True):
+ from types import SimpleNamespace
+
+ from auto_round.algorithms.parallel.data_parallel import MirrorPool
+
+ devices = [torch.device("cpu", 0), torch.device("cpu", 1)]
+ q = quantizer_cls.__new__(quantizer_cls)
+ q._BaseAlgorithm__run_ctx = None # unbound: q.model is None (the driver is stubbed)
+ pool = MirrorPool(block, devices) if with_pool else None
+ q.parallel_state = SimpleNamespace(plan=SimpleNamespace(world=2, devices=devices), pool=pool, pool_used=False)
+ return q, pool
+
+ @staticmethod
+ def _qdq(layer):
+ with torch.no_grad(): # a REAL quantize-dequantize transform on the home layer
+ layer.weight.data = ((layer.weight.data * 4).round() / 4) + 0.125
+
+ @staticmethod
+ def _mirror_pairs(block, pool):
+ for n, m in block.named_modules():
+ if hasattr(m, "global_name"):
+ for r in range(pool.world):
+ if pool.reps[r] is not block:
+ yield pool.mirror_layer(r, n), m
+
+ def test_optrtn_quantize_block_stays_on_home(self, monkeypatch):
+ import auto_round.algorithms.quantization.rtn.batched_search as _bs
+ from auto_round.algorithms.quantization.rtn.quantizer import OptimizedRTNQuantizer
+
+ block = self._block()
+ home_ids = {id(m) for _, m in block.named_modules() if hasattr(m, "global_name")}
+ q, pool = self._ddp_quantizer(OptimizedRTNQuantizer, block)
+ staged_on = []
+
+ def _staged(layer, disable_opt_rtn=None, defer_search=False):
+ staged_on.append((id(layer), defer_search))
+ return layer
+
+ def _driver(model, staged, max_batch=None):
+ for _name, layer in staged:
+ self._qdq(layer)
+
+ q._quantize_layer_via_rtn = _staged
+ monkeypatch.setattr(_bs, "run_batched_rtn_search", _driver)
+ q.quantize_block(block, None, {}, None, None, None)
+ # main's staged lane ran on the HOME layers only, and the quantizer
+ # left the pool alone (the sync is the engine's job)
+ assert len(staged_on) == 2
+ assert all(iid in home_ids and deferred for iid, deferred in staged_on)
+ assert q.parallel_state.pool_used is False
+ assert all(not torch.equal(lm.weight.data, hm.weight.data) for lm, hm in self._mirror_pairs(block, pool))
+
+ def test_sync_pushes_home_state_and_marks_pool_consumed(self):
+ from auto_round.algorithms.parallel.mirror_sync import sync_mirrors_to_home
+ from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer
+
+ block = self._block()
+ q, pool = self._ddp_quantizer(RTNQuantizer, block)
+ for _n, m in block.named_modules():
+ if hasattr(m, "global_name"):
+ self._qdq(m)
+ m.scale = torch.full((4, 1), 0.25)
+ assert sync_mirrors_to_home(q, block) is True
+ for lm, hm in self._mirror_pairs(block, pool):
+ assert torch.equal(lm.weight.data, hm.weight.data)
+ assert torch.equal(lm.scale, hm.scale)
+ assert q.parallel_state.pool_used is True
+ assert not any(hasattr(m, "imatrix") for _n, m in block.named_modules() if hasattr(m, "global_name"))
+
+ def test_sync_without_aligned_pool_is_a_noop(self):
+ from auto_round.algorithms.parallel.data_parallel import MirrorPool
+ from auto_round.algorithms.parallel.mirror_sync import sync_mirrors_to_home
+ from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer
+
+ block = self._block()
+ q, _pool = self._ddp_quantizer(RTNQuantizer, block, with_pool=False)
+ assert sync_mirrors_to_home(q, block) is False
+ # a pool built for another block must not be written either
+ other = self._block()
+ q.parallel_state.pool = MirrorPool(other, q.parallel_state.plan.devices)
+ assert sync_mirrors_to_home(q, block) is False
+ assert q.parallel_state.pool_used is False
+ assert all(hasattr(m, "imatrix") for _n, m in block.named_modules() if hasattr(m, "global_name"))
+
+ def test_composer_settle_syncs_rtn_family_and_keeps_pool(self):
+ from auto_round.algorithms.composer import AlgorithmComposer
+ from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer
+
+ block = self._block()
+ q, pool = self._ddp_quantizer(RTNQuantizer, block)
+ for _n, m in block.named_modules():
+ if hasattr(m, "global_name"):
+ self._qdq(m)
+ released = []
+ composer = AlgorithmComposer.__new__(AlgorithmComposer)
+ composer.block_quantizer = q
+ composer._coll_ctx = type("_Ctx", (), {"release_pool": lambda self: released.append(1)})()
+ assert composer._settle_block_pool(block, pool) is pool
+ assert released == []
+ for lm, hm in self._mirror_pairs(block, pool):
+ assert torch.equal(lm.weight.data, hm.weight.data)
+
+ def test_composer_settle_releases_pool_for_other_quantizers(self):
+ from auto_round.algorithms.composer import AlgorithmComposer
+ from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer
+
+ block = self._block()
+ q, pool = self._ddp_quantizer(SignRoundQuantizer, block)
+ released = []
+ composer = AlgorithmComposer.__new__(AlgorithmComposer)
+ composer.block_quantizer = q
+ composer._coll_ctx = type("_Ctx", (), {"release_pool": lambda self: released.append(1)})()
+ # not the RTN family and nothing consumed the pool: no sync, released
+ assert composer._settle_block_pool(block, pool) is None
+ assert released == [1]
+ assert q.parallel_state.pool_used is False
+
+
+class TestRequestedWorldErrors:
+ """A requested parallel world is a requirement: infeasible -> RuntimeError, never silent serial."""
+
+ def _fake_quantizer(self):
+ from types import SimpleNamespace
+
+ q = SimpleNamespace(
+ iters=10,
+ gradient_accumulate_steps=1,
+ enable_lfq=False,
+ )
+ q._get_scaler = lambda: None
+ return q
+
+ def test_infeasible_world_raises(self, monkeypatch):
+ import torch
+
+ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_
+
+ block = torch.nn.Sequential(torch.nn.Linear(4, 4))
+ with pytest.raises(RuntimeError, match="ineligible"):
+ resolve_tune_ddp_plan_(_with_policy(self._fake_quantizer()), block, [torch.zeros(1)], None, "cpu")
+
+ def test_iters0_is_not_a_decline_reason(self, monkeypatch):
+ """The DDP world shards the collection at iters=0 too (campaign
+ semantics restored): with a fake quantizer at iters=0 the only
+ ineligibility reason left must be the non-CUDA home, never iters."""
+ import torch
+
+ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_
+
+ q = self._fake_quantizer()
+ q.iters = 0
+ block = torch.nn.Sequential(torch.nn.Linear(4, 4))
+ with pytest.raises(RuntimeError) as excinfo:
+ resolve_tune_ddp_plan_(_with_policy(q), block, [torch.zeros(1)], None, "cpu")
+ assert "iters" not in str(excinfo.value)
+ assert "not a supported accelerator" in str(excinfo.value)
+
+ def test_no_policy_stays_serial(self, monkeypatch):
+ import torch
+
+ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_
+
+ block = torch.nn.Sequential(torch.nn.Linear(4, 4))
+ plan = resolve_tune_ddp_plan_(self._fake_quantizer(), block, [torch.zeros(1)], None, "cpu")
+ assert plan.world == 1
+
+
+def _memory_monitor_clean():
+ """Isolate the process-singleton memory monitor's peaks and extra-device
+ registration so tests are order-independent."""
+ from auto_round.utils.device import memory_monitor
+
+ saved = (
+ dict(memory_monitor.peak_vram),
+ float(memory_monitor.peak_ram),
+ set(getattr(memory_monitor, "_extra_devices", set())),
+ )
+ memory_monitor.peak_vram = {}
+ memory_monitor.peak_ram = 0.0
+ memory_monitor._extra_devices = set()
+ yield memory_monitor
+ memory_monitor.peak_vram, memory_monitor.peak_ram = saved[0], saved[1]
+ memory_monitor._extra_devices = saved[2]
+
+
+def _autoround_log_propagate():
+ """Temporarily enable propagation on the ``autoround`` logger so pytest's
+ caplog fixture (handler at the root logger) can capture warnings; the
+ logger is configured with propagate=False in production."""
+ import logging
+
+ logger = logging.getLogger("autoround")
+ original = logger.propagate
+ logger.propagate = True
+ yield
+ logger.propagate = original
+
+
+def _expert_layer(name):
+ """An exact-nn.Linear module that satisfies _split_expert_batches' gates."""
+ m = torch.nn.Linear(4, 4)
+ m.global_name = name
+ m.to_quantized = True
+ m.bits = 4
+ m.group_size = 4
+ m.sym = True # sym batches under BOTH neuqi and plain opt-RTN; plain asym has no search
+ m.act_bits = 16
+ return m
+
+
+def _expert_moe_block():
+ """4 same-key experts + one dense module: the standard sharded-search fixture."""
+
+ class _B(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ mods = [
+ _expert_layer("m.blk.experts.0.gate_proj"),
+ _expert_layer("m.blk.experts.1.gate_proj"),
+ _expert_layer("m.blk.experts.2.up_proj"),
+ _expert_layer("m.blk.experts.3.up_proj"),
+ ]
+ dense = _expert_layer("m.blk.mlp.down_proj")
+ for i, m in enumerate(mods + [dense]):
+ setattr(self, f"m{i}", m)
+
+ return _B()
+
+
+class TestMirrorPoolUnit:
+ """MirrorPool construction, hook sync, and release."""
+
+ def _block_with_hook(self):
+ class _B(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lin = torch.nn.Linear(4, 4)
+ self.lin.global_name = "m.lin"
+ self.lin.to_quantized = True
+ self.lin.bits = 4
+ self.lin.group_size = 4
+ self.lin.sym = False
+ self.lin.act_bits = 16
+
+ def forward(self, x):
+ return self.lin(x)
+
+ b = _B()
+ seen = []
+ b.lin.register_forward_hook(lambda mod, inp, out: seen.append(out) or out)
+ b._seen = seen
+ return b
+
+ def test_sync_hooks_copies_and_clears(self):
+ from auto_round.algorithms.parallel.data_parallel import MirrorPool
+
+ block = self._block_with_hook()
+ pool = MirrorPool(block, [torch.device("cpu", 0), torch.device("cpu", 1)])
+ rep = pool.reps[-1]
+ assert rep is not block
+ # deepcopy carried the hook; sync must keep it working...
+ pool.sync_hooks_from(block)
+ x = torch.randn(2, 4)
+ rep(x)
+ assert len(block._seen) == 1
+ # ...and after the home hook is gone, sync drops it from the mirror
+ block.lin._forward_hooks.clear()
+ pool.sync_hooks_from(block)
+ rep(x)
+ assert len(block._seen) == 1, "stale hook must not fire after sync"
+
+ def test_release_drops_mirrors(self):
+ from auto_round.algorithms.parallel.data_parallel import MirrorPool
+
+ block = self._block_with_hook()
+ pool = MirrorPool(block, [torch.device("cpu", 0), torch.device("cpu", 1)])
+ assert pool.world == 2
+ pool.release()
+ assert pool.reps == [] and pool.world == 0
+
+
+class TestComposerPoolHandoff:
+ """Step 3.6: the collection pool is handed to the iters=0 quantizer and
+ released around quantize_block (kept for the cascade at iters=0, dropped
+ before the tune's own mirrors at iters>0, dropped on failure too)."""
+
+ def _parts(self, iters, consumes=True):
+ from auto_round.algorithms.composer import AlgorithmComposer
+ from auto_round.algorithms.quantization.rtn.quantizer import OptimizedRTNQuantizer
+
+ composer = AlgorithmComposer.__new__(AlgorithmComposer)
+ events = []
+
+ class _Ctx:
+ devices = None # no DDP devices -> ensure_pool is a no-op
+
+ def __init__(self, pool):
+ self.pool = pool
+
+ def ensure_pool(self, block):
+ events.append("ensure")
+
+ def mark(self, what):
+ events.append(what)
+
+ def release_pool(self):
+ if self.pool is not None:
+ events.append("release")
+ self.pool = None
+
+ class _Pool:
+ pass
+
+ pool = _Pool()
+
+ q = OptimizedRTNQuantizer.__new__(OptimizedRTNQuantizer)
+ q.iters = iters
+ q.enable_quanted_input = False
+ q._consumes = consumes
+
+ def _qb(*a, **k):
+ # instance-attribute functions stay unbound: close over q directly
+ st = parallel_state(q)
+ seen = st.pool if st is not None else None
+ events.append(("saw_pool", seen))
+ if seen is not None and q._consumes:
+ parallel_state(q, create=True).pool_used = True
+ if q._consumes == "adopt":
+ # adoption transfers ownership (drops the pool ref)
+ parallel_state(q).pool = None
+
+ q.quantize_block = _qb
+ ctx = _Ctx(pool)
+ ctx.devices = [torch.device("cpu", 0), torch.device("cpu", 1)] # DDP on
+ composer._coll_ctx = ctx
+ composer._block = None
+ composer.block_quantizer = q
+ composer.preprocessors = []
+ return composer, ctx, pool, q, events
+
+ def _run_handoff(self, composer):
+ # mirrors the Step 3.6 handoff (pool to the quantizer, ANY consumer:
+ # OptRTN home quantize or SignRound adoption) + the real post-Step-4
+ # _settle_block_pool + the try/finally in compress_block
+ if getattr(composer._coll_ctx, "devices", None) and len(composer._coll_ctx.devices) >= 2:
+ composer._coll_ctx.ensure_pool(composer._block)
+ _pool = getattr(composer._coll_ctx, "pool", None)
+ if _pool is not None:
+ _state = parallel_state(composer.block_quantizer, create=True)
+ _state.pool = _pool
+ _state.pool_used = False
+ try:
+ composer.block_quantizer.quantize_block(None)
+ _pool = composer._settle_block_pool(composer._block, _pool)
+ composer._coll_ctx.mark("step5+6")
+ finally:
+ _state = parallel_state(composer.block_quantizer)
+ if _state is not None:
+ _state.pool = None
+ composer._coll_ctx.release_pool()
+
+ def test_iters0_pool_handed_then_kept_for_cascade(self):
+ # RTN-lane consumption: attr KEPT + used -> the pool survives into
+ # step5+6 (its sync made the mirrors exactly-home for the cascade)
+ composer, ctx, pool, q, events = self._parts(iters=0, consumes=True)
+ self._run_handoff(composer)
+ assert events == ["ensure", ("saw_pool", pool), "step5+6", "release"]
+ assert parallel_state(q).pool is None
+
+ def test_iters0_unconsumed_pool_released_before_cascade(self):
+ composer, ctx, pool, q, events = self._parts(iters=0, consumes=False)
+ self._run_handoff(composer)
+ # quantizer left unconsumed (e.g. plan vanished): released right
+ # after quantize_block, before the cascade step (the finally release
+ # is then a no-op on None)
+ assert events == ["ensure", ("saw_pool", pool), "release", "step5+6"]
+ assert ctx.pool is None
+
+ def test_iters_positive_pool_adopted_by_tune(self):
+ composer, ctx, pool, q, events = self._parts(iters=10, consumes="adopt")
+ self._run_handoff(composer)
+ # the tune adopts the pool (engage_ wraps mirrors in place) -> kept
+ # through the tune, released once at the very end
+ assert events == ["ensure", ("saw_pool", pool), "release", "step5+6"]
+
+ def test_failure_still_releases_pool(self):
+ composer, ctx, pool, q, events = self._parts(iters=0)
+
+ def _boom(*a, **k):
+ raise RuntimeError("quantize failed")
+
+ q.quantize_block = _boom
+ try:
+ self._run_handoff(composer)
+ raised = False
+ except RuntimeError:
+ raised = True
+ assert raised
+ assert parallel_state(q).pool is None
+ assert ctx.pool is None
+
+
+class TestRotationHookPoolInteraction:
+ """Real Hadamard input-location hooks x MirrorPool: the online rotation
+ rides on the mirrors (deepcopy + sync_hooks_from) and mirror forwards are
+ bit-identical to the home block; removing the home hook and re-syncing
+ drops it from the mirrors (no stale rotation)."""
+
+ def test_hadamard_input_hook_parity_and_resync(self):
+ from auto_round.algorithms.parallel.data_parallel import MirrorPool
+ from auto_round.algorithms.transforms import RotationConfig
+ from auto_round.algorithms.transforms.hadamard.apply import _apply_input_transform
+
+ class _B(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lin = torch.nn.Linear(32, 32)
+
+ def forward(self, x):
+ return self.lin(x)
+
+ b = _B()
+ _apply_input_transform(b.lin, RotationConfig(hadamard_type="hadamard"), data_type="int")
+ pool = MirrorPool(b, [torch.device("cpu", 0), torch.device("cpu", 1)])
+ rep = pool.reps[-1]
+ assert rep is not b
+ assert len(rep.lin._forward_pre_hooks) == 1
+ x = torch.randn(2, 32)
+ with torch.no_grad():
+ y_home = b(x)
+ y_rep = rep(x)
+ assert torch.equal(y_home, y_rep), "mirror forward must match home under the rotation hook"
+ # stale-hook hygiene: removing the home hook + re-sync drops it
+ b.lin._forward_pre_hooks.clear()
+ pool.sync_hooks_from(b)
+ assert len(rep.lin._forward_pre_hooks) == 0
+ with torch.no_grad():
+ y_rep2 = rep(x)
+ assert not torch.equal(y_home, y_rep2), "resync must drop the removed rotation hook"
+
+
+class TestHomeQuantizeThenCascade:
+ """End-to-end seam: the quantizer quantizes the HOME block, the engine
+ syncs the mirrors to it, and the cascade forward on the SAME pool outputs
+ exactly the serial forward of the quantized home block."""
+
+ def test_cascade_on_pool_matches_serial_home(self, monkeypatch):
+ from types import SimpleNamespace
+
+ from auto_round.algorithms.parallel.data_parallel import (
+ MirrorPool,
+ sharded_nograd_forward,
+ )
+ from auto_round.algorithms.parallel.mirror_sync import sync_mirrors_to_home
+ from auto_round.algorithms.quantization.rtn.quantizer import OptimizedRTNQuantizer
+
+ torch.manual_seed(0)
+
+ class _B(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ for i in range(4):
+ setattr(self, f"l{i}", _expert_layer(f"m.blk.l{i}"))
+
+ def forward(self, x):
+ for i in range(4):
+ x = getattr(self, f"l{i}")(x)
+ return x
+
+ block = _B() # weights stay RANDOM: FP and quantized forwards differ
+ originals = {n: m.weight.data.clone() for n, m in block.named_modules() if hasattr(m, "global_name")}
+ devices = [torch.device("cpu", 0), torch.device("cpu", 1)]
+ pool = MirrorPool(block, devices)
+
+ q = OptimizedRTNQuantizer.__new__(OptimizedRTNQuantizer)
+ q._BaseAlgorithm__run_ctx = None # unbound: q.model is None (the driver is stubbed)
+ q.parallel_state = SimpleNamespace(plan=SimpleNamespace(world=2, devices=devices), pool=pool, pool_used=False)
+ import auto_round.algorithms.quantization.rtn.batched_search as _bs
+
+ def _staged(layer, disable_opt_rtn=None, defer_search=False):
+ return layer
+
+ def _driver(model, staged, max_batch=None):
+ # a REAL quantize-dequantize: round-to-grid + offset, so an
+ # FP (unsynced) mirror layer would produce different outputs
+ with torch.no_grad():
+ for _name, layer in staged:
+ layer.weight.data = ((layer.weight.data * 4).round() / 4) + 0.125
+
+ q._quantize_layer_via_rtn = _staged
+ monkeypatch.setattr(_bs, "run_batched_rtn_search", _driver)
+ q.quantize_block(block, None, {}, None, None, None)
+ assert sync_mirrors_to_home(q, block) is True
+
+ # every home layer carries the EXACT quantize transform
+ for n, m in block.named_modules():
+ if hasattr(m, "global_name"):
+ assert torch.equal(m.weight.data, ((originals[n] * 4).round() / 4) + 0.125)
+
+ inputs = [torch.randn(1, 4) for _ in range(4)] # n=4, world=2 -> shards
+
+ def _runner(rep, inp, oth, idxs, cache_device=None):
+ out = []
+ with torch.no_grad():
+ for i in idxs:
+ out.append(rep(inp[i]))
+ return torch.cat(out, dim=0)
+
+ # mirror layers must carry the exact home state (the post-quantize
+ # sync): FP mirrors would diverge here
+ for r in range(pool.world):
+ if pool.reps[r] is block:
+ continue
+ for name, m in block.named_modules():
+ if hasattr(m, "global_name"):
+ assert torch.equal(pool.mirror_layer(r, name).weight.data, m.weight.data)
+ outs = sharded_nograd_forward(_runner, block, inputs, {}, torch.device("cpu"), devices, pool=pool)
+ with torch.no_grad():
+ serial = [block(t) for t in inputs]
+ for a, b in zip(outs, serial):
+ assert torch.equal(a, b), "cascade on quantized mirrors must match the serial home forward"
+ pool.release()
+
+
+class TestMirrorPoolMultiPass:
+ """The pool must SURVIVE its passes: sharded_nograd_forward's teardown
+ once cleared the aliased reps list, silently disabling every later reuse
+ (searches, cascade) while still pinning the mirror VRAM."""
+
+ def test_two_pool_passes_keep_the_pool_alive(self):
+ import torch as _t
+
+ from auto_round.algorithms.parallel.data_parallel import (
+ MirrorPool,
+ sharded_nograd_forward,
+ )
+
+ class _B(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lin = torch.nn.Linear(4, 4)
+
+ def forward(self, x):
+ return self.lin(x)
+
+ block = _B()
+ devices = [torch.device("cpu", 0), torch.device("cpu", 1)]
+ pool = MirrorPool(block, devices)
+ n_rep_objects = [id(r) for r in pool.reps]
+ inputs = [_t.randn(1, 4) for _ in range(4)]
+
+ def _runner(rep, inp, oth, idxs, cache_device=None):
+ out = []
+ with _t.no_grad():
+ for i in idxs:
+ out.append(rep(inp[i]))
+ return _t.cat(out, dim=0)
+
+ for pass_idx in range(2):
+ outs = sharded_nograd_forward(_runner, block, inputs, {}, torch.device("cpu"), devices, pool=pool)
+ assert len(outs) == 4
+ # the pool still holds its replicas after the pass teardown
+ assert pool.world == 2, f"pass {pass_idx}: pool lost its reps"
+ assert [id(r) for r in pool.reps] == n_rep_objects, "replica objects were rebuilt"
+ pool.release()
+ assert pool.world == 0
+
+
+class TestMergeMirrorStats:
+ """_merge_mirror_stats must keep scalar stats co-traveling with their tensor
+ sibling: imatrix_cnt (a python int) was silently dropped, so a cold expert
+ (routed rows on mirror shards, zero on home's) merged its imatrix home
+ while cnt went missing -- the normalize step then crashed with
+ AttributeError: no attribute 'imatrix_cnt'."""
+
+ _STATS = {"imatrix": "sum", "imatrix_cnt": "sum", "act_max": "max"}
+
+ def test_cold_expert_cnt_cotravels_with_imatrix(self):
+ from auto_round.algorithms.parallel.data_parallel import _merge_mirror_stats
+
+ home = torch.nn.Linear(4, 4)
+ mirror = torch.nn.Linear(4, 4)
+ mirror.imatrix = torch.ones(4)
+ mirror.imatrix_cnt = 7 # python int, exactly as both collectors write it
+ _merge_mirror_stats(home, [mirror], self._STATS)
+ assert hasattr(home, "imatrix") and torch.equal(home.imatrix, torch.ones(4))
+ assert getattr(home, "imatrix_cnt", None) == 7
+
+ def test_cnt_sums_across_mirrors(self):
+ from auto_round.algorithms.parallel.data_parallel import _merge_mirror_stats
+
+ home = torch.nn.Linear(4, 4)
+ home.imatrix = torch.zeros(4)
+ home.imatrix_cnt = 2
+ m1, m2 = torch.nn.Linear(4, 4), torch.nn.Linear(4, 4)
+ m1.imatrix_cnt, m2.imatrix_cnt = 3, 5
+ m1.imatrix = torch.ones(4)
+ m2.imatrix = torch.ones(4)
+ _merge_mirror_stats(home, [m1, m2], self._STATS)
+ assert home.imatrix_cnt == 10
+ assert torch.equal(home.imatrix, torch.full((4,), 2.0))
+
+ def test_tensor_cnt_also_merges(self):
+ from auto_round.algorithms.parallel.data_parallel import _merge_mirror_stats
+
+ home = torch.nn.Linear(4, 4)
+ mirror = torch.nn.Linear(4, 4)
+ mirror.imatrix = torch.ones(4)
+ mirror.imatrix_cnt = torch.tensor(7)
+ _merge_mirror_stats(home, [mirror], self._STATS)
+ assert home.imatrix_cnt == 7
+
+ def test_non_numeric_attrs_ignored(self):
+ from auto_round.algorithms.parallel.data_parallel import _merge_mirror_stats
+
+ home = torch.nn.Linear(4, 4)
+ mirror = torch.nn.Linear(4, 4)
+ mirror.imatrix_cnt = "bogus"
+ _merge_mirror_stats(home, [mirror], self._STATS)
+ assert not hasattr(home, "imatrix_cnt")
+
+ def test_mirror_values_are_removed_once_folded(self):
+ from auto_round.algorithms.parallel.data_parallel import _merge_mirror_stats
+
+ home = torch.nn.Linear(4, 4)
+ mirror = torch.nn.Linear(4, 4)
+ mirror.imatrix, mirror.imatrix_cnt, mirror.act_max = torch.ones(4), 3, torch.ones(4)
+ _merge_mirror_stats(home, [mirror], self._STATS)
+ _merge_mirror_stats(home, [mirror], self._STATS) # a later pass finds nothing left to add
+ assert torch.equal(home.imatrix, torch.ones(4)) and home.imatrix_cnt == 3
+ assert not any(hasattr(mirror, a) for a in ("imatrix", "imatrix_cnt", "act_max"))
+
+ def test_hook_pass_then_hookless_pass_matches_serial(self):
+ """Pool mirrors live across a block's collection passes: a sharded hook
+ pass followed by any other sharded pass must leave the serial totals
+ (the second merge used to add the mirrors' share again)."""
+ from auto_round.algorithms.parallel.data_parallel import MirrorPool, sharded_nograd_forward
+
+ class _B(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lin = torch.nn.Linear(4, 4)
+
+ def forward(self, x):
+ return self.lin(x)
+
+ def _imatrix_hook(module, inp, _out):
+ sq = (inp[0].reshape(-1, inp[0].shape[-1]).float() ** 2).sum(0)
+ if hasattr(module, "imatrix"):
+ module.imatrix += sq
+ else:
+ module.imatrix = sq
+
+ def _runner(rep, inp, _oth, idxs, cache_device=None):
+ with torch.no_grad():
+ return torch.cat([rep(inp[i]) for i in idxs], dim=0)
+
+ torch.manual_seed(0)
+ block = _B()
+ devices = [torch.device("cpu", i) for i in range(2)]
+ pool = MirrorPool(block, devices)
+ inputs = [torch.randn(1, 4) for _ in range(8)]
+ serial = sum((t.reshape(-1, 4) ** 2).sum(0) for t in inputs)
+ stats = {"imatrix": "sum"}
+ handle = block.lin.register_forward_hook(_imatrix_hook)
+ sharded_nograd_forward(_runner, block, inputs, {}, torch.device("cpu"), devices, merge_stats=stats, pool=pool)
+ handle.remove()
+ assert torch.allclose(block.lin.imatrix, serial)
+ sharded_nograd_forward(_runner, block, inputs, {}, torch.device("cpu"), devices, merge_stats=stats, pool=pool)
+ assert torch.allclose(block.lin.imatrix, serial)
+ pool.release()
+
+
+class TestCalibrationStatsContract:
+ """A block quantizer declares the statistics its calibration hooks write
+ (``calibration_stats``: attr -> "sum" | "max"); the engine folds exactly
+ those from the mirrors, keeps undeclared hook passes serial, and warns
+ when a sharded pass changes an undeclared attribute on the mirrors."""
+
+ def test_merge_map_is_engine_act_max_plus_declaration(self):
+ from types import SimpleNamespace
+
+ from auto_round.algorithms.parallel.data_parallel import calibration_merge_stats
+
+ q = SimpleNamespace(calibration_stats={"hess": "sum"})
+ assert calibration_merge_stats(q, ["h"]) == {"act_max": "max", "hess": "sum"}
+
+ def test_undeclared_hooks_keep_the_pass_serial(self):
+ from types import SimpleNamespace
+
+ from auto_round.algorithms.parallel.data_parallel import calibration_merge_stats
+
+ assert calibration_merge_stats(SimpleNamespace(calibration_stats=None), ["h"]) is None
+ # no quantizer hooks: nothing to declare, the engine's act_max still folds
+ assert calibration_merge_stats(SimpleNamespace(calibration_stats=None), []) == {"act_max": "max"}
+
+ def test_unknown_reduction_is_rejected(self):
+ from types import SimpleNamespace
+
+ from auto_round.algorithms.parallel.data_parallel import calibration_merge_stats
+
+ with pytest.raises(ValueError, match="supported shard reductions"):
+ calibration_merge_stats(SimpleNamespace(calibration_stats={"hess": "mean"}), ["h"])
+
+ def test_imatrix_quantizers_declare_their_stats(self):
+ from auto_round.algorithms.quantization.base import BaseQuantizer
+ from auto_round.algorithms.quantization.rtn.quantizer import OptimizedRTNQuantizer
+ from auto_round.algorithms.quantization.sign_roundv2.quantizer import SignRoundV2Quantizer
+
+ assert BaseQuantizer.calibration_stats is None
+ assert OptimizedRTNQuantizer.calibration_stats == {"imatrix": "sum", "imatrix_cnt": "sum"}
+ assert SignRoundV2Quantizer.calibration_stats == {"imatrix": "sum"}
+
+ @staticmethod
+ def _sharded_hook_pass(hook, stats):
+ from auto_round.algorithms.parallel.data_parallel import MirrorPool, sharded_nograd_forward
+
+ class _B(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lin = torch.nn.Linear(4, 4)
+
+ def forward(self, x):
+ return self.lin(x)
+
+ def _runner(rep, inp, _oth, idxs, cache_device=None):
+ with torch.no_grad():
+ return torch.cat([rep(inp[i]) for i in idxs], dim=0)
+
+ block = _B()
+ devices = [torch.device("cpu", i) for i in range(2)]
+ pool = MirrorPool(block, devices)
+ handle = block.lin.register_forward_hook(hook)
+ inputs = [torch.randn(1, 4) for _ in range(8)]
+ sharded_nograd_forward(_runner, block, inputs, {}, torch.device("cpu"), devices, merge_stats=stats, pool=pool)
+ handle.remove()
+ pool.release()
+ return block
+
+ def test_undeclared_mirror_write_warns(self, caplog, _autoround_log_propagate):
+ def _hook(module, inp, _out):
+ module.f9_undeclared_hess = inp[0].sum()
+
+ with caplog.at_level("WARNING"):
+ self._sharded_hook_pass(_hook, {"act_max": "max"})
+ msgs = [r.message for r in caplog.records if "not a declared calibration statistic" in r.message]
+ assert msgs and "f9_undeclared_hess" in msgs[0]
+
+ def test_bookkeeping_list_change_does_not_warn(self, caplog, _autoround_log_propagate):
+ # a hook that clears its own handle list (the MoE shape recorder in
+ # tune_memory does this) is bookkeeping, not a statistic
+ def _hook(module, _inp, _out):
+ module.f9_rec_handles_ = []
+
+ with caplog.at_level("WARNING"):
+ self._sharded_hook_pass(_hook, {"act_max": "max"})
+ assert not [r for r in caplog.records if "not a declared calibration statistic" in r.message]
+
+ def test_undeclared_tensor_list_append_warns(self, caplog, _autoround_log_propagate):
+ def _hook(module, inp, _out):
+ module.__dict__.setdefault("f9_rows", []).append(inp[0].sum())
+
+ with caplog.at_level("WARNING"):
+ self._sharded_hook_pass(_hook, {"act_max": "max"})
+ msgs = [r.message for r in caplog.records if "not a declared calibration statistic" in r.message]
+ assert msgs and "f9_rows" in msgs[0]
+
+ def test_declared_mirror_write_is_folded_silently(self, caplog, _autoround_log_propagate):
+ def _hook(module, inp, _out):
+ sq = (inp[0].reshape(-1, 4) ** 2).sum(0)
+ if hasattr(module, "f9_declared_stat"):
+ module.f9_declared_stat += sq
+ else:
+ module.f9_declared_stat = sq
+
+ with caplog.at_level("WARNING"):
+ block = self._sharded_hook_pass(_hook, {"act_max": "max", "f9_declared_stat": "sum"})
+ assert not [r for r in caplog.records if "not a declared calibration statistic" in r.message]
+ assert hasattr(block.lin, "f9_declared_stat")
+
+
+class TestCappedPassReusesPool:
+ """A capped hook pass (AR_TUNE_DDP_MAX_COLLECT_FORWARD_DEVICES) must REUSE
+ the full-width pool's first K replicas, skipping ephemeral mirrors (the
+ old behavior paid a per-pass block deepcopy plus a late full pool
+ rebuild per block)."""
+
+ def test_capped_pass_runs_subset_of_pool_replicas(self):
+ import torch as _t
+
+ from auto_round.algorithms.parallel.data_parallel import (
+ MirrorPool,
+ sharded_nograd_forward,
+ )
+
+ class _B(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lin = torch.nn.Linear(4, 4)
+
+ def forward(self, x):
+ return self.lin(x)
+
+ block = _B()
+ devices = [torch.device("cpu", i) for i in range(4)]
+ pool = MirrorPool(block, devices)
+ rep_ids = [id(r) for r in pool.reps]
+ inputs = [_t.randn(1, 4) for _ in range(8)]
+ ran = []
+
+ def _runner(rep, inp, oth, idxs, cache_device=None):
+ ran.append(id(rep))
+ out = []
+ with _t.no_grad():
+ for i in idxs:
+ out.append(rep(inp[i]))
+ return _t.cat(out, dim=0)
+
+ outs = sharded_nograd_forward(
+ _runner, block, inputs, {}, torch.device("cpu"), devices, pool=pool, max_devices=2
+ )
+ with torch.no_grad():
+ serial = [block(t) for t in inputs]
+ for a, b in zip(outs, serial):
+ assert _t.equal(a, b), "capped pool pass must match the serial forward"
+ # only the first K replicas ran, and they are the pool's own objects
+ assert set(ran) == set(rep_ids[:2]), f"capped pass must run pool.reps[:2], ran={ran}"
+ # the pool keeps its full width and identity set afterwards
+ assert pool.world == 4
+ assert [id(r) for r in pool.reps] == rep_ids, "replica objects were rebuilt"
+ pool.release()
+
+ def test_full_width_pass_with_pool_unchanged(self):
+ import torch as _t
+
+ from auto_round.algorithms.parallel.data_parallel import (
+ MirrorPool,
+ sharded_nograd_forward,
+ )
+
+ class _B(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lin = torch.nn.Linear(4, 4)
+
+ def forward(self, x):
+ return self.lin(x)
+
+ block = _B()
+ devices = [torch.device("cpu", i) for i in range(2)]
+ pool = MirrorPool(block, devices)
+ rep_ids = [id(r) for r in pool.reps]
+ inputs = [_t.randn(1, 4) for _ in range(4)]
+
+ def _runner(rep, inp, oth, idxs, cache_device=None):
+ out = []
+ with _t.no_grad():
+ for i in idxs:
+ out.append(rep(inp[i]))
+ return _t.cat(out, dim=0)
+
+ outs = sharded_nograd_forward(_runner, block, inputs, {}, torch.device("cpu"), devices, pool=pool)
+ assert len(outs) == 4
+ assert [id(r) for r in pool.reps] == rep_ids
+ pool.release()
+
+
+class TestEnsurePool:
+ """The pool is block-wide: built even when no collection pass was
+ eligible (n % world != 0 -> serial forwards), because the searches and
+ the tune consume resident mirrors regardless of forward eligibility."""
+
+ def test_ensure_pool_builds_once(self):
+ from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext
+
+ class _B(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lin = torch.nn.Linear(4, 4)
+
+ ctx = TuneParallelContext()
+ ctx.devices = [torch.device("cpu", 0), torch.device("cpu", 1)]
+ assert ctx.pool is None
+ blk = _B()
+ ctx.ensure_pool(blk)
+ first = ctx.pool
+ assert first is not None and first.world == 2
+ ctx.ensure_pool(blk) # idempotent: same pool object
+ assert ctx.pool is first
+
+ def test_ensure_pool_noop_without_devices(self):
+ from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext
+
+ ctx = TuneParallelContext()
+ ctx.devices = None
+ ctx.ensure_pool(object())
+ assert ctx.pool is None
+
+
+class TestContiguousShardSplit:
+ """The ceil/floor split: ANY sample count shards (a remainder lands on
+ an earlier replica); fewer samples than devices shrinks the pass world
+ instead of running serial; pools are withheld when the pass narrows."""
+
+ def test_bounds_remainder_on_earlier_shards(self):
+ from auto_round.algorithms.parallel.data_parallel import contiguous_shard_bounds
+
+ assert contiguous_shard_bounds(129, 4) == [0, 33, 65, 97, 129] # 33/32/32/32
+ assert contiguous_shard_bounds(128, 4) == [0, 32, 64, 96, 128]
+ assert contiguous_shard_bounds(3, 4) == [0, 1, 2, 3, 3] # caller clamps world first
+
+ def test_uneven_n_shards_instead_of_serial(self):
+ import torch as _t
+
+ from auto_round.algorithms.parallel.data_parallel import sharded_nograd_forward
+
+ class _B(_t.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lin = _t.nn.Linear(4, 4)
+
+ def forward(self, x):
+ return self.lin(x)
+
+ seen_shards = []
+
+ def _runner(rep, inp, oth, idxs, cache_device=None):
+ seen_shards.append(list(idxs))
+ out = []
+ with _t.no_grad():
+ for i in idxs:
+ out.append(rep(inp[i]))
+ return _t.cat(out, dim=0)
+
+ devices = [_t.device("cpu", 0), _t.device("cpu", 1)]
+ inputs = [_t.randn(1, 4) for _ in range(5)] # 5 % 2 != 0 -> 3/2 split
+ outs = sharded_nograd_forward(_runner, _B(), inputs, {}, _t.device("cpu"), devices)
+ assert seen_shards == [[0, 1, 2], [3, 4]]
+ assert len(outs) == 5
+
+ def test_fewer_samples_than_devices_shrinks_world(self):
+ import torch as _t
+
+ from auto_round.algorithms.parallel.data_parallel import MirrorPool, sharded_nograd_forward
+
+ class _B(_t.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lin = _t.nn.Linear(4, 4)
+
+ def forward(self, x):
+ return self.lin(x)
+
+ def _runner(rep, inp, oth, idxs, cache_device=None):
+ out = []
+ with _t.no_grad():
+ for i in idxs:
+ out.append(rep(inp[i]))
+ return _t.cat(out, dim=0)
+
+ devices = [_t.device("cpu", 0), _t.device("cpu", 1), _t.device("cpu", 2)]
+ inputs = [_t.randn(1, 4) for _ in range(2)] # n=2 < world=3
+ pool = MirrorPool(_B(), devices) # full-width pool exists
+ outs = sharded_nograd_forward(_runner, _B(), inputs, {}, _t.device("cpu"), devices, pool=pool)
+ assert len(outs) == 2 # sharded on 2 devices, not serial
+ assert pool.world == 3 # pool untouched by the narrowed pass
+
+
+class TestResolverRtnSafe:
+ """The resolver must not assume SignRound-family quantizer methods."""
+
+ def test_quantizer_without_get_scaler_resolves(self, monkeypatch):
+ from types import SimpleNamespace
+
+ import torch
+
+ from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_
+
+ q = SimpleNamespace(iters=0, gradient_accumulate_steps=1, enable_lfq=False)
+ # missing parallel_constraints and calibration_context must still
+ # resolve cleanly
+ with pytest.raises(RuntimeError, match="not a supported accelerator"):
+ resolve_tune_ddp_plan_(
+ _with_policy(q), torch.nn.Sequential(torch.nn.Linear(4, 4)), [torch.zeros(1)], None, "cpu"
+ )
+
+
+class TestCollectionShardingFailVisible:
+ """The collection context must not swallow the resolver's requirement error."""
+
+ def _composer(self):
+ from auto_round.algorithms.composer import AlgorithmComposer
+
+ composer = AlgorithmComposer.__new__(AlgorithmComposer)
+ composer.block_forward = type("R", (), {"output_config": ["hidden_states"]})()
+ composer.block_quantizer = object() # opaque; the resolver is monkeypatched
+ return composer
+
+ def test_requirement_error_propagates(self, monkeypatch):
+ import torch
+
+ import auto_round.algorithms.parallel.tune_parallel as tp
+
+ def _raise(quantizer, block, fp_inputs, fp_outputs, home, world=None, log=True):
+ raise RuntimeError("parallel tuning with world=4 is ineligible: no mirror device")
+
+ monkeypatch.setattr(tp, "resolve_tune_ddp_plan_", _raise)
+ block = torch.nn.Sequential(torch.nn.Linear(4, 4))
+ with pytest.raises(RuntimeError, match="ineligible"):
+ self._composer()._collection_context(block, [torch.zeros(1)])
+
+ def test_reachability_error_warns_and_declines(self, monkeypatch, caplog, _autoround_log_propagate):
+ import torch
+
+ import auto_round.algorithms.parallel.tune_parallel as tp
+
+ def _raise(quantizer, block, fp_inputs, fp_outputs, home, world=None, log=True):
+ raise OSError("boom")
+
+ monkeypatch.setattr(tp, "resolve_tune_ddp_plan_", _raise)
+ block = torch.nn.Sequential(torch.nn.Linear(4, 4))
+ with caplog.at_level("WARNING"):
+ out = self._composer()._collection_context(block, [torch.zeros(1)])
+ assert out.devices is None
+ assert any("resolver unreachable" in r.message for r in caplog.records)
+
+
+import unittest # noqa: E402
+from types import SimpleNamespace # noqa: E402
+
+
+def _stackable_search_fn(weight, bits, imatrix):
+ """Row-independent deterministic search: first element of each leading row."""
+ return weight.reshape(weight.shape[0], -1)[:, 0]
+
+
+class TestSyncPoolDevices:
+ """Phase barriers synchronize every distinct CUDA pool device once (CPU skipped)."""
+
+ def test_syncs_each_cuda_device_once(self):
+ from auto_round.algorithms.parallel.mirror_sync import _sync_pool_devices
+
+ devices = [
+ torch.device("cpu"),
+ torch.device("cuda:0"),
+ torch.device("cuda:1"),
+ torch.device("cuda:0"),
+ ]
+ calls = []
+ orig = torch.cuda.synchronize
+ try:
+ torch.cuda.synchronize = lambda d: calls.append(str(d))
+ elapsed = _sync_pool_devices(devices)
+ finally:
+ torch.cuda.synchronize = orig
+ # one synchronize per distinct cuda device, cpu skipped, elapsed returned
+ assert calls == ["cuda:0", "cuda:1"]
+ assert isinstance(elapsed, float)
+
+
+class TestDistributeSearchStats:
+ """Tune adoption: the merged home stats land on every mirror."""
+
+ def _pool(self, block, world):
+ mirrors = [self._build_mirror(block) for _ in range(world)]
+ reps = [block] + mirrors # home at index 0
+
+ def mirror_layer(r, name):
+ return reps[r].get_submodule(name)
+
+ return SimpleNamespace(world=len(reps), reps=reps, mirror_layer=mirror_layer)
+
+ @staticmethod
+ def _build_mirror(block):
+ import copy as _pcopy
+
+ return _pcopy.deepcopy(block)
+
+ def test_distributes_to_every_mirror(self):
+ import torch.nn as nn
+
+ from auto_round.algorithms.parallel.mirror_sync import _distribute_search_stats
+
+ block = nn.Module()
+ block.l1 = nn.Linear(4, 4, bias=False)
+ block.l1.imatrix = torch.ones(4) * 2
+ pool = self._pool(block, 3)
+ _distribute_search_stats(pool, block, [("l1", block.l1)])
+ for r in (1, 2):
+ assert torch.equal(pool.mirror_layer(r, "l1").imatrix, torch.ones(4) * 2)
+
+
+class TestHomeQuantizeMirrorParity:
+ """REAL end-to-end parity (no spies) on CPU mirrors: under a world=2 plan
+ the quantizer's own quantize_block (main's code) plus the engine's mirror
+ sync must leave the home block AND every mirror bit-identical to a plain
+ serial quantize_block run. Every quantized weight/scale/zp/data_type must
+ be torch.equal."""
+
+ WIDTHS = ((16, 32), (16, 32), (8, 32), (12, 24)) # two same-shape pairs -> real stacking
+
+ def _layer(self, name, out, inn, gen):
+ m = torch.nn.Linear(inn, out, bias=False)
+ with torch.no_grad():
+ m.weight.copy_(torch.randn(out, inn, generator=gen))
+ m.global_name = name
+ m.to_quantized = True
+ m.bits = 4
+ m.group_size = 32
+ m.sym = True
+ m.data_type = "int"
+ m.iters = 0
+ m.act_bits = 16
+ m.scale_dtype = torch.float16
+ m.super_bits = None
+ m.super_group_size = None
+ m.imatrix = torch.rand(inn, generator=gen) + 0.5
+ m.imatrix_cnt = 1 # pre-normalized: quantize_block's divide is identity
+ return m
+
+ def _block(self):
+ import copy as _copy
+
+ gen = torch.Generator().manual_seed(1234)
+
+ blk = torch.nn.Module()
+ for i, (out, inn) in enumerate(self.WIDTHS):
+ blk.add_module(f"l{i}", self._layer(f"m.blk.l{i}", out, inn, gen))
+ return blk, _copy.deepcopy(blk)
+
+ def _model(self, blk):
+ # the tree must resolve the layers' GLOBAL names (m.blk.l<i>): the
+ # driver re-attaches quantized results by that path
+ model = torch.nn.Module()
+ m = torch.nn.Module()
+ m.blk = blk
+ model.m = m
+ return model
+
+ def _quantizer(self, cls, config, model):
+ from types import SimpleNamespace
+
+ q_cls = type(
+ f"_ParityQ_{cls.__name__}",
+ (cls,),
+ {
+ "config": config,
+ "model": model,
+ "model_context": SimpleNamespace(amp_dtype=torch.float32, is_moe_model=False),
+ "compress_context": SimpleNamespace(enable_torch_compile=False),
+ },
+ )
+ return q_cls.__new__(q_cls)
+
+ def _assert_same(self, ref, got, tag):
+ assert torch.equal(ref.weight.data, got.weight.data), f"{tag} weight diverged"
+ assert torch.equal(ref.scale, got.scale), f"{tag} scale diverged"
+ rz, gz = getattr(ref, "zp", None), getattr(got, "zp", None)
+ assert (rz is None) == (gz is None), f"{tag} zp presence diverged"
+ if rz is not None:
+ if isinstance(rz, torch.Tensor):
+ assert torch.equal(rz, gz), f"{tag} zp diverged"
+ else:
+ assert rz == gz, f"{tag} zp diverged"
+ assert getattr(ref, "data_type", None) == getattr(got, "data_type", None), f"{tag} data_type diverged"
+ assert getattr(ref, "q_scale_thresh", None) == getattr(got, "q_scale_thresh", None)
+
+ def _run_parity(self, quantizer_cls, config):
+ from types import SimpleNamespace
+
+ from auto_round.algorithms.parallel.data_parallel import MirrorPool
+ from auto_round.algorithms.parallel.mirror_sync import sync_mirrors_to_home
+
+ blk_s, blk_d = self._block()
+ model_s, model_d = self._model(blk_s), self._model(blk_d)
+ self._quantizer(quantizer_cls, config, model_s).quantize_block(blk_s, None, {}, None, None, None)
+
+ devices = [torch.device("cpu", 0), torch.device("cpu", 1)]
+ pool = MirrorPool(blk_d, devices)
+ q = self._quantizer(quantizer_cls, config, model_d)
+ q.parallel_state = SimpleNamespace(plan=SimpleNamespace(world=2, devices=devices), pool=pool, pool_used=False)
+ q.quantize_block(blk_d, None, {}, None, None, None)
+ assert sync_mirrors_to_home(q, blk_d) is True
+
+ for i in range(len(self.WIDTHS)):
+ ref = model_s.get_submodule(f"m.blk.l{i}")
+ self._assert_same(ref, blk_d.get_submodule(f"l{i}"), f"home l{i}")
+ for r in range(pool.world):
+ if pool.reps[r] is not blk_d:
+ self._assert_same(ref, pool.mirror_layer(r, f"l{i}"), f"mirror{r} l{i}")
+
+ def test_opt_rtn_home_and_mirrors_match_serial(self):
+ from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig
+ from auto_round.algorithms.quantization.rtn.quantizer import OptimizedRTNQuantizer
+
+ cfg = OptimizedRTNConfig(bits=4, group_size=32, sym=True, disable_opt_rtn=False, enable_neuqi=False)
+ self._run_parity(OptimizedRTNQuantizer, cfg)
+
+ def test_plain_rtn_home_and_mirrors_match_serial(self):
+ from auto_round.algorithms.quantization.rtn.config import RTNConfig
+ from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer
+
+ cfg = RTNConfig(bits=4, group_size=32, sym=True, disable_opt_rtn=True)
+ self._run_parity(RTNQuantizer, cfg)
diff --git a/test/unit/test_cpu/quantization/test_search_dispatch.py b/test/unit/test_cpu/quantization/test_search_dispatch.py
index 610072819..01612c868 100644
--- a/test/unit/test_cpu/quantization/test_search_dispatch.py
+++ b/test/unit/test_cpu/quantization/test_search_dispatch.py
@@ -791,3 +791,62 @@ class TestPlanCapacityChunks(unittest.TestCase):
plans, _ = self._plan([("cuda:1", per, 0, 8, group_home1), ("cuda:2", per, 0, 8, group_home2)], probes, picks)
for worker, home, chunk in plans:
self.assertEqual(worker, home)
+
+
+class TestSearchSummary(unittest.TestCase):
+ """One summary line per search phase: per-device busy time (stacked and
+ per-module calls alike) and the elapsed time of the phase."""
+
+ def _line(self, summary, label="[batched-search] wrap search:"):
+ from unittest import mock
+
+ from auto_round.algorithms.quantization import search_dispatch
+
+ with (
+ mock.patch.object(search_dispatch, "perf_counters_enabled", return_value=True),
+ mock.patch.object(search_dispatch.logger, "info") as info,
+ ):
+ summary.log(label)
+ fmt, *args = info.call_args[0]
+ return fmt % tuple(args)
+
+ def test_line_keeps_the_summary_grammar(self):
+ from auto_round.algorithms.quantization.search_dispatch import SearchSummary
+
+ s = SearchSummary()
+ st = s.device("cuda:0")
+ st.update(wall=1.5, modules=10, batches=2, singletons=0)
+ st = s.device("cuda:1")
+ st.update(wall=0.5, modules=2, batches=0, singletons=2)
+ s.elapsed = 1.75
+ self.assertEqual(
+ self._line(s),
+ "[batched-search] wrap search: 12 modules, 2 batch + 2 singleton search calls, "
+ "1.75s device-wall (2.00s summed) walls {'0': 1.50, '1': 0.50}",
+ )
+
+ def test_per_module_calls_are_timed(self):
+ import time as _time
+
+ from auto_round.algorithms.quantization.search_dispatch import SearchSummary
+
+ s = SearchSummary()
+ with s.timed("cuda:0") as st:
+ _time.sleep(0.02)
+ st["singletons"] += 1
+ st["modules"] += 1
+ s.finish()
+ self.assertGreater(s.devices["cuda:0"]["wall"], 0.01)
+ self.assertGreaterEqual(s.elapsed, s.devices["cuda:0"]["wall"])
+
+ def test_merged_ranks_add_counters_and_keep_the_slowest_elapsed(self):
+ from auto_round.algorithms.quantization.search_dispatch import SearchSummary
+
+ a, b = SearchSummary(), SearchSummary()
+ a.device("cuda:0").update(wall=1.0, modules=3, batches=1, singletons=0)
+ b.device("cuda:1").update(wall=2.0, modules=2, batches=0, singletons=2)
+ a.elapsed, b.elapsed = 1.1, 2.2
+ m = SearchSummary.merged([a, b])
+ self.assertEqual(m.elapsed, 2.2)
+ self.assertEqual(m.devices["cuda:0"]["modules"] + m.devices["cuda:1"]["modules"], 5)
+ self.assertIn("5 modules, 1 batch + 2 singleton search calls, 2.20s device-wall (3.00s summed)", self._line(m))
diff --git a/test/unit/test_cpu/quantization/test_spmd_ranks.py b/test/unit/test_cpu/quantization/test_spmd_ranks.py
new file mode 100644
index 000000000..e1a1b5572
--- /dev/null
+++ b/test/unit/test_cpu/quantization/test_spmd_ranks.py
@@ -0,0 +1,377 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Thread ranks (auto_round.algorithms.parallel.spmd): collectives, turnstile,
+samplers, failure handling, and the unchanged SignRound loop run per rank
+on CPU against the serial run."""
+
+import logging
+import threading
+
+import pytest
+import torch
+
+from auto_round.algorithms.parallel import spmd
+
+CPU2 = [torch.device("cpu"), torch.device("cpu", 1)]
+
+
+def _run(fns, devices=None):
+ devices = devices or CPU2[: len(fns)]
+ return spmd.run_ranks(fns, devices, spmd.RankGroup(devices))
+
+
+class TestCollectives:
+ def test_weighted_mean_reproduces_the_union(self):
+ # rank 0: 2 tokens with loss 1.0, rank 1: 6 tokens with loss 3.0
+ out = _run(
+ [
+ lambda: spmd.current_rank().all_reduce_weighted_mean(1.0, 2),
+ lambda: spmd.current_rank().all_reduce_weighted_mean(3.0, 6),
+ ]
+ )
+ assert out == [2.5, 2.5]
+
+ def test_mean_is_identical_on_every_rank(self):
+ vals = [1.0, 2.0, 4.0, 8.0]
+ devs = [torch.device("cpu")] + [torch.device("cpu", i) for i in range(1, 4)]
+ out = _run([lambda v=v: spmd.current_rank().all_reduce_mean(v) for v in vals], devs)
+ assert out == [3.75] * 4
+
+ def test_grads_average_across_ranks_like_one_batch(self):
+ torch.manual_seed(0)
+ base = torch.nn.Linear(4, 2)
+ xs = [torch.randn(3, 4), torch.randn(3, 4)]
+ reps = [base, torch.nn.Linear(4, 2)]
+ reps[1].load_state_dict(base.state_dict())
+
+ def _fn(r):
+ def _go():
+ reps[r](xs[r]).pow(2).mean().backward()
+ spmd.current_rank().sync_gradients(reps[r])
+
+ return _go
+
+ _run([_fn(0), _fn(1)])
+ ref = torch.nn.Linear(4, 2)
+ ref.load_state_dict(base.state_dict())
+ ref(torch.cat(xs)).pow(2).mean().backward()
+ for p_ref, p0, p1 in zip(ref.parameters(), reps[0].parameters(), reps[1].parameters()):
+ assert torch.equal(p0.grad, p1.grad)
+ assert torch.allclose(p0.grad, p_ref.grad, atol=1e-6)
+
+ def test_rank0_runs_in_the_calling_thread(self):
+ main = threading.current_thread()
+ out = _run([lambda: threading.current_thread() is main, lambda: threading.current_thread() is main])
+ assert out == [True, False]
+
+
+class TestTurnstileAndFailures:
+ def test_first_phases_run_in_rank_order(self):
+ events = []
+ lock = threading.Lock()
+
+ def _fn(r):
+ def _go():
+ with lock:
+ events.append(("start", r))
+ spmd.current_rank().all_reduce_mean(r)
+ with lock:
+ events.append(("after", r))
+
+ return _go
+
+ devs = [torch.device("cpu")] + [torch.device("cpu", i) for i in range(1, 4)]
+ _run([_fn(r) for r in range(4)], devs)
+ starts = [r for kind, r in events if kind == "start"]
+ assert starts == [0, 1, 2, 3]
+ # nobody passes the collective before the last rank started
+ assert events.index(("start", 3)) < min(events.index(("after", r)) for r in range(4))
+
+ def test_warm_blocks_start_every_rank_at_once(self):
+ started = threading.Event()
+
+ def _rank0():
+ # rank 1 must start before rank 0 reaches its first collective
+ assert started.wait(timeout=5)
+ spmd.current_rank().all_reduce_mean(0.0)
+
+ def _rank1():
+ started.set()
+ spmd.current_rank().all_reduce_mean(1.0)
+
+ spmd.run_ranks([_rank0, _rank1], CPU2, spmd.RankGroup(CPU2, serial_first_phase=False))
+
+ def test_block_signature_tracks_shapes_and_devices(self):
+ a, b = torch.nn.Linear(4, 4), torch.nn.Linear(4, 8)
+ assert spmd.block_signature(a, CPU2) == spmd.block_signature(torch.nn.Linear(4, 4), CPU2)
+ assert spmd.block_signature(a, CPU2) != spmd.block_signature(b, CPU2)
+ assert spmd.block_signature(a, CPU2) != spmd.block_signature(a, CPU2[:1])
+
+ def test_peer_failure_raises_the_real_error_without_hanging(self):
+ def _ok():
+ spmd.current_rank().all_reduce_mean(1.0)
+ spmd.current_rank().all_reduce_mean(2.0)
+
+ def _boom():
+ spmd.current_rank().all_reduce_mean(1.0)
+ raise ValueError("rank 1 failed")
+
+ with pytest.raises(ValueError, match="rank 1 failed"):
+ _run([_ok, _boom])
+
+ def test_failure_before_first_collective_releases_the_next_rank(self):
+ def _boom():
+ raise KeyError("early")
+
+ with pytest.raises(KeyError):
+ _run([_boom, lambda: spmd.current_rank().all_reduce_mean(1.0)])
+
+
+class TestRankScope:
+ def test_device_manager_sees_the_rank_device_only_inside_the_rank(self):
+ from auto_round.utils.device_manager import device_manager
+
+ outside = device_manager.device
+ seen = _run([lambda: device_manager.device, lambda: (device_manager.device, device_manager.device_list)])
+ assert seen[0] == "cpu"
+ assert seen[1] == ("cpu:1", ["cpu:1"])
+ assert device_manager.device == outside
+
+ def test_index_sampler_draws_from_the_rank_shard(self):
+ from auto_round.utils.distributed import make_index_sampler
+
+ def _draws():
+ s = make_index_sampler(8, 2)
+ return sorted(i for _ in range(2) for i in s.next_batch())
+
+ out = _run([_draws, _draws])
+ assert out == [[0, 1, 2, 3], [4, 5, 6, 7]]
+ assert make_index_sampler(8, 2).nsamples == 8 # outside ranks: the whole pool
+
+ def test_non_zero_ranks_log_warnings_only(self, caplog):
+ from auto_round.logger import logger
+
+ logger.propagate = True
+ try:
+ with caplog.at_level(logging.INFO, logger="autoround"):
+
+ def _log(r):
+ def _go():
+ logger.info("info from rank %d", r)
+ logger.warning("warning from rank %d", r)
+
+ return _go
+
+ _run([_log(0), _log(1)])
+ finally:
+ logger.propagate = False
+ msgs = [r.getMessage() for r in caplog.records]
+ assert "info from rank 0" in msgs and "warning from rank 0" in msgs
+ assert "info from rank 1" not in msgs and "warning from rank 1" in msgs
+
+
+_DS = [
+ "CPU-tier rank parity sample with enough tokens for quantization",
+ "Another parity sample with a different length padding the batch",
+ "Third sample keeps the calibration pool at four entries",
+ "Fourth sample completes the pinned draw schedule",
+]
+
+
+class _Fixed:
+ def __init__(self, draws):
+ self._draws, self._i = list(draws), 0
+
+ def next_batch(self):
+ out = self._draws[self._i % len(self._draws)]
+ self._i += 1
+ return out
+
+
+def _pinned_sampler(nsamples, batch_size):
+ # serial: batches [0, 1] then [2, 3]; rank r draws the r-th sample of the
+ # same batch, so both runs see the same samples per iteration
+ rank = spmd.current_rank()
+ if rank is None:
+ return _Fixed([[0, 1], [2, 3]])
+ return _Fixed([[0], [2]]) if rank.rank == 0 else _Fixed([[1], [3]])
+
+
+def _quantize(model_path, monkeypatch, world, **extra):
+ import auto_round.algorithms.parallel.tune_parallel as tp
+ import auto_round.algorithms.quantization.sign_round.quantizer as sr
+ from auto_round import AutoRound
+ from auto_round.algorithms.parallel.data_parallel import DDPPlan, ParallelPolicy
+
+ monkeypatch.setattr(sr, "make_index_sampler", _pinned_sampler)
+ engaged = []
+ if world > 1:
+ # CPU ranks: the resolvers decline non-accelerator homes, so pin the
+ # tune plan to (cpu, cpu:1) and keep the collection serial
+ monkeypatch.setattr(spmd, "resolve_tune_ddp_plan_", lambda *a, **k: DDPPlan(2, CPU2, 2))
+ monkeypatch.setattr(tp, "resolve_tune_ddp_plan_", lambda *a, **k: DDPPlan(1, CPU2[:1], 0))
+ orig = spmd.quantize_block_on_ranks
+
+ def _counted(*a, **k):
+ res = orig(*a, **k)
+ engaged.append((res[0], type(a[0]).__name__))
+ return res
+
+ monkeypatch.setattr(spmd, "quantize_block_on_ranks", _counted)
+ ar = AutoRound(
+ model_path,
+ bits=4,
+ group_size=32,
+ sym=extra.pop("sym", False),
+ iters=2,
+ nsamples=4,
+ seqlen=8,
+ batch_size=2,
+ dataset=list(_DS),
+ device_map="cpu",
+ enable_torch_compile=False,
+ disable_model_free=True,
+ **extra,
+ )
+ if world > 1:
+ ar.compress_context.parallel_policy = ParallelPolicy(world=world)
+ from auto_round.logger import logger
+
+ losses = []
+
+ class _LossLines(logging.Handler):
+ def emit(self, record):
+ msg = record.getMessage()
+ if "loss iter" in msg:
+ losses.append(msg[msg.index("loss iter") :])
+
+ handler = _LossLines(level=logging.INFO)
+ logger.addHandler(handler)
+ try:
+ model, _ = ar.quantize()
+ finally:
+ logger.removeHandler(handler)
+ weights = {k: v.detach().clone() for k, v in model.state_dict().items()}
+ return weights, engaged, losses
+
+
+def test_signround_on_two_cpu_ranks_matches_serial(tiny_opt_model_path, monkeypatch):
+ """The unchanged SignRound loop, run once per rank with averaged
+ gradients, lands on the serial run's quantized weights bit-exactly."""
+ serial, _, serial_losses = _quantize(tiny_opt_model_path, monkeypatch, world=1)
+ ranks, engaged, rank_losses = _quantize(tiny_opt_model_path, monkeypatch, world=2)
+ assert engaged and all(e for e, _ in engaged)
+ assert set(serial) == set(ranks)
+ diff = [k for k in serial if not torch.equal(serial[k], ranks[k])]
+ assert not diff, f"rank run diverged from serial: {diff[:8]}"
+ # the padded samples carry a valid-token mask: the global loss (masked
+ # sums over valid tokens, weighted by each rank's token count) reports
+ # the serial value
+ assert serial_losses and rank_losses == serial_losses
+
+
+def test_signroundv2_on_two_cpu_ranks_matches_serial(tiny_opt_model_path, monkeypatch):
+ """SignRoundV2 defers its wrap-time init-scale searches; warm ranks shard
+ them (each rank searches its share, the scales are exchanged), and the
+ result still matches the serial run bit-exactly."""
+ sharded = []
+ orig = spmd.run_rank_sharded_wrap_searches
+
+ def _counted(rank, wrappers):
+ res = orig(rank, wrappers)
+ sharded.append(res)
+ return res
+
+ monkeypatch.setattr(spmd, "run_rank_sharded_wrap_searches", _counted)
+ serial, _, _ = _quantize(tiny_opt_model_path, monkeypatch, world=1, sym=True, enable_alg_ext=True)
+ ranks, engaged, _ = _quantize(tiny_opt_model_path, monkeypatch, world=2, sym=True, enable_alg_ext=True)
+ assert engaged and all(e and name == "SignRoundV2Quantizer" for e, name in engaged), engaged
+ assert any(sharded), "no block sharded its wrap searches"
+ diff = [k for k in serial if not torch.equal(serial[k], ranks[k])]
+ assert not diff, f"rank run diverged from serial: {diff[:8]}"
+
+
+def test_pull_tuning_pool_keeps_the_rank_shards_in_place():
+ """A rank must not bulk-pull the shared, device-distributed pool onto its
+ own device (every other rank's shard would be copied there)."""
+ from auto_round.algorithms.quantization.base import BaseQuantizer
+
+ q = BaseQuantizer.__new__(BaseQuantizer)
+ pool = [torch.zeros(1), torch.zeros(1)]
+ out = _run([lambda: q.pull_tuning_pool(pool, "cpu", None, 1, "pool"), lambda: None])
+ assert out[0] is pool
+
+
+class _FakeWrapper:
+ def __init__(self, idx, searched):
+ self.idx, self._searched, self.init_scale, self.final = idx, searched, None, None
+
+ def _run_deferred_search_now(self):
+ self._searched.append((spmd.current_rank().rank, self.idx))
+ self.init_scale = self.final = torch.tensor(float(self.idx))
+
+ def finalize_batched_search(self, init_scale):
+ self.init_scale = self.final = init_scale
+
+
+def test_wrap_searches_shard_across_warm_ranks(monkeypatch):
+ from auto_round.algorithms.quantization import search_dispatch
+ from auto_round.logger import logger
+
+ monkeypatch.setattr(search_dispatch, "run_batched_wrap_search", lambda ws, **kw: False)
+ monkeypatch.setattr(search_dispatch, "perf_counters_enabled", lambda: True)
+ searched = []
+ per_rank = [[_FakeWrapper(i, searched) for i in range(5)] for _ in range(2)]
+
+ def _fn(r):
+ return lambda: spmd.run_rank_sharded_wrap_searches(spmd.current_rank(), per_rank[r])
+
+ lines = []
+
+ class _Lines(logging.Handler):
+ def emit(self, record):
+ lines.append(record.getMessage())
+
+ handler = _Lines(level=logging.INFO)
+ logger.addHandler(handler)
+ try:
+ out = spmd.run_ranks([_fn(0), _fn(1)], CPU2, spmd.RankGroup(CPU2, serial_first_phase=False))
+ finally:
+ logger.removeHandler(handler)
+ assert out == [True, True]
+ # every wrapper searched exactly once, by its owner rank (index % world)
+ assert sorted(searched, key=lambda t: t[1]) == [(i % 2, i) for i in range(5)]
+ for ws in per_rank:
+ assert [float(w.final) for w in ws] == [0.0, 1.0, 2.0, 3.0, 4.0]
+ # one summary line for the block, covering every rank's share and device
+ summary = [m for m in lines if m.startswith("[batched-search] wrap search:")]
+ assert len(summary) == 1, lines
+ assert "5 modules, 0 batch + 5 singleton search calls" in summary[0], summary
+ assert "'cpu': " in summary[0] and "'cpu:1': " in summary[0], summary
+
+
+def test_wrap_searches_stay_local_in_the_serial_first_phase():
+ out = _run([lambda: spmd.run_rank_sharded_wrap_searches(spmd.current_rank(), []), lambda: None])
+ assert out[0] is False
+
+
+def test_adamround_on_two_cpu_ranks_matches_serial(tiny_opt_model_path, monkeypatch):
+ """AdamRound runs the same unchanged loop per rank; its update uses
+ gradient magnitudes, so the averaged rank gradients must reproduce the
+ serial update."""
+ serial, _, _ = _quantize(tiny_opt_model_path, monkeypatch, world=1, enable_adam=True)
+ ranks, engaged, _ = _quantize(tiny_opt_model_path, monkeypatch, world=2, enable_adam=True)
+ assert engaged and all(e and name == "AdamRoundQuantizer" for e, name in engaged), engaged
+ diff = [k for k in serial if not torch.equal(serial[k], ranks[k])]
+ assert not diff, f"rank run diverged from serial: {diff[:8]}"
diff --git a/test/unit/test_cpu/quantization/test_tune_parallel_context.py b/test/unit/test_cpu/quantization/test_tune_parallel_context.py
new file mode 100644
index 000000000..c55cca634
--- /dev/null
+++ b/test/unit/test_cpu/quantization/test_tune_parallel_context.py
@@ -0,0 +1,130 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for the collection-phase TuneParallelContext.
+
+The engine primitives (sharded_nograd_forward, ...) are faked: these tests
+pin the context's collection-path semantics and serial fallbacks. The tune
+itself runs on thread ranks (test_spmd_ranks).
+"""
+
+import logging
+from unittest import mock
+
+import pytest
+import torch
+
+from auto_round.algorithms.parallel.tune_parallel import TuneParallelContext
+
+
+class TestCollectionContext:
+ def test_for_collection_non_list_inputs_keeps_serial(self):
+ composer = mock.Mock()
+ ctx = TuneParallelContext.for_collection(composer, block=torch.nn.Linear(2, 2), fp_inputs={"a": [1]})
+ assert ctx.devices is None
+
+ def test_collect_forward_serial_when_devices_none(self):
+ ctx = TuneParallelContext()
+ bf = mock.Mock(return_value="out")
+ assert ctx.collect_forward(bf, "blk", "inp", "oth", out_dev="cpu") == "out"
+ bf.assert_called_once_with("blk", "inp", "oth", cache_device="cpu")
+
+ def test_collect_forward_sharded_when_devices_set(self):
+ ctx = TuneParallelContext()
+ ctx.devices = [torch.device("cpu"), torch.device("cpu")]
+ with mock.patch(
+ "auto_round.algorithms.parallel.tune_parallel.sharded_nograd_forward",
+ return_value="sharded",
+ ) as snf:
+ assert ctx.collect_forward("bf", "blk", "inp", "oth") == "sharded"
+ from auto_round.algorithms.parallel.data_parallel import ENGINE_CALIBRATION_STATS
+
+ snf.assert_called_once_with(
+ "bf",
+ "blk",
+ "inp",
+ "oth",
+ None,
+ ctx.devices,
+ merge_stats=ENGINE_CALIBRATION_STATS,
+ pool=None,
+ max_devices=0,
+ )
+
+ def test_collect_forward_undeclared_stats_stay_serial(self):
+ """merge_stats=None (quantizer hooks without a calibration_stats
+ declaration): the pass must not shard -- the mirrors' statistics
+ could not be folded back."""
+ ctx = TuneParallelContext()
+ ctx.devices = [torch.device("cpu"), torch.device("cpu")]
+ bf = mock.Mock(return_value="serial")
+ with mock.patch("auto_round.algorithms.parallel.tune_parallel.sharded_nograd_forward") as snf:
+ assert ctx.collect_forward(bf, "blk", "inp", "oth", hook_pass=True, merge_stats=None) == "serial"
+ snf.assert_not_called()
+ bf.assert_called_once_with("blk", "inp", "oth", cache_device=None)
+
+ def test_collect_forward_hook_pass_cap_from_policy(self):
+ ctx = TuneParallelContext()
+ ctx.devices = [torch.device("cpu")] * 8
+ with mock.patch(
+ "auto_round.algorithms.parallel.tune_parallel.sharded_nograd_forward",
+ return_value="sharded",
+ ) as snf:
+ ctx.collect_forward("bf", "blk", "inp", "oth", hook_pass=True)
+ # default: no cap (full-width hook passes)
+ assert snf.call_args.kwargs["max_devices"] == 0
+ # the cap is set on the context (folded from the env into the
+ # ParallelPolicy at the entry)
+ ctx.collect_forward_cap = 2
+ ctx.collect_forward("bf", "blk", "inp", "oth", hook_pass=True)
+ assert snf.call_args.kwargs["max_devices"] == 2
+ # non-hook passes stay uncapped
+ ctx.collect_forward("bf", "blk", "inp", "oth", hook_pass=False)
+ assert snf.call_args.kwargs["max_devices"] == 0
+
+ def test_capped_hook_pass_keeps_the_pool(self):
+ """The cap limits WHICH pool replicas run, never pool eligibility:
+ the full-width pool is passed through with max_devices set."""
+ ctx = TuneParallelContext()
+ ctx.devices = [torch.device("cpu")] * 8
+ ctx.collect_forward_cap = 2
+ sentinel = object()
+ ctx.pool = sentinel
+ blk = torch.nn.Linear(2, 2) # pool eligibility requires a real Module
+ with mock.patch(
+ "auto_round.algorithms.parallel.tune_parallel.sharded_nograd_forward",
+ return_value="sharded",
+ ) as snf:
+ assert ctx.collect_forward("bf", blk, [1] * 8, "oth", hook_pass=True) == "sharded"
+ assert snf.call_args.kwargs["pool"] is sentinel
+ assert snf.call_args.kwargs["max_devices"] == 2
+
+ def test_distribute_pools_noop_without_devices(self):
+ ctx = TuneParallelContext()
+ pool = [torch.zeros(1), torch.zeros(1)]
+ with mock.patch("auto_round.algorithms.parallel.tune_parallel.distribute_pool") as dp:
+ ctx.distribute_pools(pool, None)
+ dp.assert_not_called()
+
+
+def _autoround_log_propagate():
+ """Temporarily enable propagation on the ``autoround`` logger so pytest's
+ caplog fixture (handler at the root logger) can capture warnings; the
+ logger is configured with propagate=False in production."""
+
+ _logger = logging.getLogger("autoround")
+ original = _logger.propagate
+ _logger.propagate = True
+ yield
+ _logger.propagate = original
diff --git a/test/unit/test_cuda/quantization/test_ddp_mirror.py b/test/unit/test_cuda/quantization/test_ddp_mirror.py
new file mode 100644
index 000000000..e4c276ed4
--- /dev/null
+++ b/test/unit/test_cuda/quantization/test_ddp_mirror.py
@@ -0,0 +1,87 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""CUDA-tier tests for DDP mirror building.
+
+The resolver requires single-device block placement: a block whose weights
+span several CUDA devices is declined loudly.
+"""
+
+import pytest
+import torch
+
+from auto_round.algorithms.parallel.data_parallel import resolve_tune_ddp_plan_
+
+_requires_multi_cuda = pytest.mark.skipif(
+ torch.cuda.device_count() < 2, reason="needs >=2 CUDA devices for a sharded source block"
+)
+
+
+class _TinyBlock(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.l0 = torch.nn.Linear(8, 8, bias=False)
+ self.l1 = torch.nn.Linear(8, 8, bias=False)
+
+ def forward(self, x):
+ return self.l1(torch.nn.functional.relu(self.l0(x)))
+
+
+def _make_sharded_block():
+ block = _TinyBlock().cuda(0)
+ block.l0.tuning_device = "cuda:0"
+ block.l1.tuning_device = "cuda:1"
+ block.l1.to("cuda:1") # emulate the auto block device map shard
+ # plain-attr tensors + a params-dict entry, the wrapper state _relocate covers
+ block.l0.anchor = torch.zeros(4, device="cuda:0")
+ block.l1.anchor = torch.zeros(4, device="cuda:1")
+ block.l0.params = {"v": torch.nn.Parameter(torch.ones(8, device="cuda:0"), requires_grad=True)}
+ block.l1.params = {"v": torch.nn.Parameter(torch.ones(8, device="cuda:1"), requires_grad=True)}
+ return block
+
+
+def _make_whole_block():
+ block = _TinyBlock().cuda(0)
+ block.l0.tuning_device = "cuda:0"
+ block.l1.tuning_device = "cuda:0"
+ block.l0.anchor = torch.zeros(4, device="cuda:0")
+ block.l1.anchor = torch.zeros(4, device="cuda:0")
+ block.l0.params = {"v": torch.nn.Parameter(torch.ones(8, device="cuda:0"), requires_grad=True)}
+ block.l1.params = {"v": torch.nn.Parameter(torch.ones(8, device="cuda:0"), requires_grad=True)}
+ return block
+
+
+def _quantizer():
+ from types import SimpleNamespace
+
+ q = SimpleNamespace(iters=10, gradient_accumulate_steps=1, enable_lfq=False)
+ q._get_scaler = lambda: None
+ return q
+
+
+def _quantizer_with_policy(world=2):
+ from types import SimpleNamespace
+
+ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy
+
+ q = _quantizer()
+ q.compress_context = SimpleNamespace(parallel_policy=ParallelPolicy(world=world))
+ return q
+
+
+@_requires_multi_cuda
+class TestShardedSourceDeclines:
+ def test_spanning_block_raises_with_placement_reason(self, monkeypatch):
+ block = _make_sharded_block()
+ with pytest.raises(RuntimeError, match="spans 2 CUDA devices"):
+ resolve_tune_ddp_plan_(_quantizer_with_policy(), block, [torch.zeros(1, 8)], None, "cuda:0")
diff --git a/test/unit/test_cuda/quantization/test_ddp_parity.py b/test/unit/test_cuda/quantization/test_ddp_parity.py
new file mode 100644
index 000000000..4fe03f57c
--- /dev/null
+++ b/test/unit/test_cuda/quantization/test_ddp_parity.py
@@ -0,0 +1,135 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""CUDA-tier serial-vs-DDP result parity: RTN iters=0 and the default
+SignRound tune at iters=2.
+
+Both arms quantize the SAME tiny model (same fixture path -> identical
+weights) with the SAME calibration samples. The DDP arm engages the
+parallel lane via a world=2 ParallelPolicy on the compress context and receives the SAME samples per
+rank through pinned draws: each iteration the serial lane processes batch
+[a, b] while rank 0 draws [a] and rank 1 draws [b]
+-- mean-of-shard-means equals the full-batch mean, so SignSGD consumes
+identical gradients and both lanes evolve identically.
+
+The final quantized weights are asserted BIT-EXACT (torch.equal). For
+iters=0 this is guaranteed (the same deterministic per-module quantize on
+identical weights). For iters=2 the two lanes are different floating-point
+programs (batched reduction vs shard+average, fp32 gradient exchange), so
+bitwise equality is expected via the discrete sign steps + the final
+round-to-grid; a mismatch is a real signal to investigate (transport
+precision or reduction order), matched exactly.
+"""
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(torch.cuda.device_count() < 2, reason="needs 2+ CUDA devices")
+
+_DS = [
+ "CUDA-tier DDP parity sample with enough tokens for quantization",
+ "Another parity sample with a different length padding the batch",
+ "Third sample keeps the calibration pool at four entries",
+ "Fourth sample completes the pinned draw schedule",
+]
+
+
+class _FixedSampler:
+ """Yields a fixed global-index schedule (list of lists) in order."""
+
+ def __init__(self, draws):
+ self._draws = list(draws)
+ self._i = 0
+
+ def next_batch(self):
+ out = self._draws[self._i]
+ self._i += 1
+ return out
+
+
+def _autoround(model_path, iters, disable_opt_rtn=False):
+ from auto_round import AutoRound
+
+ return AutoRound(
+ model_path,
+ bits=4,
+ group_size=32,
+ sym=False,
+ iters=iters,
+ nsamples=4,
+ seqlen=8,
+ batch_size=2,
+ dataset=list(_DS),
+ device_map="0",
+ enable_torch_compile=False,
+ disable_opt_rtn=disable_opt_rtn,
+ disable_model_free=True,
+ )
+
+
+def _run(model_path, iters, disable_opt_rtn, monkeypatch, pin_draws, world=1):
+ import auto_round.algorithms.quantization.sign_round.quantizer as v1
+ from auto_round.algorithms.parallel import spmd
+ from auto_round.algorithms.parallel.data_parallel import ParallelPolicy
+
+ if pin_draws:
+ # serial lane: batches [0, 1] then [2, 3]; rank r draws the r-th
+ # sample of the SAME global batch per iteration
+ def _pinned(nsamples, batch):
+ rank = spmd.current_rank()
+ if rank is None:
+ return _FixedSampler([[0, 1], [2, 3]])
+ return _FixedSampler([[0], [2]]) if rank.rank == 0 else _FixedSampler([[1], [3]])
+
+ monkeypatch.setattr(v1, "make_index_sampler", _pinned)
+ ar = _autoround(model_path, iters, disable_opt_rtn)
+ if world > 1:
+ # the policy is set on the compress context (the CLI does it the same way)
+ ar.compress_context.parallel_policy = ParallelPolicy(world=world)
+ model, layer_config = ar.quantize()
+ sd = model.state_dict()
+ state = {}
+ for name, cfg in layer_config.items():
+ if cfg.get("data_type") != "int":
+ continue
+ key = name + ".weight"
+ assert key in sd, f"quantized layer {name} has no weight in state_dict"
+ state[name] = sd[key].detach().cpu().clone()
+ assert state, "no quantized layers found"
+ return state
+
+
+def _assert_same_weights(serial, ddp):
+ assert set(serial) == set(ddp), "quantized layer sets differ"
+ for name in serial:
+ assert torch.equal(serial[name], ddp[name]), (
+ f"layer {name}: serial vs DDP final weights differ "
+ f"(max abs diff {float((serial[name] - ddp[name]).abs().max())})"
+ )
+
+
+def test_rtn_iters0_serial_vs_ddp_bitexact(tiny_opt_model_path, monkeypatch):
+ """Plain RTN (iters=0, disable_opt_rtn): sharded collection forwards plus
+ the home quantize and mirror sync must reproduce the serial lane bit-exactly."""
+ serial = _run(tiny_opt_model_path, 0, True, monkeypatch, pin_draws=False)
+ ddp = _run(tiny_opt_model_path, 0, True, monkeypatch, pin_draws=False, world=2)
+ _assert_same_weights(serial, ddp)
+
+
+def test_signround_iters2_serial_vs_ddp_bitexact(tiny_opt_model_path, monkeypatch):
+ """Default SignRound tune at iters=2: pinned shard draws give every
+ replica the serial lane's samples, so both lanes must land on identical
+ final quantized weights."""
+ serial = _run(tiny_opt_model_path, 2, False, monkeypatch, pin_draws=True)
+ ddp = _run(tiny_opt_model_path, 2, False, monkeypatch, pin_draws=True, world=2)
+ _assert_same_weights(serial, ddp)