Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

22.4 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Batched zero-shot (iters=0) search driver.

Layers whose staged key -- (device, weight shape/dtype, bits, group size,
symmetry, threshold, resolved quant callable) -- matches are stacked along a
leading dim and quantized with ONE ``weight_quant_func`` call. The quant math
is row-independent, and the per-column imatrix is pre-expanded to each
module's full weight shape before stacking so the quant function's internal
flatten-expand becomes an identity: results are bit-identical to the serial
``unwrapper({})`` path. Write-back goes through the wrapper's ``_apply_qdq``
(the unwrapper's own conventions), so serial and batched outputs can never
drift apart.
"""

from collections import OrderedDict
from collections.abc import Callable
from typing import Any

import torch

from auto_round.algorithms.quantization.search_dispatch import (
    SearchSummary,
    _fn_key,
    _plan_capacity_chunks,
    _search_fn_stackable,
    _wrap_batch_max_elems,
    dump_oom_tensor_census_,
    group_items_by_device,
    run_items_by_device,
    search_sweep_budget,
)
from auto_round.logger import logger
from auto_round.utils import set_module
from auto_round.wrapper import WrapperLinear


def _staged_weight(wrapper: WrapperLinear) -> torch.Tensor:
    """The weight exactly as _qdq_weight would pass it to the quant func."""
    weight = wrapper.orig_layer.weight
    import transformers

    if type(wrapper.orig_layer) == transformers.pytorch_utils.Conv1D:
        weight = weight.t()
    return weight


def _staged_imatrix(wrapper: WrapperLinear, weight: torch.Tensor) -> torch.Tensor | None:
    """The RAW per-column imatrix (or None) -- expansion happens chunk-time.

    Staging the expanded full-size copy would pin N x weight bytes across the
    whole staging phase; the chunk-time expansion below is transient and
    bounded by the batch cap. Groups are homogeneous on imatrix presence (it
    is part of the staged key), so chunks are all-None or all-tensor.
    """
    del weight
    return getattr(wrapper.orig_layer, "imatrix", None)


def _expand_imatrix(im: torch.Tensor | None, weight: torch.Tensor, group_size: int) -> torch.Tensor | None:
    """Expand one module's raw imatrix to its full weight shape (chunk-time).

    Mirrors the quant funcs' own imatrix handling exactly (data_type/int.py):
    the raw per-column vector is group-padded (fill 1e-5) BEFORE being row
    expanded, so the stacked flat layout byte-matches what the serial
    per-module call produces after its internal flatten-pad-expand. Skipping
    the pad misplaces the group alignment for every module after the first
    when ``in % group_size != 0``; casting to the weight dtype (bf16/fp16
    weights with an fp32 hook-collected imatrix) silently diverges the
    search loss weighting from the serial path.
    """
    if im is None:
        return None
    im = im.to(weight.device)  # device only: keep the collected precision
    if im.dim() == 1 and weight.dim() == 2:
        out_f, in_f = weight.shape
        pad_len = 0
        if isinstance(group_size, int) and 0 < group_size <= in_f and in_f % group_size != 0:
            pad_len = ((in_f + group_size - 1) // group_size) * group_size - in_f
        if pad_len:
            im = torch.nn.functional.pad(im, (0, pad_len), value=1e-5)
        return im.unsqueeze(0).expand(out_f, -1).contiguous()
    if im.dim() == 2 and weight.dim() == 2 and im.shape[0] == 1:
        # row vector form: same treatment
        out_f, in_f = weight.shape
        pad_len = 0
        if isinstance(group_size, int) and 0 < group_size <= in_f and in_f % group_size != 0:
            pad_len = ((in_f + group_size - 1) // group_size) * group_size - in_f
        if pad_len:
            im = torch.nn.functional.pad(im, (0, pad_len), value=1e-5)
        return im.expand(out_f, -1).contiguous()
    if im.shape == weight.shape:
        return im.contiguous()
    return im


def _extra_quant_kwargs_key(wrapper: WrapperLinear) -> tuple[tuple[str, str], ...] | None:
    """Per-module _extra_quant_kwargs hook values as a hashable key term.

    Any wrapper subclass defining the hook injects arbitrary per-module kwargs
    into the quant call; two modules with differing extras must never share a
    stacked chunk (same hazard class as differing global_scale).
    """
    hook = getattr(wrapper, "_extra_quant_kwargs", None)
    if not callable(hook):
        return None
    return tuple(sorted((str(k), repr(v)) for k, v in hook().items()))


def _staged_key(wrapper: WrapperLinear, weight: torch.Tensor, imatrix: torch.Tensor | None) -> tuple[
    str,
    tuple[int, ...],
    str,
    int | None,
    int | None,
    bool | None,
    float,
    bool,
    str | tuple[Any, ...],
    str,
    int | None,
    int | None,
    float | None,
    tuple[tuple[str, str], ...] | None,
]:
    layer = wrapper.orig_layer
    # The stacked call assembles quant kwargs once from chunk[0], so every
    # per-layer kwarg _quant_call_kwargs injects must be part of the key:
    # modules differing in any of them never share a batch (nv-fp4 computes
    # weight_global_scale per layer; scale_dtype/super_* come from
    # per-layer config pins). imatrix is passed as a stacked override.
    global_scale = getattr(wrapper, "weight_global_scale", None)
    return (
        str(weight.device),
        tuple(weight.shape),
        str(weight.dtype),
        getattr(layer, "bits", None),
        getattr(layer, "group_size", None),
        getattr(layer, "sym", None),
        float(getattr(wrapper, "q_scale_thresh", 1e-5)),
        imatrix is not None,
        # normalize away per-wrapper torch.compile wrappers: the underlying
        # eager original is shared per config, while each compiled wrapper is
        # unique (repr-keyed) and would silently collapse every group to size 1
        _fn_key(getattr(wrapper.weight_quant_func, "_torchdynamo_orig_callable", None) or wrapper.weight_quant_func),
        str(getattr(layer, "scale_dtype", None)),
        getattr(layer, "super_bits", None),
        getattr(layer, "super_group_size", None),
        None if global_scale is None else float(global_scale),
        _extra_quant_kwargs_key(wrapper),
    )


def _stackable_layout(wrapper: WrapperLinear) -> bool:
    """Whether the wrapper's weight layout is safe for a stacked batch call.

    The stacked call bypasses ``_qdq_weight`` and hands the quant callable a
    3-D ``(N, out, in)`` tensor, so only row-group layouts are stackable:
    ``group_size`` -1 (per-channel) or a positive int never straddle the
    leading-dim module boundary, while ``0`` (per-tensor) would share ONE
    scale across every module in the chunk and 2-D block tuples (e.g. fp8
    ``(128, 128)``) expect a 2-D weight. Activation-only layers (``bits >=
    16``) must finish through their own ``unwrapper`` so their weights stay
    untouched and the act metadata is still processed.
    """
    layer = wrapper.orig_layer
    bits = getattr(layer, "bits", None)
    if bits is None or int(bits) >= 16:
        return False
    group_size = getattr(layer, "group_size", None)
    if isinstance(group_size, bool) or not isinstance(group_size, int) or group_size == 0:
        return False
    return group_size == -1 or group_size > 0


def swap_wrapper_callables_to_eager(wrapper: WrapperLinear) -> Callable[[], None] | None:
    """Point compiled wrapper callables at their eager originals.

    Returns a restore fn or None. ``unwrapper({})`` may call BOTH the
    (possibly torch.compile-wrapped) weight_quant_func and -- for
    act-quantized layers, act_bits <= 8 -- the compiled act_quant_func (the
    act tail's first-ever call in the iters=0 lane); on worker threads
    either first call would race dynamo's trace lock, exactly like the
    stacked path.
    """
    restores = []

    for attr in ("weight_quant_func", "act_quant_func"):
        fn = getattr(wrapper, attr, None)
        if fn is None:
            continue
        orig = getattr(fn, "_torchdynamo_orig_callable", None)
        if orig is None:
            continue
        setattr(wrapper, attr, orig)
        restores.append((attr, fn))

    if not restores:
        return None

    def _restore() -> None:
        for attr, fn in restores:
            setattr(wrapper, attr, fn)

    return _restore


def _unwrap_with_cpu_fallback(w: WrapperLinear, name: str) -> torch.nn.Module:
    """unwrapper({}) under the serial lane's OOM->CPU contract.

    The per-module search can OOM on the device exactly like the serial
    path's own search (base.py); there the run falls back to CPU instead of
    crashing. The batched lane previously called unwrapper({}) unguarded --
    a module that OOMed took the whole run down, violating the progressive
    batched -> per-module -> CPU ladder.

    The rebuilt wrapper keeps the failed wrapper's optimized-RTN and NeuQI
    settings: the fallback must only change WHERE the search runs, not
    WHICH search runs (the WrapperLinear defaults would silently downgrade
    an optimized or NeuQI layer to plain RTN).
    """
    try:
        return w.unwrapper({})
    except torch.OutOfMemoryError:
        dump_oom_tensor_census_("rtn batched unwrapper")
        logger.warning("[rtn-batch] per-module search OOM for %s; falling back to CPU", name)
        layer = w.orig_layer if hasattr(w, "orig_layer") else w
        layer = layer.to("cpu")
        layer = WrapperLinear(
            layer,
            enable_minmax_tuning=False,
            enable_norm_bias_tuning=False,
            enable_round_tuning=False,
            enable_torch_compile=False,
            disable_opt_rtn=getattr(w, "disable_opt_rtn", True),
            enable_neuqi=getattr(w, "enable_neuqi", False),
            iters=0,
        )
        return layer.unwrapper({})


@torch.no_grad()
def run_batched_rtn_search(
    model: torch.nn.Module, staged: list[tuple[str, WrapperLinear]], max_batch: int | None = None
) -> None:
    """Finish deferred zero-shot wrappers on stacked same-shape batches.

    Args:
        model: The model tree (for set_module re-attachment).
        staged: list of (layer_name, wrapper) with the search deferred.
        max_batch: optional explicit chunk size.

    Every staged module is finished by this function itself -- grouped and
    stacked, singleton-per-module, or via the per-module OOM fallback -- so
    the caller never has leftover wrappers to finish.
    """
    entries = []
    for layer_name, wrapper in staged:
        weight = _staged_weight(wrapper)
        imatrix = _staged_imatrix(wrapper, weight)
        entries.append({"name": layer_name, "w": wrapper, "weight": weight, "im": imatrix})

    device_groups = OrderedDict()
    for e in entries:
        device_groups.setdefault(str(e["weight"].device), []).append(e)

    # Offload pass: the searches are weight-local, so chunks may run on any
    # device with headroom (the zero-shot lane leaves every non-home GPU idle).
    # Chunks are assigned round-robin over viable worker devices; the weights
    # and imatrices move to the worker for the stacked call and results write
    # back cross-device through _apply_qdq.
    plan_entries = []  # (home_device, per-module chain, sweep budget, max modules, group)
    for dev, batch in device_groups.items():
        by_key = OrderedDict()
        for e in batch:
            by_key.setdefault(_staged_key(e["w"], e["weight"], e["im"]), []).append(e)
        for group in by_key.values():
            fn0 = group[0]["w"].weight_quant_func
            elements = group[0]["weight"].numel() + (group[0]["im"].numel() if group[0]["im"] is not None else 0)
            per_module = elements * 4 * 4  # the fp32 copy chain per module
            max_n = 1024 if max_batch is None else max(1, max_batch)
            elem_cap = _wrap_batch_max_elems()
            if elem_cap is not None:
                max_n = min(max_n, max(1, elem_cap // max(elements, 1)))
            # layouts that must not share a stacked call (per-tensor scales,
            # 2-D block groups, activation-only layers) plan as singletons:
            # _run_chunk finishes each through the module's own unwrapper,
            # exactly like the serial lane
            if not _stackable_layout(group[0]["w"]):
                max_n = 1
            # singleton and non-stackable groups plan like any other group:
            # the RTN lane runs them through the same per-module path, and a
            # roomy idle device may still take them
            plan_entries.append((dev, per_module, search_sweep_budget(fn0), max_n, group))

    # capacity-weighted planning: chunks form directly on the device that
    # runs them, sized to its tracked headroom (see _plan_capacity_chunks)
    planned = _plan_capacity_chunks(plan_entries)

    buckets = OrderedDict()
    for worker, _home, chunk in planned:
        buckets.setdefault(str(worker), []).append(chunk)

    # per-worker calls and search time for the summary line
    summary = SearchSummary()
    summary.offloaded = sum(1 for worker, home, _chunk in planned if worker != home)

    def _worker_of(chunk: list[dict[str, Any]]) -> str:
        for wk, cs in buckets.items():
            if any(c is chunk for c in cs):
                return str(wk)
        return str(chunk[0]["weight"].device)  # not found (should not happen): stay home

    def _chunk_working_set(chunk: list[dict[str, Any]]) -> int:
        e0 = chunk[0]
        elements = e0["weight"].numel() + (e0["im"].numel() if e0["im"] is not None else 0)
        from auto_round.algorithms.quantization.search_dispatch import search_transient_bytes

        return search_transient_bytes(e0["w"].weight_quant_func, elements * len(chunk))

    @torch.no_grad()
    def _run_chunk_guarded(chunk: list[dict[str, Any]], worker: str, threaded: bool = False) -> None:
        """_run_chunk with an execution-time headroom check: halve an oversized
        chunk before running it (plan-time estimates can drift)."""
        if len(chunk) > 1:
            from auto_round.utils.device import probe_usable_bytes

            usable = probe_usable_bytes(str(worker))
            if usable is not None and usable - 512 * 2**20 < _chunk_working_set(chunk):
                mid = len(chunk) // 2
                _run_chunk_guarded(chunk[:mid], worker, threaded)
                _run_chunk_guarded(chunk[mid:], worker, threaded)
                return
        _run_chunk(chunk, worker, threaded)

    @torch.no_grad()
    def _run_chunk(chunk: list[dict[str, Any]], worker: str, threaded: bool = False) -> None:
        st = summary.device(worker)
        st["modules"] += len(chunk)
        if len(chunk) > 1:
            st["batches"] += 1
        else:
            st["singletons"] += 1
        w0 = chunk[0]["w"]
        dev = str(chunk[0]["weight"].device)
        worker = str(worker)
        if len(chunk) == 1 and (not _search_fn_stackable(w0.weight_quant_func) or not _stackable_layout(w0)):
            # non-stackable callable or non-stackable layout: the module's
            # own serial path, on the weight's device (the stacked call is
            # unsafe for it)
            _restore = swap_wrapper_callables_to_eager(chunk[0]["w"]) if (threaded or worker != dev) else None
            try:
                layer = _unwrap_with_cpu_fallback(chunk[0]["w"], chunk[0]["name"])
            finally:
                if _restore is not None:
                    _restore()
            set_module(model, chunk[0]["name"], layer)
            return
        # stackable chunks of ANY size (including 1-row: singleton groups or
        # max_batch=1 per-module spreading) ride the stacked machinery, which
        # moves weights+imatrix to the assigned worker and writes results
        # back through finalize -- identical search semantics for
        # row-independent fns
        weights = [e["weight"] for e in chunk]
        ims = [e["im"] for e in chunk]
        fn = w0.weight_quant_func
        if worker != dev or threaded:
            # the compiled path runs slower than eager for this bandwidth-bound
            # grid search; eager also sidesteps the cross-thread dynamo trace
            # race entirely -- including for home-staying chunks, which the
            # multi-device path also executes on worker threads
            fn = getattr(fn, "_torchdynamo_orig_callable", None) or fn
        try:
            # materialization is inside the guard on purpose: an OOM while
            # moving or stacking the batch must take the same per-module
            # ladder as an OOM inside the stacked call, not abort the run
            if worker != dev:
                weights = [w.to(worker) for w in weights]
                ims = [im.to(worker) if im is not None else None for im in ims]
            stacked_w = torch.stack(weights)
            stacked_im = None
            if ims and ims[0] is not None:
                _gs = w0.orig_layer.group_size
                stacked_im = torch.stack([_expand_imatrix(im, w, _gs) for im, w in zip(ims, weights)])
            kwargs = w0._quant_call_kwargs(
                torch.tensor(0.0), torch.tensor(1.0), torch.tensor(1.0), imatrix_override=stacked_im
            )
            kwargs = _relocate_tensor_kwargs(kwargs, str(stacked_w.device))
            qdq, scale, zp = fn(stacked_w, **kwargs)
        except torch.OutOfMemoryError:
            # release the partial batch before the retry: the ladder needs
            # the headroom the failed materialization consumed
            weights = ims = stacked_w = stacked_im = kwargs = None
            logger.warning(
                "[rtn-batch] stacked search OOM (%d modules); finishing this chunk per-module "
                "(shrink batches with AR_SEARCH_BATCH_GB or disable with AR_DISABLE_BATCHED_SEARCH=1)",
                len(chunk),
            )
            dump_oom_tensor_census_("rtn batched search")
            for e in chunk:
                _restore = swap_wrapper_callables_to_eager(e["w"]) if (threaded or worker != dev) else None
                try:
                    layer = _unwrap_with_cpu_fallback(e["w"], e["name"])
                finally:
                    if _restore is not None:
                        _restore()
                set_module(model, e["name"], layer)
            return
        n = len(chunk)
        scale_parts = _split_leading(scale, n)
        zp_parts = _split_leading(zp, n)
        import transformers

        for i, e in enumerate(chunk):
            res = qdq[i]
            w = e["w"]
            # mirror the serial _qdq_weight output contract: cast back to the
            # stored dtype and restore the HF Conv1D [in, out] layout (staging
            # transposed it)
            res = res.to(w.orig_layer.weight.dtype)
            if type(w.orig_layer) == transformers.pytorch_utils.Conv1D:
                res = res.t()
            # route the write-back through unwrapper({}) with the precomputed
            # search result injected: _apply_qdq alone would skip the
            # unwrapper tail (bias/meta update, static-act rescale, act
            # metadata, WrapperWALayer attachment) and leave act-quantized
            # layers (act_bits <= 8) silently inconsistent with the serial
            # path. The injection short-circuits _qdq_weight's recompute so
            # no compiled weight_quant_func runs here; the act tail may still
            # call its compiled act_quant_func, so swap BOTH callables to
            # eager on worker threads before unwrapping.
            _restore = swap_wrapper_callables_to_eager(w) if (threaded or worker != dev) else None
            try:
                w._presolved_qdq = (res, scale_parts[i], zp_parts[i])
                layer = w.unwrapper({})
            finally:
                if _restore is not None:
                    _restore()
            set_module(model, e["name"], layer)

    def _run_chunks(wk: str, cs: list[list[dict[str, Any]]]) -> None:
        with summary.timed(wk):
            for c in cs:
                _run_chunk_guarded(c, _worker_of(c), threaded=True)

    if len(buckets) > 1:
        keyed = OrderedDict()
        for wk, cs in buckets.items():
            keyed.setdefault(str(wk), []).append((len(keyed), (str(wk), cs)))
        run_items_by_device(keyed, lambda _idx, item: _run_chunks(item[0], item[1]))
    else:
        for wk, cs in buckets.items():
            _run_chunks(str(wk), cs)

    if summary.devices:
        summary.finish().log("[rtn-batch]")


def _relocate_tensor_kwargs(kwargs: dict[str, Any], device: str) -> dict[str, Any]:
    """Move accelerator-valued kwargs onto the compute device.

    The weight-local searches normally run on the weight's own device, so
    tensor kwargs (e.g. an nv-fp4 ``global_scale``) live wherever the layer
    lives; under worker offload the stacked call runs elsewhere and a
    left-behind operand would fault at the first cross-device op (the mapped-
    lane init_scale crash class). CPU scalars broadcast and are left alone.
    """
    for key, val in kwargs.items():
        if isinstance(val, torch.Tensor) and val.device.type != "cpu" and str(val.device) != device:
            kwargs[key] = val.to(device)
    return kwargs


def _split_leading(result: Any, n: int) -> list[Any]:
    """Split a quant result along the (possibly flattened) leading batch dim.

    Stacked calls may return per-module rows flattened into the leading dim
    (e.g. scale ``[N*out, 1]``) or keep a batch dim (``[N, out, groups]``);
    both reshape to ``[N, -1, ...]`` identically. Scalars pass through.
    Dicts are split recursively: GGUF K-quant funcs return scale/zp as dicts
    (``{"scale": ..., "d_scale": ...}``) whose tensor values carry the
    flattened batch dim -- sharing the whole dict per module would give every
    layer N-times the expected metadata (caught at GGUF export validation).
    Non-tensor dict values (e.g. ``None``) are shared as-is.
    """
    if isinstance(result, dict):
        split_values = {key: _split_leading(value, n) for key, value in result.items()}
        return [{key: values[i] for key, values in split_values.items()} for i in range(n)]
    if not isinstance(result, torch.Tensor):
        return [result] * n
    if result.dim() == 0 or result.numel() == 1:
        return [result] * n
    return result.reshape(n, -1, *result.shape[1:])