PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""QAD quantizer: whole-network joint distillation from an offline teacher cache."""
from __future__ import annotations
import hashlib
import json
import math
import os
import torch
from auto_round import envs
from auto_round.algorithms.quantization.qad.config import QADConfig
from auto_round.algorithms.quantization.qad.teacher_cache import TeacherTopKCache
from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer
from auto_round.algorithms.quantization.sign_round.sign_sgd import SignSGD
from auto_round.algorithms.registry import register_pipeline_member
from auto_round.logger import logger
from auto_round.utils import check_is_cpu, clear_memory, htcore, is_hpex_available
from auto_round.utils.device_manager import device_manager
try: # CUDA-only fast path; falls back to the chunked PyTorch implementation.
import triton
import triton.language as tl
_HAS_TRITON = True
except Exception: # pragma: no cover - triton is optional
_HAS_TRITON = False
def _streaming_logsumexp(s: torch.Tensor, tau: float, vocab_chunk: int) -> torch.Tensor:
"""Numerically-stable ``logsumexp(s/τ)`` over dim -1, streamed in vocab chunks.
PyTorch fallback used when Triton / CUDA is unavailable. Keeps a running max
and running sum so the largest temporary is ``[M, chunk]`` instead of the
full ``[M, vocab]`` ``exp`` (the online-softmax trick that lets Unsloth's
chunked kernel handle 256K-vocab models).
"""
M, V = s.shape
running_max = torch.full((M,), float("-inf"), device=s.device, dtype=torch.float32)
running_sum = torch.zeros(M, device=s.device, dtype=torch.float32)
for c0 in range(0, V, vocab_chunk):
c1 = min(c0 + vocab_chunk, V)
sc = s[:, c0:c1].float() / tau # [M, cw]
chunk_max = sc.max(dim=-1).values # [M]
new_max = torch.maximum(running_max, chunk_max)
running_sum = running_sum * torch.exp(running_max - new_max) + torch.exp(sc - new_max.unsqueeze(1)).sum(dim=-1)
running_max = new_max
return running_max + torch.log(running_sum)
_QAD_TRITON_BLOCK = 8192 # register-friendly vocab tile for the Triton kernels
if _HAS_TRITON:
@triton.jit
def _qad_logsumexp_kernel(
logits_ptr,
logz_ptr,
stride_b,
stride_t,
S1,
VOCAB,
INV_TAU,
BLOCK: tl.constexpr,
):
"""One program per shifted row: online ``logsumexp(s/τ)`` over the vocab.
Row ``r`` maps to token ``(b, t) = (r // S1, r % S1)`` so the kernel reads
straight from the ``[B, seq, vocab]`` logits without a reshape/copy of the
shifted slice. The ``exp`` tile is at most ``[BLOCK]`` wide.
"""
row = tl.program_id(0)
b = row // S1
t = row % S1
base = logits_ptr + b.to(tl.int64) * stride_b + t.to(tl.int64) * stride_t
m = -float("inf")
acc = 0.0
for start in range(0, VOCAB, BLOCK):
offs = start + tl.arange(0, BLOCK)
msk = offs < VOCAB
x = tl.load(base + offs, mask=msk, other=-float("inf")).to(tl.float32) * INV_TAU
cmax = tl.max(x, 0)
new_m = tl.maximum(m, cmax)
acc = acc * tl.exp(m - new_m) + tl.sum(tl.exp(x - new_m), 0)
m = new_m
tl.store(logz_ptr + row, m + tl.log(acc))
@triton.jit
def _qad_softmax_kernel(
logits_ptr,
out_ptr,
logz_ptr,
prob_ptr,
mask_ptr,
scale_ptr,
stride_b,
stride_t,
out_stride_b,
out_stride_t,
S1,
VOCAB,
INV_TAU,
FULL: tl.constexpr,
BLOCK: tl.constexpr,
):
"""Write ``softmax(s/τ)`` of one row/vocab-block straight into ``out``.
``out`` is the full ``[B, seq, vocab]`` gradient buffer; only the first
``S1`` positions per batch are written (the last predicts nothing), so the
shifted-out position keeps its zero initialization. No full-vocab softmax
is ever materialized outside this buffer.
"""
row = tl.program_id(0)
blk = tl.program_id(1)
b = row // S1
t = row % S1
offs = blk * BLOCK + tl.arange(0, BLOCK)
msk = offs < VOCAB
x = (
tl.load(
logits_ptr + b.to(tl.int64) * stride_b + t.to(tl.int64) * stride_t + offs,
mask=msk,
other=-float("inf"),
).to(tl.float32)
* INV_TAU
)
logz = tl.load(logz_ptr + row)
y = tl.exp(x - logz)
if FULL:
target = tl.load(prob_ptr + row.to(tl.int64) * VOCAB + offs, mask=msk, other=0).to(tl.float32)
y -= target
valid = tl.load(mask_ptr + row)
scale = tl.load(scale_ptr).to(tl.float32)
y = tl.where(valid, y * scale, 0.0)
tl.store(out_ptr + b.to(tl.int64) * out_stride_b + t.to(tl.int64) * out_stride_t + offs, y, mask=msk)
def _triton_logsumexp(logits: torch.Tensor, tau: float) -> torch.Tensor:
"""Per-row ``logsumexp(s/τ)`` over the shifted logits via Triton -> ``[M]``."""
B, S, V = logits.shape
S1 = S - 1
M = B * S1
logz = torch.empty(M, device=logits.device, dtype=torch.float32)
sb, st, _ = logits.stride()
_qad_logsumexp_kernel[(M,)](logits, logz, sb, st, S1, V, 1.0 / tau, BLOCK=_QAD_TRITON_BLOCK, num_warps=8)
return logz
def _triton_softmax_into(out, logits, logz, tau, target, mask, scale, full_mode) -> None:
"""Store completed dense gradients, casting only after FP32 arithmetic."""
B, S, V = logits.shape
S1 = S - 1
M = B * S1
n_blk = triton.cdiv(V, _QAD_TRITON_BLOCK)
sb, st, _ = logits.stride()
osb, ost, _ = out.stride()
_qad_softmax_kernel[(M, n_blk)](
logits,
out,
logz,
target,
mask,
scale,
sb,
st,
osb,
ost,
S1,
V,
1.0 / tau,
FULL=full_mode,
BLOCK=_QAD_TRITON_BLOCK,
num_warps=8,
)
class _QADDistillLoss(torch.autograd.Function):
"""Memory-lean forward-KL distillation loss, à la Unsloth's fused cross-entropy.
The target is the full-precision teacher's (temperature-scaled) distribution
``p_T = softmax(t/τ)`` -- *not* the hard next-token label. Per position:
L = KL(p_T || p_S) = Σ_k p_T(k)·log p_T(k) − Σ_k p_T(k)·log p_S(k)
= Σ_k p_T log p_T + logsumexp(s/τ) − Σ_k p_T(k)·(s_k/τ)
using ``log p_S(k) = s_k/τ − logsumexp(s/τ)`` and ``Σ_k p_T(k) = 1``. The
labels are only used as a validity mask (``-100`` = ignore). As in
Unsloth's ``Fast_CrossEntropyLoss``:
* the **forward** keeps only the per-row normalizer ``logsumexp`` (``[M]``),
computed by a Triton kernel that streams over the vocabulary (CUDA) or by
the chunked PyTorch fallback;
* the **backward** recomputes ``softmax(s/τ)`` from the saved logits -- again
via a Triton kernel on CUDA -- and writes the gradient
``(softmax(s/τ) − p_T)`` straight into the (sole) output buffer.
The one-hot label of plain cross-entropy is simply replaced by the teacher
distribution ``p_T`` (the teacher Top-K scattered back to vocab, or the full
distribution when ``teacher_index is None``). No full-vocab log-softmax ever
enters the autograd graph, so the retained memory is just the logits (the
model output, unavoidable) plus the ``[M]`` normalizer.
"""
@staticmethod
def forward(ctx, logits, teacher_index, teacher_prob, labels, tau, eps, vocab_chunk):
# logits: [B, seq, vocab]
# teacher_prob: [B, seq, K] (Top-K) or [B, seq, vocab] (full)
# teacher_index: [B, seq, K] (Top-K) or None (full)
# labels: [B, seq] (-100 = ignore)
B, S, V = logits.shape
tau = float(tau)
vocab_chunk = int(vocab_chunk) if vocab_chunk else V
vocab_chunk = max(1, min(vocab_chunk, V))
use_triton = _HAS_TRITON and logits.is_cuda
# The kernels index rows via strides; only the last (vocab) dim must be
# contiguous. Model LM-head logits already are, but be safe.
if use_triton and logits.stride(-1) != 1:
logits = logits.contiguous()
S1 = S - 1
M = B * S1
mask = labels[:, 1:].reshape(M) != -100
n_valid = int(mask.sum().item())
ctx.shape = (B, S, V)
ctx.tau = tau
ctx.eps = float(eps)
ctx.vocab_chunk = vocab_chunk
ctx.n_valid = n_valid
ctx.full_mode = teacher_index is None
ctx.use_triton = use_triton
if n_valid == 0:
ctx.save_for_backward(logits)
return logits.new_zeros(())
# Per-row log-normalizer logsumexp(s/τ) -> [M], float32.
if use_triton:
logz = _triton_logsumexp(logits, tau)
else:
logz = _streaming_logsumexp(logits[:, :-1, :].reshape(M, V), tau, vocab_chunk)
s3 = logits[:, :-1, :] # [B, S1, V] view
logz3 = logz.view(B, S1)
if teacher_index is None:
# Full-distribution teacher (already sums to 1). The [B, S1, chunk]
# product is transient; nothing full-vocab beyond the resident probs.
p_full = teacher_prob[:, :-1, :].float() # [B, S1, V] (already resident)
# FP16 storage does not preserve unit row mass. Both the KL formula
# and its softmax-minus-target derivative require normalized targets.
p_full = p_full / p_full.sum(dim=-1, keepdim=True).clamp_min(ctx.eps)
cross = torch.zeros(B, S1, device=logits.device, dtype=torch.float32)
neg_ent = torch.zeros(B, S1, device=logits.device, dtype=torch.float32)
for c0 in range(0, V, vocab_chunk):
c1 = min(c0 + vocab_chunk, V)
pc = p_full[:, :, c0:c1]
cross += (pc * (s3[:, :, c0:c1].float() / tau)).sum(dim=-1)
neg_ent += torch.xlogy(pc, pc).sum(dim=-1)
ti = logits.new_empty(0)
pT = p_full
else:
K = teacher_prob.shape[-1]
ti = teacher_index[:, :-1, :] # [B, S1, K]
# Cache already holds softmax(topk(t)/τ) over the K set; the
# renormalization is only a guard against storage round-off.
pT = teacher_prob[:, :-1, :].float()
pT = pT / pT.sum(dim=-1, keepdim=True).clamp_min(ctx.eps)
s_sel = s3.gather(-1, ti).float() / tau # [B, S1, K]
cross = (pT * s_sel).sum(dim=-1) # [B, S1]
neg_ent = torch.xlogy(pT, pT).sum(dim=-1) # Σ p_T log p_T
# KL(p_T || p_S) = Σ p_T log p_T − Σ p_T log p_S = neg_ent + logz − cross.
# The teacher-entropy term is constant w.r.t. the student, so the gradient
# is identical to soft-target CE; including it makes the logged loss a true
# KL (0 at a perfect match) instead of KL + H(p_T).
mask3 = mask.view(B, S1)
loss_row = torch.where(mask3, logz3 - cross + neg_ent, torch.zeros_like(logz3))
loss = (tau * tau) * loss_row.sum() / n_valid
ctx.save_for_backward(logits, logz, pT, ti, mask)
return loss
@staticmethod
def backward(ctx, grad_out):
B, S, V = ctx.shape
if ctx.n_valid == 0:
(logits,) = ctx.saved_tensors
return torch.zeros_like(logits), None, None, None, None, None, None
logits, logz, pT, ti, mask = ctx.saved_tensors
tau = ctx.tau
vocab_chunk = ctx.vocab_chunk
S1 = S - 1
grad_logits = torch.zeros_like(logits)
seg = grad_logits[:, :-1, :] # [B, S1, V] view -> in-place writes land in grad_logits
scale = grad_out.float() * (tau / ctx.n_valid)
mask3 = mask.view(B, S1, 1)
logz3 = logz.view(B, S1, 1)
s3 = logits[:, :-1, :]
# Recompute softmax(s/τ) and write it straight into the gradient buffer
# (no full-vocab softmax retained anywhere else).
if ctx.use_triton:
_triton_softmax_into(grad_logits, logits, logz, tau, pT.contiguous(), mask, scale, ctx.full_mode)
else:
for c0 in range(0, V, vocab_chunk):
c1 = min(c0 + vocab_chunk, V)
sc = s3[:, :, c0:c1].float() / tau
grad = torch.exp(sc - logz3)
if ctx.full_mode:
grad -= pT[:, :, c0:c1]
grad = torch.where(mask3, grad * scale, 0.0)
seg[:, :, c0:c1] = grad.to(grad_logits.dtype)
if not ctx.full_mode:
# Overwrite the sparse target positions with their complete FP32
# gradient. Subtracting from an already-cast dense gradient would
# lose near-equal student/teacher probabilities in FP16/BF16.
selected = torch.exp(s3.gather(-1, ti).float() / tau - logz3) - pT
selected = torch.where(mask3, selected * scale, 0.0)
seg.scatter_(2, ti, selected.to(grad_logits.dtype))
return grad_logits, None, None, None, None, None, None
@register_pipeline_member(QADConfig)
class QADQuantizer(SignRoundQuantizer):
"""Whole-model QAD: distill the FP teacher's cached Top-K into the student.
Unlike block-by-block tuning, QAD optimizes the entire network jointly with a
single full-model forward to the LM head (``whole_model_tuning``), so there is
no nblocks fusing and no probing of the modules between the last block and the
LM head:
* The full-precision teacher is forwarded once (offline); only the Top-K
``(index, prob)`` per token is cached to disk.
* All quantizable layers are wrapped and tuned jointly against a
temperature-scaled soft-label cross-entropy over the teacher Top-K, so
neither the teacher model nor full-vocab logits stay resident.
"""
#: Run a single full-model forward instead of the per-block loop.
whole_model_tuning = True
requires_original_weights = True
def __init__(self, config: QADConfig) -> None:
super().__init__(config)
self.qad_top_k = getattr(config, "topk", 32)
self.qad_temperature = getattr(config, "lfq_temperature", 1.0)
self.qad_workspace = getattr(config, "qad_workspace", None)
# Loss mixing weights: total = kd_weight * KD(teacher) + ce_weight * CE(label).
# kd_weight=0 => pure hard-label QAT (teacher cache is skipped);
# ce_weight>0 => mix the ground-truth cross-entropy into the distillation.
self.qad_kd_weight = float(getattr(config, "qad_kd_weight", 1.0))
self.qad_ce_weight = float(getattr(config, "qad_ce_weight", 0.0))
self._qad_use_teacher = self.qad_kd_weight > 0.0
# Optionally train the non-quantized FP params (lm_head/embeddings/norms/
# biases) jointly with the quantization params (QAT-style full tuning).
self.tune_non_quant_params = bool(getattr(config, "tune_non_quant_params", False))
self.non_quant_params_lr = getattr(config, "non_quant_params_lr", None)
# Vocab chunk width for the streaming KD loss (caps the softmax temporary
# at [N, chunk] in both forward and backward). 0/None = whole vocab.
self.qad_loss_vocab_chunk = getattr(config, "qad_loss_vocab_chunk", 32768)
# Recompute block activations in backward instead of storing them.
self.gradient_checkpointing = getattr(config, "gradient_checkpointing", True)
self._teacher_cache: TeacherTopKCache | None = None
# ── optimizer selection (signround / adamw) ───────────────────────────────
def _get_optimizer(self, optimizer):
"""Resolve the QAD optimizer: SignSGD ('signround') or AdamW ('adamw')."""
opt = str(optimizer or "signround").lower()
self._use_adamw = opt in ("adamw", "adam")
if self._use_adamw:
self.momentum = None # AdamW manages momentum internally
return torch.optim.AdamW
return SignSGD
def _get_scaler(self):
if not getattr(self, "_use_adamw", False):
return super()._get_scaler()
scaler = None
if self.model_context.amp and not check_is_cpu(device_manager.device):
from torch.amp import GradScaler
device_type = torch.device(device_manager.device).type
scaler = GradScaler(device_type, init_scale=1024, growth_interval=100000)
return scaler
def _scale_loss_and_backward(self, scaler, loss):
if not getattr(self, "_use_adamw", False):
return super()._scale_loss_and_backward(scaler, loss)
if scaler is not None:
loss = scaler.scale(loss)
loss.backward()
if is_hpex_available():
htcore.mark_step()
return loss
def _step(self, scaler, optimizer, lr_schedule):
if not getattr(self, "_use_adamw", False):
return super()._step(scaler, optimizer, lr_schedule)
if scaler is not None:
scaler.step(optimizer)
optimizer.zero_grad()
lr_schedule.step()
scaler.update()
else:
optimizer.step()
optimizer.zero_grad()
lr_schedule.step()
if is_hpex_available():
htcore.mark_step()
# ── workspace / cache management ──────────────────────────────────────────
def get_calibration_token_ids(self, token_ids, calibration):
"""Use original forward inputs, never the block reconstruction labels."""
samples = getattr(calibration, "model_input_samples", None)
if token_ids is not None and len(token_ids) and isinstance(token_ids[0], dict):
samples = token_ids
target = self.config.nsamples
if target is None:
target = len(token_ids) if token_ids is not None else self.calibration_context.nsamples
if samples is not None and len(samples) >= target:
return samples[:target]
collect = getattr(calibration, "collect_token_ids", None)
if callable(collect):
return collect(target, include_model_inputs=True)
# Custom calibrators may already provide raw tokens, but cannot recover
# the context that a -100 label has destroyed.
if token_ids is not None and any((self._as_batch_ids(ids, "cpu") < 0).any() for ids in token_ids):
raise ValueError("QAD requires original input_ids and attention_mask, not loss-masked labels.")
return list(token_ids)[:target] if token_ids is not None else None
def _resolve_workspace(self) -> str:
ws = self.qad_workspace
# Treat None / NaN / empty as "unset" and fall back to AutoRound's workspace.
unset = ws is None or (isinstance(ws, float) and math.isnan(ws)) or (isinstance(ws, str) and not ws.strip())
if unset:
ws = os.path.join(envs.AR_WORK_SPACE, "qad_teacher_cache")
os.makedirs(ws, exist_ok=True)
return ws
@staticmethod
def _as_batch_ids(ids, device) -> torch.Tensor:
"""Normalize one sample's token ids to a ``[1, seq]`` long tensor on *device*."""
if isinstance(ids, dict):
ids = ids.get("labels", ids["input_ids"])
if not torch.is_tensor(ids):
ids = torch.as_tensor(ids)
ids = ids.to(device=device)
if ids.dim() == 1:
ids = ids.unsqueeze(0)
return ids.long()
@staticmethod
def _strip_accelerate_hooks(model) -> None:
"""Remove accelerate dispatch hooks so a plain full-model forward works."""
if hasattr(model, "hf_device_map") and len(getattr(model, "hf_device_map", {})) > 0:
try:
import accelerate
accelerate.hooks.remove_hook_from_submodules(model)
except Exception: # pragma: no cover - best effort
pass
@staticmethod
def _safe_input_ids(ids: torch.Tensor, vocab_size: int | None) -> torch.Tensor:
"""Return a forward-safe copy of *ids* with out-of-range tokens replaced.
Legacy tensor-only callers can carry the ``-100`` loss-ignore sentinel.
Those are invalid embedding indices, so replace anything ``< 0`` or ``>=
vocab_size`` with ``0``. The original ids are still used as loss labels so
the masked positions are correctly ignored.
"""
safe = ids.clone()
invalid = safe < 0
if vocab_size is not None:
invalid = invalid | (safe >= vocab_size)
if invalid.any():
safe[invalid] = 0
return safe
# @torch.no_grad()
def _forward_logits(self, model, batch_ids, vocab_size: int | None) -> torch.Tensor:
"""Forward original tokens and attention, independently of loss labels.
Tensor-only callers retain the legacy sentinel handling. Calibration
supplies dictionaries so even loss-ignored EOS tokens keep their context.
"""
if isinstance(batch_ids, dict):
input_ids = batch_ids["input_ids"]
attention_mask = batch_ids.get("attention_mask", torch.ones_like(input_ids))
else:
input_ids = batch_ids
attention_mask = (input_ids != -100).long()
safe_ids = self._safe_input_ids(input_ids, vocab_size)
return model(input_ids=safe_ids, attention_mask=attention_mask, use_cache=False).logits
def _load_model_batch(self, sample_indices, token_ids, device):
"""Stack raw inputs/masks separately from the teacher's cached labels."""
samples = [token_ids[int(i)] for i in sample_indices]
if not isinstance(samples[0], dict):
return self._load_ids_batch(sample_indices, token_ids, device)
return {
key: torch.cat([self._as_batch_ids(sample[key], device) for sample in samples], dim=0)
for key in ("input_ids", "attention_mask")
}
@staticmethod
def _enable_gradient_checkpointing(model) -> None:
"""Turn on HF activation checkpointing (``use_reentrant=False``).
``use_reentrant=False`` is required because QAD freezes every non-wrapper
parameter, so the tensor entering the first decoder block
(``inputs_embeds``) does not require grad; the reentrant implementation
would then drop the gradient at the checkpoint boundary and starve the
wrapper params. The saved-tensor-hooks implementation has no such
restriction. No-op when the model does not support checkpointing.
Because the base model is fully frozen, we also call
``enable_input_require_grads()`` (the PEFT/LoRA trick): it hooks the
embedding output to ``requires_grad_(True)`` so the checkpointed region
has a grad-requiring input and the autograd graph stays connected
(otherwise ``loss`` ends up with no ``grad_fn``).
"""
if not getattr(model, "supports_gradient_checkpointing", False):
logger.warning("QAD: model does not support gradient checkpointing; ignoring the flag.")
return
try:
model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
except TypeError: # older transformers without the kwargs argument
model.gradient_checkpointing_enable()
# Keep the graph connected through the frozen embeddings.
enable_input_grads = getattr(model, "enable_input_require_grads", None)
if callable(enable_input_grads):
enable_input_grads()
@staticmethod
def _disable_gradient_checkpointing(model) -> None:
"""Best-effort revert of :meth:`_enable_gradient_checkpointing`."""
fn = getattr(model, "gradient_checkpointing_disable", None)
if callable(fn):
try:
fn()
except Exception: # pragma: no cover - best effort
pass
disable_input_grads = getattr(model, "disable_input_require_grads", None)
if callable(disable_input_grads):
try:
disable_input_grads()
except Exception: # pragma: no cover - best effort
pass
@staticmethod
def _teacher_cache_signature(model, token_ids) -> dict:
"""Fingerprint of the run so a cache is never reused across models/data.
Hash every parameter/buffer and all forward inputs. Config/path alone
cannot distinguish fine-tuned or in-memory teachers. Transfer weight
chunks to CPU so fingerprinting never duplicates a whole GPU model.
"""
cfg = getattr(model, "config", None)
weights = hashlib.sha256()
def update_tensor(digest, name, tensor):
tensor = tensor.detach()
digest.update(f"{name}:{tensor.dtype}:{tuple(tensor.shape)}".encode())
flat = tensor.reshape(-1)
chunk_size = max(1, (16 * 1024 * 1024) // tensor.element_size())
for start in range(0, flat.numel(), chunk_size):
chunk = flat[start : start + chunk_size].cpu().contiguous().view(torch.uint8)
digest.update(chunk.numpy().tobytes())
for name, tensor in model.state_dict().items():
update_tensor(weights, name, tensor)
data_hash = hashlib.sha256()
for index, sample in enumerate(token_ids if token_ids is not None else []):
fields = sample if isinstance(sample, dict) else {"labels": sample}
for key, tensor in sorted(fields.items()):
update_tensor(data_hash, f"{index}:{key}", torch.as_tensor(tensor))
seqlen = None
if token_ids is not None and len(token_ids) > 0:
first = token_ids[0]
if isinstance(first, dict):
first = first["input_ids"]
if torch.is_tensor(first):
seqlen = int(first.reshape(-1).shape[0])
else:
try:
seqlen = len(first)
except TypeError:
seqlen = None
return {
"model_type": getattr(cfg, "model_type", None),
"vocab_size": getattr(cfg, "vocab_size", None),
"name_or_path": getattr(cfg, "_name_or_path", None),
"model_class": f"{type(model).__module__}.{type(model).__qualname__}",
"config": json.dumps(cfg.to_dict(), sort_keys=True, default=str) if hasattr(cfg, "to_dict") else str(cfg),
"weights_sha256": weights.hexdigest(),
"inputs_sha256": data_hash.hexdigest(),
"nsamples": None if token_ids is None else len(token_ids),
"seqlen": seqlen,
}
@torch.no_grad()
def _build_teacher_cache_from_model(self, model, token_ids, device) -> None:
"""Forward the FP model to the LM head once per sample and cache the Top-K.
Uses the real model forward (``model(input_ids).logits``), so no module
between the last block and the LM head needs to be probed. Samples already
on disk are skipped; each sample's logits are freed right after caching.
"""
if self._teacher_cache is not None:
self._teacher_cache.shutdown()
self._teacher_cache = TeacherTopKCache(
self._resolve_workspace(),
self.qad_top_k,
signature=self._teacher_cache_signature(model, token_ids),
temperature=float(self.qad_temperature),
)
cache = self._teacher_cache
vocab_size = getattr(getattr(model, "config", None), "vocab_size", None)
was_training = model.training
model.eval()
n_built = 0
for sample_idx, ids in enumerate(token_ids):
batch_ids = self._as_batch_ids(ids, device)
# Skip only when the cached file was produced by *this* sample; a
# stale file from another model/dataset is rebuilt (overwritten).
if cache.is_valid_for(sample_idx, batch_ids):
continue
inputs = self._load_model_batch([sample_idx], token_ids, device)
logits = self._forward_logits(model, inputs, vocab_size) # [1, seq, vocab]
# Store the original ids (with any -100) so they serve as loss labels.
cache.build_sample(sample_idx, batch_ids, logits)
del logits
n_built += 1
if was_training:
model.train()
if n_built:
logger.info("QAD: cached teacher Top-%d for %d samples at %s", self.qad_top_k, n_built, cache.workspace)
clear_memory()
# ── whole-model tuning entry point ────────────────────────────────────────
@torch.no_grad()
def _calibrate_activations(self, model, token_ids, device) -> None:
"""Prepare activation ranges before wrapping, independently of the teacher cache."""
from auto_round.data_type.utils import register_act_max_hooks, update_block_global_scale_if_needed
from auto_round.utils import set_amax_for_all_moe_layers
handles = register_act_max_hooks(model, self.config.act_data_type)
was_training = model.training
try:
if handles:
model.eval()
vocab_size = getattr(getattr(model, "config", None), "vocab_size", None)
for sample_idx in range(len(token_ids)):
batch_ids = self._load_model_batch([sample_idx], token_ids, device)
self._forward_logits(model, batch_ids, vocab_size)
finally:
for handle in handles:
handle.remove()
model.train(was_training)
if handles:
set_amax_for_all_moe_layers(model, attr_name="act_max")
update_block_global_scale_if_needed(model, self.config.data_type or "int", self.config.group_size or -1)
def quantize_model(self, model, token_ids, device=None) -> dict:
"""Tune jointly, releasing the teacher workspace even if setup fails."""
try:
return self._quantize_model(model, token_ids, device)
finally:
if self._teacher_cache is not None:
self._teacher_cache.shutdown()
def _quantize_model(self, model, token_ids, device=None) -> dict:
"""Tune the whole model jointly against the cached teacher distribution.
Steps: (1) cache the FP teacher Top-K via a real full-model forward,
(2) wrap every quantizable layer, (3) optimize all rounding/min-max
params jointly with a full-model forward → KD loss → backward, and
(4) return the best params for common composer finalization.
"""
import copy
from auto_round.compressors.utils import IndexSampler, collect_best_params
device = device or device_manager.device
if token_ids is None or len(token_ids) == 0:
raise ValueError("QAD requires calibration token_ids but none were provided.")
# Make the FP model resident on the compute device and hook-free.
self._strip_accelerate_hooks(model)
model.to(device)
# A previous distributed block stage can leave placement hints on the
# original modules. This pass is strictly single-device.
for module in model.modules():
if hasattr(module, "tuning_device"):
del module.tuning_device
# Freeze before creating wrapper parameters. Clear any gradients left
# by a preceding stage; FP tuning is selectively re-enabled below.
existing_tuning = {id(p) for module in model.modules() for p in getattr(module, "params", {}).values()}
for parameter in model.parameters():
if id(parameter) not in existing_tuning:
parameter.requires_grad_(False)
parameter.grad = None
# Whole-model tuning bypasses the composer's per-block calibration.
# Run this even when KD is disabled or all teacher samples are cached.
self._calibrate_activations(model, token_ids, device)
# 1) Offline teacher cache from a real full-model forward (no probing).
# Skipped entirely for a pure hard-label QAT run (``--qad_kd_weight 0``).
if self._qad_use_teacher:
self._build_teacher_cache_from_model(model, token_ids, device)
else:
logger.info("QAD: KD weight is 0 -> hard-label QAT only; skipping the teacher cache.")
# 2) Wrap every quantizable layer across the whole model.
quantized_layers, unquantized_layers = self.wrapper_block(
model,
self.enable_minmax_tuning,
self.enable_norm_bias_tuning,
enable_torch_compile=self.compress_context.enable_torch_compile,
device=device,
enable_neuqi=getattr(self.config, "enable_neuqi", False),
)
logger.info("QAD: wrapped %d quantizable layers (%d kept FP).", len(quantized_layers), len(unquantized_layers))
# The wrappers have now copied any warm-start params (e.g. from a prior
# SignRound pass in ``--algs signround,qad``) into their tuning params.
# Report how many layers were warm-started and drop the CPU stash.
n_warm = 0
for _n, m in model.named_modules():
orig = getattr(m, "orig_layer", None)
if orig is not None and getattr(orig, "_ar_warm_start", None):
n_warm += 1
del orig._ar_warm_start
if hasattr(orig, "_ar_warm_start_wrapper_cls"):
del orig._ar_warm_start_wrapper_cls
if n_warm:
logger.info("QAD: warm-started %d layers from the previous block-wise pass.", n_warm)
else:
logger.info("QAD: no warm-start params found; tuning starts from RTN init.")
# 3) Collect trainable params and build the optimizer / lr schedule.
round_params, minmax_params = [], []
for _n, m in model.named_modules():
if hasattr(m, "orig_layer"):
for key in m.params.keys():
(minmax_params if ("min" in key or "max" in key) else round_params).append(m.params[key])
# Optionally add the non-quantized FP params (lm_head/embeddings/norms/
# biases). Exclude the wrapper tuning params and the frozen original
# quantized weights (everything under a wrapper's ``orig_layer``).
fp_params = self._collect_non_quant_params(model) if self.tune_non_quant_params else []
fp_optimizer_params, fp_master_pairs = self._prepare_fp_optimizer_params(fp_params)
if not round_params and not minmax_params and not fp_params:
logger.warning("QAD: no trainable quantization parameters found; nothing to tune.")
if self._teacher_cache is not None:
self._teacher_cache.shutdown()
return {}
base_lr = self.lr if self.lr is not None else (1.0 / self.iters if self.iters > 0 else 1e-3)
minmax_lr = self.minmax_lr if self.minmax_lr is not None else base_lr
extra_kwargs = {} if self.momentum is None else {"momentum": self.momentum}
param_groups = [{"params": round_params, "lr": torch.tensor(float(base_lr))}]
if self.enable_minmax_tuning and minmax_params:
param_groups.append({"params": minmax_params, "lr": torch.tensor(float(minmax_lr))})
if fp_params:
fp_lr = self.non_quant_params_lr if self.non_quant_params_lr is not None else base_lr
param_groups.append({"params": fp_optimizer_params, "lr": torch.tensor(float(fp_lr))})
logger.info(
"QAD: also tuning %d non-quantized FP parameter tensors (lm_head/embeddings/norms/biases) at lr=%.2e.",
len(fp_params),
float(fp_lr),
)
optimizer = self.optimizer(param_groups, lr=torch.tensor(float(base_lr)), weight_decay=0, **extra_kwargs)
if self.lr_scheduler is None:
lr_schedule = torch.optim.lr_scheduler.LinearLR(
optimizer, start_factor=1.0, end_factor=0.0, total_iters=max(1, self.iters)
)
else:
lr_schedule = copy.deepcopy(self.lr_scheduler)
scaler = self._get_scaler()
# 4) Joint tuning loop: full-model forward → KD loss → step.
if self.gradient_checkpointing:
# HF only runs the checkpointed path when the module is in training
# mode, so switch to train(). LLM decoder dropout is 0 by default, so
# the forward stays deterministic; the wrapper params still train via
# STE. Recomputing activations in backward is the whole point here.
self._enable_gradient_checkpointing(model)
model.train()
logger.info("QAD: gradient checkpointing enabled (activations recomputed in backward).")
else:
model.eval() # deterministic forward; wrapper params still train via STE
vocab_size = getattr(getattr(model, "config", None), "vocab_size", None)
nsamples = len(token_ids)
batch_size = min(self.calibration_context.batch_size, nsamples)
grad_accum = max(1, self.gradient_accumulate_steps)
global_bs = min(nsamples, batch_size * grad_accum)
sampler = IndexSampler(nsamples, global_bs)
best_loss = float("inf")
best_iter = -1
best_params: dict = {}
# The best-param snapshot covers every wrapped layer of the whole model
# (``value`` alone is weight-sized), so always keep it on CPU instead of
# duplicating it on the accelerator.
best_cache_device = "cpu"
# Non-quantized FP params are tuned in place on the model, so the best-loss
# snapshot must cover them too (unwrapper only restores the wrapper params).
best_fp_state: list | None = None
from tqdm import tqdm
# Whole-model tuning has no per-block loop, so progress is tracked over
# the joint tuning iterations instead of over blocks.
pbar = tqdm(range(self.iters), desc="QAD tuning")
try:
for it in pbar:
global_indices = sampler.next_batch()
if self._qad_use_teacher:
self._teacher_cache.prefetch(global_indices)
optimizer.zero_grad()
total_loss = 0.0
for start in range(0, len(global_indices), batch_size):
idx = global_indices[start : start + batch_size]
# Cached labels supervise the loss only. Forward the original
# input tokens/masks, including any loss-ignored separators.
if self._qad_use_teacher:
batch_ids, t_index, t_prob = self._load_teacher_batch(idx, device)
else:
batch_ids, t_index, t_prob = self._load_ids_batch(idx, token_ids, device), None, None
model_inputs = self._load_model_batch(idx, token_ids, device)
logits = self._forward_logits(model, model_inputs, vocab_size)
loss = self._qad_loss(logits, t_index, t_prob, batch_ids)
self._scale_loss_and_backward(scaler, loss)
total_loss += loss.item()
# Drop the large per-microbatch logits/graph before the next
# forward so the cached allocator can reuse the blocks.
del logits, loss, batch_ids, model_inputs, t_index, t_prob
# The loss was measured before the optimizer update: snapshot
# that same state, including any jointly tuned FP parameters.
if total_loss < best_loss:
best_loss = total_loss
best_iter = it
if not self.not_use_best_mse:
best_params = collect_best_params(model, best_cache_device)
if fp_params:
best_fp_state = self._snapshot_fp_params(fp_params, best_cache_device)
self._copy_fp_grads_to_masters(fp_master_pairs)
self._step(scaler, optimizer, lr_schedule)
self._copy_fp_masters_to_model(fp_master_pairs)
# Switch: with ``--not_use_best_mse`` keep the LAST iteration's
# params instead of the best-loss ones. Collect them explicitly --
# an empty dict would make ``unwrapper`` fall back to
# value=0/scale=1 (plain RTN), not the tuned state.
if self.not_use_best_mse and it == self.iters - 1:
best_params = collect_best_params(model, best_cache_device)
if fp_params:
best_fp_state = self._snapshot_fp_params(fp_params, best_cache_device)
pbar.set_postfix(loss=f"{total_loss:.3e}", best=f"{best_loss:.3e}@{best_iter}")
logger.debug("QAD iter %d loss %.5e", it, total_loss)
# # Periodically release cached blocks to curb fragmentation during
# # the long whole-model tuning loop (cheap relative to a full step).
# if self.iters <= 1 or (it % 10 == 0):
# clear_memory()
finally:
# 5) Restore algorithm-specific FP params and release the prefetch pool.
# Common wrapper finalization belongs to the composer.
pbar.close()
if best_iter >= 0:
logger.info(
"QAD: best loss %.4e at iter %d/%d%s.",
best_loss,
best_iter,
self.iters,
" (not used: --not_use_best_mse keeps the last iter)" if self.not_use_best_mse else "",
)
if self.gradient_checkpointing:
self._disable_gradient_checkpointing(model)
model.eval()
# Restore the best-loss FP params (tuned in place on the model) before
# baking the quantized weights.
if best_fp_state is not None:
self._restore_fp_params(fp_params, best_fp_state)
if self._teacher_cache is not None:
self._teacher_cache.shutdown()
clear_memory()
return best_params
def _collect_non_quant_params(self, model) -> list:
"""Return the model's non-quantized FP params to train jointly (QAT-style).
Excludes every parameter that belongs to a wrapper: both the tuning
params (``value``/``min_scale``/``max_scale``) and the frozen original
quantized weight/bias (everything under ``wrapper.orig_layer``). What
remains are the parameters of modules kept at full precision -- e.g.
``lm_head``, embeddings, norms and biases -- which are enabled for grad
and returned.
"""
quant_param_ids: set[int] = set()
for _n, m in model.named_modules():
if hasattr(m, "orig_layer"):
for p in m.params.values():
quant_param_ids.add(id(p))
for p in m.orig_layer.parameters(recurse=True):
quant_param_ids.add(id(p))
fp_params = []
for p in model.parameters():
if id(p) not in quant_param_ids:
p.requires_grad_(True)
fp_params.append(p)
return fp_params
@staticmethod
def _prepare_fp_optimizer_params(fp_params):
"""Keep low-precision forward weights but optimize FP32 master copies."""
optimizer_params, pairs = [], []
for param in fp_params:
if param.dtype in (torch.float16, torch.bfloat16):
master = torch.nn.Parameter(param.detach().float().clone())
optimizer_params.append(master)
pairs.append((param, master))
else:
optimizer_params.append(param)
return optimizer_params, pairs
@staticmethod
def _copy_fp_grads_to_masters(pairs):
for param, master in pairs:
master.grad = None if param.grad is None else param.grad.detach().float()
param.grad = None
@staticmethod
@torch.no_grad()
def _copy_fp_masters_to_model(pairs):
for param, master in pairs:
param.copy_(master)
@staticmethod
def _snapshot_fp_params(fp_params: list, device) -> list:
"""Detach-clone the FP params to *device* (CPU) as a best-loss snapshot."""
return [p.detach().to(device, copy=True) for p in fp_params]
@staticmethod
def _restore_fp_params(fp_params: list, snapshot: list) -> None:
"""Copy a snapshot back into the live FP params (in place)."""
for p, saved in zip(fp_params, snapshot):
p.data.copy_(saved.to(p.device))
def _load_ids_batch(self, sample_indices, token_ids, device) -> torch.Tensor:
"""Stack raw calibration ``input_ids`` for a batch (pure-QAT path).
Used when the teacher cache is disabled (``--qad_kd_weight 0``): the
hard-label CE only needs the token ids (with their ``-100`` ignore
sentinels preserved as loss labels), so they are read straight from
``token_ids`` instead of from the on-disk teacher files.
"""
ids_list = [self._as_batch_ids(token_ids[int(i)], device).squeeze(0) for i in sample_indices]
return torch.stack(ids_list, dim=0)
def _load_teacher_batch(self, sample_indices, device):
"""Load and stack cached ``(input_ids, index, prob)`` for a batch.
Returns ``(input_ids, index, prob)`` where ``input_ids`` is ``[B, seq]``,
``index`` is ``[B, seq, K]`` (Top-K mode) or ``None`` (full-distribution
mode), and ``prob`` is ``[B, seq, K]`` / ``[B, seq, vocab]``.
"""
ids_list, idx_list, prob_list = [], [], []
full_mode = False
for i in sample_indices:
input_ids, index, prob = self._teacher_cache.load(int(i), device)
if index is None:
full_mode = True
ids_list.append(input_ids)
idx_list.append(index)
prob_list.append(prob)
batch_ids = torch.stack(ids_list, dim=0)
teacher_prob = torch.stack(prob_list, dim=0)
teacher_index = None if full_mode else torch.stack(idx_list, dim=0)
return batch_ids, teacher_index, teacher_prob
def _qad_kd_loss(self, student_logits, teacher_index, teacher_prob, labels):
"""Temperature-scaled forward KL against the FP teacher's distribution.
Per next-token position, the loss is ``KL(p_T || p_S)`` with
``p_T = softmax(t/τ)`` from the cached teacher logits and
``p_S = softmax(s/τ)``. When
*teacher_index* is given, the sum is over the teacher's Top-K set (teacher
probs renormalized within it); when it is ``None`` the teacher is the full
vocabulary distribution. Positions with label ``-100`` are ignored.
The heavy lifting is delegated to :class:`_QADDistillLoss`, a custom
autograd Function that -- following Unsloth's fused cross-entropy -- keeps
only the per-row ``logsumexp`` in the graph and recomputes ``softmax`` in
the backward (Triton on CUDA, chunked PyTorch otherwise), so no full-vocab
``log_softmax`` tensor is ever retained.
Args:
student_logits: ``[B, seq, vocab]``.
teacher_index: ``[B, seq, K]`` teacher Top-K token ids, or ``None``
for a full-distribution teacher.
teacher_prob: ``[B, seq, K]`` / ``[B, seq, vocab]`` teacher probs.
labels: ``[B, seq]`` token ids (``-100`` = ignore).
Returns:
Scalar loss tensor (mean over valid next-token positions).
"""
return _QADDistillLoss.apply(
student_logits,
teacher_index,
teacher_prob,
labels,
float(self.qad_temperature),
1e-6,
self.qad_loss_vocab_chunk,
)
def _qad_ce_loss(self, student_logits, labels):
"""Hard-label next-token cross-entropy over the student logits.
This is the standard causal-LM objective: at position ``t`` the target is
the ground-truth token ``labels[t+1]`` (``-100`` positions are ignored).
It is expressed as a one-hot special case of :class:`_QADDistillLoss`
(teacher mass 1.0 on the label, temperature 1.0), so it reuses the same
memory-lean streaming/Triton path as the KD loss and shares the exact
next-token shift and ignore mask -- no extra full-vocab log-softmax is
materialized.
Args:
student_logits: ``[B, seq, vocab]``.
labels: ``[B, seq]`` token ids (``-100`` = ignore).
Returns:
Scalar cross-entropy loss (mean over valid next-token positions).
"""
# Target for student position ``t`` is the token at ``t+1``; align a
# one-hot teacher index to the same ``[:, :-1]`` slice _QADDistillLoss
# consumes. Invalid (-100) targets are set to 0 for the gather and are
# dropped anyway by the shared ``labels[:, 1:] != -100`` mask.
next_labels = torch.full_like(labels, -100)
next_labels[:, :-1] = labels[:, 1:]
ce_index = next_labels.clamp_min(0).unsqueeze(-1) # [B, seq, 1]
ce_prob = torch.ones_like(ce_index, dtype=torch.float32) # one-hot mass
return _QADDistillLoss.apply(
student_logits,
ce_index,
ce_prob,
labels,
1.0, # hard-label CE uses no temperature smoothing
1e-6,
self.qad_loss_vocab_chunk,
)
def _qad_loss(self, student_logits, teacher_index, teacher_prob, labels):
"""Combined QAD objective: ``kd_weight * KD + ce_weight * CE``.
* ``kd_weight > 0`` adds the teacher-distribution distillation term
(:meth:`_qad_kd_loss`); *teacher_index*/*teacher_prob* come from the
offline cache and may be ``None`` when distillation is disabled.
* ``ce_weight > 0`` adds the hard-label cross-entropy (:meth:`_qad_ce_loss`),
turning QAD into quantization-aware training (QAT).
At least one weight is positive (validated by the config).
"""
loss = None
if self.qad_kd_weight > 0.0 and teacher_prob is not None:
loss = self.qad_kd_weight * self._qad_kd_loss(student_logits, teacher_index, teacher_prob, labels)
if self.qad_ce_weight > 0.0:
ce = self.qad_ce_weight * self._qad_ce_loss(student_logits, labels)
loss = ce if loss is None else loss + ce
if loss is None: # defensive: config guarantees at least one active term
raise RuntimeError("QAD loss has no active term; set --qad_kd_weight or --qad_ce_weight > 0.")
return loss