PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat: export SVDQuant NVFP4 checkpoints for vLLM-Omni (#2445)
Wang, Chang7c0aed071e81
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.pyindex 6605e5b7f..00aaa4e84 100644--- a/auto_round/algorithms/composer.py+++ b/auto_round/algorithms/composer.py@@ -474,6 +474,11 @@ class AlgorithmComposer:for pre in self.preprocessors:pre.pre_quantize_block(block_ctx)+ act_dynamic = self.scheme.act_dynamic if (self.scheme and self.scheme.act_dynamic is not None) else True+ data_type = self.scheme.data_type if self.scheme else "int"+ group_size = self.scheme.group_size if self.scheme else -1+ act_data_type = self.scheme.act_data_type if self.scheme else data_type+reference_next_input = None# ── Step 3: Quantizer calibration (act_max, imatrix, etc.) ─────────────if fp_inputs is not None:@@ -494,18 +499,20 @@ class AlgorithmComposer:# ── Step 3.5: MoE scale alignment + global scale update ─────────────────# Must run after calibration hooks (act_max collected) and before quantize_block.- act_dynamic = self.scheme.act_dynamic if (self.scheme and self.scheme.act_dynamic is not None) else True- data_type = self.scheme.data_type if self.scheme else "int"- group_size = self.scheme.group_size if self.scheme else -1- act_data_type = self.scheme.act_data_type if self.scheme else data_typeif act_data_type is not None or not act_dynamic:from auto_round.compressors.utils import is_nv_fp- from auto_round.data_type.utils import update_block_global_scale_if_neededfrom auto_round.utils import set_amax_for_all_moe_layersif is_nv_fp(act_data_type) or not act_dynamic:set_amax_for_all_moe_layers(block, attr_name="act_max")- update_block_global_scale_if_needed(block, data_type, group_size)++ # SVDQuant may have replaced the original linears during pre-quantize.+ # Update NVFP4 scales after that transformation even on the zero-shot+ # path, where fp_inputs is None and the calibration block above is skipped.+ if act_data_type is not None or not act_dynamic:+ from auto_round.data_type.utils import update_block_global_scale_if_needed++ update_block_global_scale_if_needed(block, data_type, group_size)if q_inputs is not None and fp_inputs is not q_inputs:# Release the fp pool at its last use (before tuning): the q pool isdiff --git a/auto_round/algorithms/transforms/svdquant/apply.py b/auto_round/algorithms/transforms/svdquant/apply.pyindex fffff0beb..a9182f5b9 100644--- a/auto_round/algorithms/transforms/svdquant/apply.py+++ b/auto_round/algorithms/transforms/svdquant/apply.py@@ -136,6 +136,7 @@ class SVDQuantTransform(BasePreprocessor):def __init__(self, config: SVDQuantConfig) -> None:super().__init__(config)self._svd_driver = None+ self._grouping_backend = Noneself._configured_block_names: tuple[str, ...] = ()self._block_groups: dict[str, list[SmoothSearchGroup]] = {}self._smooth_calibration: dict[str, SmoothGroupCalibration] = {}@@ -167,12 +168,19 @@ class SVDQuantTransform(BasePreprocessor):def prepare_run(self, composer=None) -> None:self._block_groups.clear()+ formats = getattr(self.compress_context, "formats", None) or ()+ self._grouping_backend = "omni" if any(fmt.format_name == "svdquant_omni" for fmt in formats) else Noneif self.model is None:return- self._resolve_model_adapter(self.model)+ if self._grouping_backend == "omni":+ self._target_modules = self.config.target_modules+ else:+ self._resolve_model_adapter(self.model)for block_name in self._configured_block_names:block = self.model.get_submodule(block_name)- self._block_groups[block_name] = discover_svdquant_groups(block, self._is_target)+ self._block_groups[block_name] = discover_svdquant_groups(+ block, self._is_target, grouping_backend=self._grouping_backend+ )logger.info("SVDQuant: resolved %d projection groups across %d blocks.",sum(len(groups) for groups in self._block_groups.values()),@@ -217,12 +225,13 @@ class SVDQuantTransform(BasePreprocessor):def register_fp_input_forward_hooks(self, block) -> list:if not self.config.smooth_enabled:return []- self._resolve_model_adapter(self.model, block)+ if self._grouping_backend != "omni":+ self._resolve_model_adapter(self.model, block)self._clear_smooth_calibration()block_name = str(getattr(block, "global_name", ""))groups = self._block_groups.get(block_name)if groups is None:- groups = discover_svdquant_groups(block, self._is_target)+ groups = discover_svdquant_groups(block, self._is_target, grouping_backend=self._grouping_backend)self._block_groups[block_name] = groupsself._smooth_calibration = {group.key: SmoothGroupCalibration(group, self.config.smooth_max_calibration_calls) for group in groups@@ -256,10 +265,11 @@ class SVDQuantTransform(BasePreprocessor):raise ValueError(f"SVDQuant requires one block at a time, got {ctx.block_names!r}.")block_name = ctx.block_nameblock = ctx.model.get_submodule(block_name)- self._resolve_model_adapter(ctx.model, block)+ if self._grouping_backend != "omni":+ self._resolve_model_adapter(ctx.model, block)groups = self._block_groups.get(block_name)if groups is None:- groups = discover_svdquant_groups(block, self._is_target)+ groups = discover_svdquant_groups(block, self._is_target, grouping_backend=self._grouping_backend)self._block_groups[block_name] = groupsif self.config.smooth_enabled:@@ -619,6 +629,8 @@ class SVDQuantTransform(BasePreprocessor):)lora_down.weight.copy_(down)lora_up.weight.copy_(up)+ lora_down.requires_grad_(projection.weight.requires_grad)+ lora_up.requires_grad_(projection.weight.requires_grad)self._mark_unquantized(lora_down)self._mark_unquantized(lora_up)self._copy_quant_attrs(projection, residual, suffix=".residual_linear")@@ -700,8 +712,11 @@ class SVDQuantTransform(BasePreprocessor):device=module.weight.device,)residual.weight.copy_(weight.to(module.weight.dtype))+ # Preserve freezing: cached NVFP4 scales must not acquire an autograd graph.+ residual.weight.requires_grad_(module.weight.requires_grad)if bias is not None:residual.bias.copy_(bias.detach().to(module.weight.dtype))+ residual.bias.requires_grad_(module.bias.requires_grad)return residual@staticmethod@@ -711,7 +726,9 @@ class SVDQuantTransform(BasePreprocessor):@staticmethoddef _copy_quant_attrs(src: torch.nn.Module, dst: torch.nn.Module, suffix: str) -> None:- for attr in _SCHEME_ATTRS | _RUNTIME_QUANT_ATTRS:+ # The residual has different weights (and may be smoothed), so its+ # NVFP4 global scale must be calculated again before quantization.+ for attr in (_SCHEME_ATTRS | _RUNTIME_QUANT_ATTRS) - {"weight_global_scale"}:if hasattr(src, attr):setattr(dst, attr, getattr(src, attr))if getattr(dst, "bits", None) == 4 and getattr(dst, "data_type", None) in _MXFP4_ALIASES:diff --git a/auto_round/algorithms/transforms/svdquant/smooth_adapters/__init__.py b/auto_round/algorithms/transforms/svdquant/smooth_adapters/__init__.pyindex 02d79ab0e..0c5f9c2b9 100644--- a/auto_round/algorithms/transforms/svdquant/smooth_adapters/__init__.py+++ b/auto_round/algorithms/transforms/svdquant/smooth_adapters/__init__.py@@ -22,11 +22,16 @@ from auto_round.algorithms.transforms.svdquant.smooth_adapters.base import (generic_linear_groups,)from auto_round.algorithms.transforms.svdquant.smooth_adapters.flux import discover_flux_groups, supports_flux_block+from auto_round.algorithms.transforms.svdquant.smooth_adapters.omni import discover_omni_groupsfrom auto_round.algorithms.transforms.svdquant.smooth_adapters.sdxl import discover_sdxl_groups, supports_sdxl_block-def discover_svdquant_groups(block: torch.nn.Module, is_target: TargetPredicate) -> list[SmoothSearchGroup]:+def discover_svdquant_groups(+ block: torch.nn.Module, is_target: TargetPredicate, *, grouping_backend: str | None = None+) -> list[SmoothSearchGroup]:"""Discover shared-input projection groups for one quantization block."""+ if grouping_backend == "omni":+ return discover_omni_groups(block, is_target)if supports_flux_block(block):return discover_flux_groups(block, is_target)if supports_sdxl_block(block):diff --git a/auto_round/algorithms/transforms/svdquant/smooth_adapters/omni.py b/auto_round/algorithms/transforms/svdquant/smooth_adapters/omni.pynew file mode 100644index 000000000..2390bfa6e--- /dev/null+++ b/auto_round/algorithms/transforms/svdquant/smooth_adapters/omni.py@@ -0,0 +1,62 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Shared-input QKV grouping for vLLM-Omni's fused SVDQuant linear layers."""++import torch++from auto_round.algorithms.transforms.svdquant.smooth_adapters.base import (+ SmoothSearchGroup,+ TargetPredicate,+ generic_linear_groups,+ module_global_name,+)++_QKV_NAMES = (("to_q", "to_k", "to_v"), ("q_proj", "k_proj", "v_proj"))+++def discover_omni_groups(block: torch.nn.Module, is_target: TargetPredicate) -> list[SmoothSearchGroup]:+ """Group explicit self-attention QKV without model-class-specific layouts."""+ groups = []+ consumed = set()+ for local_name, attention in block.named_modules():+ if getattr(attention, "is_cross_attention", None) is not False:+ continue+ projection_names = next((names for names in _QKV_NAMES if all(hasattr(attention, name) for name in names)), ())+ if not projection_names:+ continue+ paths = tuple(f"{local_name}.{name}" if local_name else name for name in projection_names)+ projections = tuple(getattr(attention, name) for name in projection_names)+ if not all(+ isinstance(module, torch.nn.Linear) and id(module) not in consumed and is_target(path, module)+ for path, module in zip(paths, projections)+ ):+ continue+ if len({module.in_features for module in projections}) != 1 or len({id(module) for module in projections}) != 3:+ continue+ names = tuple(module_global_name(block, path) for path in paths)+ groups.append(+ SmoothSearchGroup(+ key=module_global_name(block, f"{local_name}.qkv" if local_name else "qkv"),+ projection_names=names,+ projections=projections,+ projection_input_key=names[0],+ projection_input_module=projections[0],+ evaluation_input_key=module_global_name(block, local_name),+ evaluation_module=attention,+ )+ )+ consumed.update(id(module) for module in projections)+ groups.extend(generic_linear_groups(block, is_target, consumed=consumed))+ return groupsdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex e697a1d43..4c777606d 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -594,7 +594,7 @@ class _CompressorBuilder:configs_for_routing = alg_configs if isinstance(alg_configs, list) else [alg_configs]preprocessor_configs, _, quant_config = _resolve_quant_config_for_routing(configs_for_routing)is_svdquant = any(type(config).__name__ == "SVDQuantConfig" for config in preprocessor_configs)- if is_svdquant:+ if is_svdquant and "svdquant_omni" not in {name.strip().lower() for name in (format or "").split(",")}:format = "svdquant_nunchaku"# Any preprocessor that requires calibration data (e.g. AWQ, SVDQuantdiff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 27a1a4212..64f821cf2 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -520,6 +520,10 @@ def update_fused_layer_global_scales(def _update_global_scales(modules: list[Module]):"""Update global scales for a list of modules."""+ from auto_round.algorithms.transforms.svdquant.wrapper import SVDQuantLinear++ # SVDQuant keeps the quantized projection inside its residual branch.+ modules = [module.residual_linear if isinstance(module, SVDQuantLinear) else module for module in modules]scales = _collect_scales(modules)if not scales:returndiff --git a/auto_round/export/formats/backends/__init__.py b/auto_round/export/formats/backends/__init__.pyindex e74dbf3e1..14c509aff 100644--- a/auto_round/export/formats/backends/__init__.py+++ b/auto_round/export/formats/backends/__init__.py@@ -21,6 +21,7 @@ from auto_round.export.formats.backends.auto_gptq import AutoGPTQFormatfrom auto_round.export.formats.backends.llm_compressor import LLMCompressorFormatfrom auto_round.export.formats.backends.mlx import MLXFormatfrom auto_round.export.formats.backends.svdquant_nunchaku import SVDQuantNunchakuFormat+from auto_round.export.formats.backends.svdquant_omni import SVDQuantOmniFormat__all__ = ["AutoAWQFormat",@@ -32,4 +33,5 @@ __all__ = ["LLMCompressorFormat","MLXFormat","SVDQuantNunchakuFormat",+ "SVDQuantOmniFormat",]diff --git a/auto_round/export/formats/backends/svdquant_omni.py b/auto_round/export/formats/backends/svdquant_omni.pynew file mode 100644index 000000000..8c130ba38--- /dev/null+++ b/auto_round/export/formats/backends/svdquant_omni.py@@ -0,0 +1,72 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Standard format backend for vLLM-Omni SVDQuant NVFP4 checkpoints."""++from __future__ import annotations++from auto_round.export.formats.backends.svdquant_nunchaku import SVDQuantNunchakuFormat+from auto_round.export.formats.base import OutputFormat++[email protected]("svdquant_omni")+class SVDQuantOmniFormat(SVDQuantNunchakuFormat):+ """Reuse full-model SVDQuant export handling, but select the Omni tensor ABI."""++ support_schemes = ["NVFP4"]+ format_name = "svdquant_omni"++ @classmethod+ def check_scheme_args(cls, scheme) -> bool:+ from auto_round.export.svdquant_omni import validate_nvfp4_scheme++ return validate_nvfp4_scheme(scheme)++ def check_and_reset_format(self, scheme, ctx):+ self.check_scheme_args(scheme)+ self._validate_svd_layer_overrides(ctx.model, ctx.layer_config)+ # Do not resolve Nunchaku architecture mappings or reset W4A4 to fake.+ return None, scheme, ctx.layer_config, ctx.quant_block_list++ def save_quantized(+ self,+ output_dir,+ model=None,+ tokenizer=None,+ layer_config=None,+ inplace=True,+ device="cpu",+ serialization_dict=None,+ *,+ adapter=None,+ model_adapter=None,+ **kwargs,+ ):+ if adapter is not None and model_adapter is not None:+ raise TypeError("Pass only one of model_adapter and adapter.")+ if output_dir is None:+ return model+ from auto_round.export.svdquant_omni import save_svdquant_omni++ self._validate_svd_layer_overrides(model, layer_config)+ if adapter is None:+ adapter = model_adapter+ # Nunchaku's named adapters and transform-selected mapping describe a+ # different runtime ABI; do not resolve or inherit them for Omni.+ if isinstance(adapter, str):+ if adapter.strip().lower() not in {"auto", "identity"}:+ raise ValueError("svdquant_omni requires an Omni-compatible adapter object, not Nunchaku adapter names")+ adapter = None+ save_svdquant_omni(model, output_dir, device=device, adapter=adapter)+ return modeldiff --git a/auto_round/export/svdquant_omni.py b/auto_round/export/svdquant_omni.pynew file mode 100644index 000000000..b8037f64a--- /dev/null+++ b/auto_round/export/svdquant_omni.py@@ -0,0 +1,284 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Convert logical SVDQuant records to vLLM-Omni's NVFP4 tensor layout.++The default export fuses shared-input self-attention QKV for Omni's loaders.+Tuned NVFP4 residuals retain their scales; untuned NVFP4 residuals use RTN.+"""++from __future__ import annotations++import json+import os+from dataclasses import replace+from pathlib import Path++import torch++from auto_round.data_type.nvfp import calculate_gparam, get_reciprocal, nv_fp4+from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear, pack_fp4_to_uint8+from auto_round.export.svdquant_nunchaku import (+ IdentitySVDQuantModelAdapter,+ SVDQuantModelAdapter,+ _export_key,+ _source_records,+ _validate_adapter_provenance,+)+from auto_round.wrapper import WrapperWALayer+++def _fuse_qkv_records(model, records):+ """Join already grouped QKV; never recompute their low-rank decomposition."""+ by_name = {record.prefix: record for record in records}+ consumed, fused = set(), []+ for name, attention in model.named_modules():+ if getattr(attention, "is_cross_attention", None) is not False:+ continue+ for names, target in ((("to_q", "to_k", "to_v"), "to_qkv"), (("q_proj", "k_proj", "v_proj"), "qkv_proj")):+ prefixes = tuple(f"{name}.{suffix}" if name else suffix for suffix in names)+ if not all(prefix in by_name for prefix in prefixes):+ continue+ group = tuple(by_name[prefix] for prefix in prefixes)+ first = group[0]+ if not all(+ torch.equal(first.lora_down, record.lora_down) and torch.equal(first.smooth, record.smooth)+ for record in group[1:]+ ):+ raise ValueError(f"{name}: Omni QKV export requires shared low-rank input and smoothing parameters")+ bias = None+ if any(record.bias is not None for record in group):+ bias = torch.cat(+ [+ (+ record.bias+ if record.bias is not None+ else record.residual_weight.new_zeros(record.residual_weight.shape[0])+ )+ for record in group+ ]+ )+ fused.append(+ replace(+ first,+ prefix=f"{name}.{target}" if name else target,+ residual_weight=torch.cat([record.residual_weight for record in group]),+ lora_up=torch.cat([record.lora_up for record in group]),+ bias=bias,+ sources=tuple(source for record in group for source in record.sources),+ )+ )+ consumed.update(prefixes)+ return tuple(record for record in records if record.prefix not in consumed) + tuple(fused)+++def validate_nvfp4_scheme(scheme) -> bool:+ """Require the NVFP4 W4A4 scheme supported by Omni's SVDQuant runtime."""+ aliases = frozenset({"nv_fp", "nv_fp4", "nv_fp4_with_static_gs"})+ rules = dict(+ data_type=aliases,+ bits=4,+ group_size=16,+ sym=True,+ act_data_type=aliases,+ act_bits=4,+ act_group_size=16,+ act_sym=True,+ act_dynamic=True,+ )+ for name, expected in rules.items():+ actual = getattr(scheme, name, None)+ if isinstance(expected, frozenset):+ # Quantization wrappers persist the selected function's registry+ # name; these RTN implementations retain the same NVFP4 format.+ dtype = (+ {+ "rtn_nv_fp4_with_static_gs": "nv_fp4_with_static_gs",+ "opt_rtn_nv_fp4": "nv_fp4",+ }.get(actual, actual)+ if isinstance(actual, str)+ else actual+ )+ valid = isinstance(dtype, str) and dtype in expected+ else:+ valid = type(actual) is type(expected) and actual == expected+ if not valid:+ raise ValueError(+ f"svdquant_omni only supports NVFP4 W4A4 group16: " f"got {name}={actual!r}, expected {expected!r}"+ )+ return True++[email protected]_mode()+def _pack_residual(weight: torch.Tensor, *, scale=None, global_scale=None) -> dict[str, torch.Tensor]:+ """Reuse the existing NVFP4 quantizer and FP4 encoder; only adapt layout."""+ if weight.ndim != 2 or weight.shape[1] % 16 or 0 in weight.shape:+ raise ValueError("NVFP4 residual must be a nonempty matrix with input width divisible by 16")+ if not torch.isfinite(weight).all():+ raise ValueError("NVFP4 residual must contain only finite values")+ weight = weight.float()+ n, k = weight.shape+ if scale is not None:+ if global_scale is None:+ raise ValueError("tuned NVFP4 export requires weight_global_scale")+ packed = QuantLinear(4, 16, k, n, False, data_type="nv_fp4", act_bits=16)+ linear = torch.nn.Module()+ linear.weight = weight+ packed.pack(linear, scale, global_scale=global_scale, device=weight.device)+ return {+ "qweight": packed.weight_packed.view(torch.int8),+ "wscales": packed.weight_scale.reshape(n, k // 16).T,+ "wtscale": global_scale.float().reciprocal().reshape(1).bfloat16(),+ "wcscales": torch.ones(n, dtype=torch.bfloat16, device=weight.device),+ }+ alpha = get_reciprocal(calculate_gparam(weight)).reshape(1).bfloat16()+ if weight.count_nonzero() == 0:+ alpha.fill_(1)+ if not torch.isfinite(alpha.float()).all() or not (alpha > 0).all():+ raise ValueError("NVFP4 tensor scale must be representable as positive finite BF16")+ # Bound the temporary allocation in the existing FP4 encoder.+ chunks, scales = [], []+ for chunk in weight.split(128):+ quantized, scale, _ = nv_fp4(chunk, global_scale=alpha.float().reciprocal())+ denominator = scale.reshape(chunk.shape[0], k // 16, 1) * alpha.float()+ normalized = quantized.reshape(chunk.shape[0], k // 16, 16) * get_reciprocal(denominator)+ chunks.append(pack_fp4_to_uint8(normalized.reshape_as(chunk)))+ scales.append(scale.reshape(chunk.shape[0], k // 16).to(torch.float8_e4m3fn))+ return {+ "qweight": torch.cat(chunks).view(torch.int8),+ "wscales": torch.cat(scales).T,+ "wtscale": alpha,+ "wcscales": torch.ones(n, dtype=torch.bfloat16, device=weight.device),+ }++[email protected]_mode()+def collect_svdquant_omni_tensors(+ model: torch.nn.Module, *, device: str = "cpu", adapter: SVDQuantModelAdapter | None = None+) -> tuple[dict, dict]:+ """Serialize adapter-selected projections without assuming a model architecture."""+ fuse_qkv = adapter is None+ adapter = adapter or IdentitySVDQuantModelAdapter()+ sources = _source_records(model)+ for source in sources:+ validate_nvfp4_scheme(source.scheme)+ records = tuple(adapter.map_modules(model, sources))+ if fuse_qkv:+ records = _fuse_qkv_records(model, records)+ if not records:+ raise ValueError("model adapter produced no SVDQuant export records")+ _validate_adapter_provenance(sources, records)+ adapter.validate_records(sources, records)+ tensors, ranks = {}, set()+ for record in records:+ validate_nvfp4_scheme(record.scheme)+ if record.omitted_suffixes - {"bias"}:+ raise ValueError(f"{record.prefix}: cannot omit required Omni tensors")+ weight, down, up = record.residual_weight, record.lora_down, record.lora_up+ smooth = record.smooth.float()+ n, k = weight.shape+ rank = down.shape[0]+ if rank <= 0 or down.shape != (rank, k) or up.shape != (n, rank) or smooth.shape != (k,):+ raise ValueError(f"{record.prefix}: incompatible low-rank or smoothing dimensions")+ if not torch.isfinite(smooth).all() or not (smooth > 0).all():+ raise ValueError(f"{record.prefix}: smooth factors must be positive and finite")+ if record.bias is not None and record.bias.shape != (n,):+ raise ValueError(f"{record.prefix}: incompatible bias dimensions")+ ranks.add(rank)+ residuals = []+ for source in record.sources:+ residual = model.get_submodule(source.name).residual_linear+ while isinstance(residual, WrapperWALayer):+ residual = residual.orig_layer+ residuals.append(residual)+ scale = global_scale = None+ if any(hasattr(residual, "scale") for residual in residuals):+ if not all(+ hasattr(residual, "scale") and hasattr(residual, "weight_global_scale") for residual in residuals+ ):+ raise ValueError("tuned Omni residuals require NVFP4 scale and weight_global_scale")+ global_scale = residuals[0].weight_global_scale.to(device).float().reshape(1)+ if not all(+ torch.equal(global_scale, residual.weight_global_scale.to(device).float().reshape(1))+ for residual in residuals+ ) or not torch.equal(weight, torch.cat([source.residual_weight for source in record.sources])):+ raise ValueError("adapter must preserve tuned residuals and their shared global scale")+ scale = torch.cat([residual.scale.reshape(residual.weight.shape[0], -1) for residual in residuals]).to(+ device+ )+ values = _pack_residual(weight.to(device), scale=scale, global_scale=global_scale)+ values.update(+ proj_down=(down.float() * smooth).T.bfloat16(),+ proj_up=up.bfloat16(),+ smooth_factor=smooth.reciprocal().bfloat16(),+ )+ if record.bias is not None:+ values["bias"] = record.bias.bfloat16()+ for suffix, value in values.items():+ if suffix in record.omitted_suffixes:+ continue+ key = _export_key(record, suffix)+ if key in tensors:+ raise ValueError(f"duplicate Omni tensor key {key!r}")+ value = value.detach().cpu().contiguous()+ if value.is_floating_point() and not torch.isfinite(value.float()).all():+ raise ValueError(f"{key}: nonfinite export tensor")+ tensors[key] = value+ if len(ranks) != 1:+ raise ValueError("all exported projections must have the same rank")+ prefixes = tuple(f"{source.name}." for source in sources)+ extras = adapter.extra_tensors(model)+ if not extras:+ extras = {name: value for name, value in model.state_dict().items() if not name.startswith(prefixes)}+ for name, value in extras.items():+ if name in tensors:+ raise ValueError(f"duplicate Omni tensor key {name!r}")+ tensors[name] = value.detach().cpu().contiguous()+ quantization = dict(+ quant_method="svdquant",+ precision="nvfp4",+ rank=ranks.pop(),+ act_unsigned=False,+ modules_to_not_convert=[+ name+ for name, module in model.named_modules()+ if isinstance(module, torch.nn.Linear) and not f"{name}.".startswith(prefixes)+ ],+ )+ return tensors, quantization+++def save_svdquant_omni(+ model: torch.nn.Module, output_dir: str | Path, *, device: str = "cpu", adapter: SVDQuantModelAdapter | None = None+) -> str:+ """Save a Diffusers-style component using the caller-selected model adapter."""+ from safetensors.torch import save_file++ output = Path(output_dir)+ output.mkdir(parents=True, exist_ok=True)+ filename = output / "diffusion_pytorch_model.safetensors"+ if filename.exists():+ raise FileExistsError(filename)+ tensors, quantization = collect_svdquant_omni_tensors(model, device=device, adapter=adapter)+ config = dict(model.config)+ config["quantization_config"] = quantization+ temporary = output / ".svdquant.tmp.safetensors"+ try:+ save_file(tensors, str(temporary), metadata={"format": "pt"})+ os.replace(temporary, filename)+ finally:+ temporary.unlink(missing_ok=True)+ for name, value in (("config.json", config), ("quantization_config.json", quantization)):+ (output / name).write_text(json.dumps(value, indent=2, default=os.fspath) + "\n", encoding="utf-8")+ return str(filename)diff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex 6cc0fde63..645553003 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -723,6 +723,7 @@ class SupportedFormats:"mlx","auto_round:mlx","svdquant_nunchaku",+ "svdquant_omni",)self._gguf_format = tuple(sorted(GGUF_CONFIG.keys()))self._support_list = self._support_format + self._gguf_formatdiff --git a/test/unit/common/export/test_svdquant_omni.py b/test/unit/common/export/test_svdquant_omni.pynew file mode 100644index 000000000..2409da38b--- /dev/null+++ b/test/unit/common/export/test_svdquant_omni.py@@ -0,0 +1,249 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Coverage for the Omni-specific tensor layout, independent of model architecture."""++import json+from dataclasses import replace+from types import SimpleNamespace++import pytest+import torch+from safetensors.torch import load_file++from auto_round.algorithms.composer import AlgorithmComposer, BlockContext+from auto_round.algorithms.quantization.rtn.config import RTNConfig+from auto_round.algorithms.transforms.svdquant import SVDQuantConfig+from auto_round.algorithms.transforms.svdquant.wrapper import SVDQuantLinear+from auto_round.data_type.nvfp import nv_fp4+from auto_round.data_type.utils import update_block_global_scale_if_needed+from auto_round.export.svdquant_nunchaku import IdentitySVDQuantModelAdapter+from auto_round.export.svdquant_omni import save_svdquant_omni+from auto_round.formats import get_formats+from auto_round.schemes import PRESET_SCHEMES+from auto_round.utils.device_manager import device_manager+from auto_round.wrapper import WrapperWALayer+++def test_omni_tensor_layout_and_adapter_mapping(tmp_path):+ model = torch.nn.Module()+ model.config = {"_name_or_path": tmp_path}+ residual = torch.nn.Linear(32, 16, dtype=torch.bfloat16)+ for name, value in PRESET_SCHEMES["NVFP4"].to_dict().items():+ setattr(residual, name, value)+ down = torch.nn.Linear(32, 8, bias=False, dtype=torch.bfloat16)+ up = torch.nn.Linear(8, 16, bias=False, dtype=torch.bfloat16)+ model.projection = SVDQuantLinear(residual, down, up, torch.full((32,), 2, dtype=torch.bfloat16))+ model.unquantized = torch.nn.Linear(32, 16)++ class RenameAdapter(IdentitySVDQuantModelAdapter):+ def map_modules(self, model, records):+ return [replace(record, prefix="runtime.projection") for record in super().map_modules(model, records)]++ for adapter, prefix in ((None, "projection"), (RenameAdapter(), "runtime.projection")):+ if adapter is not None:+ # Also exercise export of tuned residuals through the same ABI.+ residual.data_type, residual.bits, residual.group_size = "nv_fp4", 4, 16+ residual.weight_global_scale = torch.tensor([4.0])+ quantized, residual.scale, _ = nv_fp4(residual.weight.float(), global_scale=residual.weight_global_scale)+ residual.weight.data = quantized.bfloat16()+ output = tmp_path / prefix+ output_format = get_formats("svdquant_omni", SimpleNamespace(scheme="NVFP4"))[0]+ assert output_format.save_quantized(output, model=model, adapter=adapter) is model+ tensors = load_file(output / "diffusion_pytorch_model.safetensors")+ assert set(tensors) == {+ *(+ f"{prefix}.{suffix}"+ for suffix in (+ "qweight",+ "wscales",+ "wtscale",+ "wcscales",+ "proj_down",+ "proj_up",+ "smooth_factor",+ "bias",+ )+ ),+ "unquantized.weight",+ "unquantized.bias",+ }+ assert tensors[f"{prefix}.qweight"].shape == (16, 16)+ assert tensors[f"{prefix}.qweight"].dtype == torch.int8+ assert tensors[f"{prefix}.wscales"].shape == (2, 16)+ assert tensors[f"{prefix}.wscales"].dtype == torch.float8_e4m3fn+ assert tensors[f"{prefix}.wtscale"].shape == (1,)+ assert tensors[f"{prefix}.wtscale"].dtype == torch.bfloat16+ if adapter is not None:+ torch.testing.assert_close(tensors[f"{prefix}.wtscale"], torch.tensor([0.25], dtype=torch.bfloat16))+ torch.testing.assert_close(tensors[f"{prefix}.wscales"].float(), residual.scale.reshape(16, 2).T)+ torch.testing.assert_close(tensors[f"{prefix}.wcscales"], torch.ones(16, dtype=torch.bfloat16))+ torch.testing.assert_close(tensors[f"{prefix}.proj_down"], (down.weight.float() * 2).T.bfloat16())+ torch.testing.assert_close(tensors[f"{prefix}.proj_up"], up.weight)+ torch.testing.assert_close(tensors[f"{prefix}.smooth_factor"], torch.full((32,), 0.5, dtype=torch.bfloat16))+ torch.testing.assert_close(tensors[f"{prefix}.bias"], residual.bias)+ torch.testing.assert_close(tensors["unquantized.weight"], model.unquantized.weight)+ config = json.loads((output / "config.json").read_text())+ quantization = json.loads((output / "quantization_config.json").read_text())+ assert config["quantization_config"] == quantization+ assert quantization == {+ "quant_method": "svdquant",+ "precision": "nvfp4",+ "rank": 8,+ "act_unsigned": False,+ "modules_to_not_convert": ["unquantized"],+ }+++def test_omni_format_rejects_incompatible_schemes(tmp_path, monkeypatch):+ output_format = get_formats("svdquant_omni", SimpleNamespace(scheme="NVFP4"))[0]+ assert output_format.output_format == "svdquant_omni"+ assert not output_format.is_supported_immediate_packing()+ assert not output_format.is_supported_immediate_saving()+ for preset in ("MXFP4", "W4A16", "NVFP4_E5M3"):+ with pytest.raises(ValueError, match="NVFP4 W4A4 group16"):+ get_formats("svdquant_omni", SimpleNamespace(scheme=preset))+ for field, value in (("bits", 8), ("group_size", 32), ("act_bits", 16), ("act_dynamic", False)):+ scheme = PRESET_SCHEMES["NVFP4"].copy()+ setattr(scheme, field, value)+ with pytest.raises(ValueError, match=field):+ output_format.check_scheme_args(scheme)+ model = torch.nn.Module()+ residual = torch.nn.Linear(32, 16)+ for name, value in PRESET_SCHEMES["MXFP4"].to_dict().items():+ setattr(residual, name, value)+ model.projection = SVDQuantLinear(+ residual, torch.nn.Linear(32, 8, bias=False), torch.nn.Linear(8, 16, bias=False), torch.ones(32)+ )+ with pytest.raises(ValueError, match="NVFP4 W4A4 group16"):+ save_svdquant_omni(model, tmp_path / "invalid")+ with pytest.raises(ValueError, match="incompatible residual scheme"):+ output_format._validate_svd_layer_overrides(model, {"projection": "MXFP4"})++ from auto_round import AutoRound+ from auto_round.algorithms.quantization.rtn.config import RTNConfig+ from auto_round.algorithms.transforms.svdquant import SVDQuantConfig++ created = {}++ class FakeCompressor:+ def __init__(self, config, **kwargs):+ created.update(kwargs)++ monkeypatch.setattr("auto_round.autoround._build_model_type_ctor_kwargs", lambda *args, **kwargs: ("llm", {}))+ monkeypatch.setattr("auto_round.autoround._get_compressor_class", lambda model_type, base_cls: FakeCompressor)+ for format in ("svdquant_omni", "svdquant_omni,fake"):+ AutoRound(+ "dummy-model",+ scheme="NVFP4",+ format=format,+ alg_configs=[SVDQuantConfig(smooth_enabled=False), RTNConfig(disable_opt_rtn=True)],+ )+ assert created["format"] == format+++def test_omni_zero_shot_rtn_fuses_qkv_with_shared_scale(tmp_path, monkeypatch):+ monkeypatch.setattr(device_manager, "_device_map", "cpu")+ monkeypatch.setattr(device_manager, "_device_list", ["cpu"])+ monkeypatch.setattr(device_manager, "_major_device", "cpu")++ class SelfAttention(torch.nn.Module):+ is_cross_attention = False++ def __init__(self):+ super().__init__()+ self.to_q = torch.nn.Linear(32, 32, bias=False)+ self.to_k = torch.nn.Linear(32, 32, bias=False)+ self.to_v = torch.nn.Linear(32, 32, bias=False)++ class WanBlock(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.attn = SelfAttention()++ model = torch.nn.Module()+ model.blocks = torch.nn.ModuleList([WanBlock()])+ model.config = {"_name_or_path": str(tmp_path)}+ scheme = PRESET_SCHEMES["NVFP4"].copy()+ for name, module in model.named_modules():+ module.global_name = name+ if isinstance(module, torch.nn.Linear):+ for key, value in scheme.to_dict().items():+ setattr(module, key, value)+ module.scale_dtype = torch.float32++ block_name = "blocks.0"+ orchestrator = SimpleNamespace(+ model_context=SimpleNamespace(+ model=model,+ amp=False,+ amp_dtype=torch.float32,+ is_diffusion=True,+ is_moe_model=False,+ output_config=None,+ ),+ compress_context=SimpleNamespace(+ enable_torch_compile=False,+ low_gpu_mem_usage=False,+ cache_device=torch.device("cpu"),+ clear_memory=lambda: None,+ formats=[SimpleNamespace(format_name="svdquant_omni")],+ ),+ calibration_context=SimpleNamespace(batch_size=1, batch_dim=0),+ scheme_context=scheme,+ scale_dtype=torch.float32,+ nblocks=1,+ quant_block_list=[[block_name]],+ data_type=scheme.data_type,+ batch_dim=0,+ batch_size=1,+ cache_device="cpu",+ amp=False,+ amp_dtype=torch.float32,+ shared_cache_keys=(),+ )+ composer = AlgorithmComposer(+ [+ SVDQuantConfig(rank=2, smooth_enabled=False, residual_iters=1, low_rank_dtype="fp32"),+ RTNConfig(disable_opt_rtn=True),+ ],+ orchestrator=orchestrator,+ )+ composer.prepare_run()++ block = model.blocks[0]+ # Match the Orchestrator's initial NVFP4 scale setup before SVD replacement.+ update_block_global_scale_if_needed(block, scheme.data_type, scheme.group_size)+ ctx = BlockContext(+ model=model,+ block_names=[block_name],+ block_name=block_name,+ block_index=0,+ block_cnt=1,+ )+ composer.compress_block(block, fp_inputs=None, input_others={}, block_ctx=ctx)++ residual_scales = []+ for name in ("to_q", "to_k", "to_v"):+ residual = getattr(block.attn, name).residual_linear+ while isinstance(residual, WrapperWALayer):+ residual = residual.orig_layer+ residual_scales.append(residual.weight_global_scale)+ torch.testing.assert_close(residual_scales[0], residual_scales[1])+ torch.testing.assert_close(residual_scales[0], residual_scales[2])++ output_dir = tmp_path / "svdquant_omni"+ save_svdquant_omni(model, output_dir)+ tensors = load_file(output_dir / "diffusion_pytorch_model.safetensors")+ assert "blocks.0.attn.to_qkv.wtscale" in tensors