Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat: export SVDQuant NVFP4 checkpoints for vLLM-Omni (#2445)

Wang, Chang7c0aed071e81
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.py
index 6605e5b7f..00aaa4e84 100644
--- a/auto_round/algorithms/composer.py
+++ b/auto_round/algorithms/composer.py
@@ -474,6 +474,11 @@ class AlgorithmComposer:
for pre in self.preprocessors:
pre.pre_quantize_block(block_ctx)
+ act_dynamic = self.scheme.act_dynamic if (self.scheme and self.scheme.act_dynamic is not None) else True
+ data_type = self.scheme.data_type if self.scheme else "int"
+ group_size = self.scheme.group_size if self.scheme else -1
+ act_data_type = self.scheme.act_data_type if self.scheme else data_type
+
reference_next_input = None
# ── Step 3: Quantizer calibration (act_max, imatrix, etc.) ─────────────
if fp_inputs is not None:
@@ -494,18 +499,20 @@ class AlgorithmComposer:
# ── Step 3.5: MoE scale alignment + global scale update ─────────────────
# Must run after calibration hooks (act_max collected) and before quantize_block.
- act_dynamic = self.scheme.act_dynamic if (self.scheme and self.scheme.act_dynamic is not None) else True
- data_type = self.scheme.data_type if self.scheme else "int"
- group_size = self.scheme.group_size if self.scheme else -1
- act_data_type = self.scheme.act_data_type if self.scheme else data_type
if act_data_type is not None or not act_dynamic:
from auto_round.compressors.utils import is_nv_fp
- from auto_round.data_type.utils import update_block_global_scale_if_needed
from auto_round.utils import set_amax_for_all_moe_layers
if is_nv_fp(act_data_type) or not act_dynamic:
set_amax_for_all_moe_layers(block, attr_name="act_max")
- update_block_global_scale_if_needed(block, data_type, group_size)
+
+ # SVDQuant may have replaced the original linears during pre-quantize.
+ # Update NVFP4 scales after that transformation even on the zero-shot
+ # path, where fp_inputs is None and the calibration block above is skipped.
+ if act_data_type is not None or not act_dynamic:
+ from auto_round.data_type.utils import update_block_global_scale_if_needed
+
+ update_block_global_scale_if_needed(block, data_type, group_size)
if q_inputs is not None and fp_inputs is not q_inputs:
# Release the fp pool at its last use (before tuning): the q pool is
diff --git a/auto_round/algorithms/transforms/svdquant/apply.py b/auto_round/algorithms/transforms/svdquant/apply.py
index fffff0beb..a9182f5b9 100644
--- a/auto_round/algorithms/transforms/svdquant/apply.py
+++ b/auto_round/algorithms/transforms/svdquant/apply.py
@@ -136,6 +136,7 @@ class SVDQuantTransform(BasePreprocessor):
def __init__(self, config: SVDQuantConfig) -> None:
super().__init__(config)
self._svd_driver = None
+ self._grouping_backend = None
self._configured_block_names: tuple[str, ...] = ()
self._block_groups: dict[str, list[SmoothSearchGroup]] = {}
self._smooth_calibration: dict[str, SmoothGroupCalibration] = {}
@@ -167,12 +168,19 @@ class SVDQuantTransform(BasePreprocessor):
def prepare_run(self, composer=None) -> None:
self._block_groups.clear()
+ formats = getattr(self.compress_context, "formats", None) or ()
+ self._grouping_backend = "omni" if any(fmt.format_name == "svdquant_omni" for fmt in formats) else None
if self.model is None:
return
- self._resolve_model_adapter(self.model)
+ if self._grouping_backend == "omni":
+ self._target_modules = self.config.target_modules
+ else:
+ self._resolve_model_adapter(self.model)
for block_name in self._configured_block_names:
block = self.model.get_submodule(block_name)
- self._block_groups[block_name] = discover_svdquant_groups(block, self._is_target)
+ self._block_groups[block_name] = discover_svdquant_groups(
+ block, self._is_target, grouping_backend=self._grouping_backend
+ )
logger.info(
"SVDQuant: resolved %d projection groups across %d blocks.",
sum(len(groups) for groups in self._block_groups.values()),
@@ -217,12 +225,13 @@ class SVDQuantTransform(BasePreprocessor):
def register_fp_input_forward_hooks(self, block) -> list:
if not self.config.smooth_enabled:
return []
- self._resolve_model_adapter(self.model, block)
+ if self._grouping_backend != "omni":
+ self._resolve_model_adapter(self.model, block)
self._clear_smooth_calibration()
block_name = str(getattr(block, "global_name", ""))
groups = self._block_groups.get(block_name)
if groups is None:
- groups = discover_svdquant_groups(block, self._is_target)
+ groups = discover_svdquant_groups(block, self._is_target, grouping_backend=self._grouping_backend)
self._block_groups[block_name] = groups
self._smooth_calibration = {
group.key: SmoothGroupCalibration(group, self.config.smooth_max_calibration_calls) for group in groups
@@ -256,10 +265,11 @@ class SVDQuantTransform(BasePreprocessor):
raise ValueError(f"SVDQuant requires one block at a time, got {ctx.block_names!r}.")
block_name = ctx.block_name
block = ctx.model.get_submodule(block_name)
- self._resolve_model_adapter(ctx.model, block)
+ if self._grouping_backend != "omni":
+ self._resolve_model_adapter(ctx.model, block)
groups = self._block_groups.get(block_name)
if groups is None:
- groups = discover_svdquant_groups(block, self._is_target)
+ groups = discover_svdquant_groups(block, self._is_target, grouping_backend=self._grouping_backend)
self._block_groups[block_name] = groups
if self.config.smooth_enabled:
@@ -619,6 +629,8 @@ class SVDQuantTransform(BasePreprocessor):
)
lora_down.weight.copy_(down)
lora_up.weight.copy_(up)
+ lora_down.requires_grad_(projection.weight.requires_grad)
+ lora_up.requires_grad_(projection.weight.requires_grad)
self._mark_unquantized(lora_down)
self._mark_unquantized(lora_up)
self._copy_quant_attrs(projection, residual, suffix=".residual_linear")
@@ -700,8 +712,11 @@ class SVDQuantTransform(BasePreprocessor):
device=module.weight.device,
)
residual.weight.copy_(weight.to(module.weight.dtype))
+ # Preserve freezing: cached NVFP4 scales must not acquire an autograd graph.
+ residual.weight.requires_grad_(module.weight.requires_grad)
if bias is not None:
residual.bias.copy_(bias.detach().to(module.weight.dtype))
+ residual.bias.requires_grad_(module.bias.requires_grad)
return residual
@staticmethod
@@ -711,7 +726,9 @@ class SVDQuantTransform(BasePreprocessor):
@staticmethod
def _copy_quant_attrs(src: torch.nn.Module, dst: torch.nn.Module, suffix: str) -> None:
- for attr in _SCHEME_ATTRS | _RUNTIME_QUANT_ATTRS:
+ # The residual has different weights (and may be smoothed), so its
+ # NVFP4 global scale must be calculated again before quantization.
+ for attr in (_SCHEME_ATTRS | _RUNTIME_QUANT_ATTRS) - {"weight_global_scale"}:
if hasattr(src, attr):
setattr(dst, attr, getattr(src, attr))
if getattr(dst, "bits", None) == 4 and getattr(dst, "data_type", None) in _MXFP4_ALIASES:
diff --git a/auto_round/algorithms/transforms/svdquant/smooth_adapters/__init__.py b/auto_round/algorithms/transforms/svdquant/smooth_adapters/__init__.py
index 02d79ab0e..0c5f9c2b9 100644
--- a/auto_round/algorithms/transforms/svdquant/smooth_adapters/__init__.py
+++ b/auto_round/algorithms/transforms/svdquant/smooth_adapters/__init__.py
@@ -22,11 +22,16 @@ from auto_round.algorithms.transforms.svdquant.smooth_adapters.base import (
generic_linear_groups,
)
from auto_round.algorithms.transforms.svdquant.smooth_adapters.flux import discover_flux_groups, supports_flux_block
+from auto_round.algorithms.transforms.svdquant.smooth_adapters.omni import discover_omni_groups
from auto_round.algorithms.transforms.svdquant.smooth_adapters.sdxl import discover_sdxl_groups, supports_sdxl_block
-def discover_svdquant_groups(block: torch.nn.Module, is_target: TargetPredicate) -> list[SmoothSearchGroup]:
+def discover_svdquant_groups(
+ block: torch.nn.Module, is_target: TargetPredicate, *, grouping_backend: str | None = None
+) -> list[SmoothSearchGroup]:
"""Discover shared-input projection groups for one quantization block."""
+ if grouping_backend == "omni":
+ return discover_omni_groups(block, is_target)
if supports_flux_block(block):
return discover_flux_groups(block, is_target)
if supports_sdxl_block(block):
diff --git a/auto_round/algorithms/transforms/svdquant/smooth_adapters/omni.py b/auto_round/algorithms/transforms/svdquant/smooth_adapters/omni.py
new file mode 100644
index 000000000..2390bfa6e
--- /dev/null
+++ b/auto_round/algorithms/transforms/svdquant/smooth_adapters/omni.py
@@ -0,0 +1,62 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shared-input QKV grouping for vLLM-Omni's fused SVDQuant linear layers."""
+
+import torch
+
+from auto_round.algorithms.transforms.svdquant.smooth_adapters.base import (
+ SmoothSearchGroup,
+ TargetPredicate,
+ generic_linear_groups,
+ module_global_name,
+)
+
+_QKV_NAMES = (("to_q", "to_k", "to_v"), ("q_proj", "k_proj", "v_proj"))
+
+
+def discover_omni_groups(block: torch.nn.Module, is_target: TargetPredicate) -> list[SmoothSearchGroup]:
+ """Group explicit self-attention QKV without model-class-specific layouts."""
+ groups = []
+ consumed = set()
+ for local_name, attention in block.named_modules():
+ if getattr(attention, "is_cross_attention", None) is not False:
+ continue
+ projection_names = next((names for names in _QKV_NAMES if all(hasattr(attention, name) for name in names)), ())
+ if not projection_names:
+ continue
+ paths = tuple(f"{local_name}.{name}" if local_name else name for name in projection_names)
+ projections = tuple(getattr(attention, name) for name in projection_names)
+ if not all(
+ isinstance(module, torch.nn.Linear) and id(module) not in consumed and is_target(path, module)
+ for path, module in zip(paths, projections)
+ ):
+ continue
+ if len({module.in_features for module in projections}) != 1 or len({id(module) for module in projections}) != 3:
+ continue
+ names = tuple(module_global_name(block, path) for path in paths)
+ groups.append(
+ SmoothSearchGroup(
+ key=module_global_name(block, f"{local_name}.qkv" if local_name else "qkv"),
+ projection_names=names,
+ projections=projections,
+ projection_input_key=names[0],
+ projection_input_module=projections[0],
+ evaluation_input_key=module_global_name(block, local_name),
+ evaluation_module=attention,
+ )
+ )
+ consumed.update(id(module) for module in projections)
+ groups.extend(generic_linear_groups(block, is_target, consumed=consumed))
+ return groups
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index e697a1d43..4c777606d 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -594,7 +594,7 @@ class _CompressorBuilder:
configs_for_routing = alg_configs if isinstance(alg_configs, list) else [alg_configs]
preprocessor_configs, _, quant_config = _resolve_quant_config_for_routing(configs_for_routing)
is_svdquant = any(type(config).__name__ == "SVDQuantConfig" for config in preprocessor_configs)
- if is_svdquant:
+ if is_svdquant and "svdquant_omni" not in {name.strip().lower() for name in (format or "").split(",")}:
format = "svdquant_nunchaku"
# Any preprocessor that requires calibration data (e.g. AWQ, SVDQuant
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 27a1a4212..64f821cf2 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -520,6 +520,10 @@ def update_fused_layer_global_scales(
def _update_global_scales(modules: list[Module]):
"""Update global scales for a list of modules."""
+ from auto_round.algorithms.transforms.svdquant.wrapper import SVDQuantLinear
+
+ # SVDQuant keeps the quantized projection inside its residual branch.
+ modules = [module.residual_linear if isinstance(module, SVDQuantLinear) else module for module in modules]
scales = _collect_scales(modules)
if not scales:
return
diff --git a/auto_round/export/formats/backends/__init__.py b/auto_round/export/formats/backends/__init__.py
index e74dbf3e1..14c509aff 100644
--- a/auto_round/export/formats/backends/__init__.py
+++ b/auto_round/export/formats/backends/__init__.py
@@ -21,6 +21,7 @@ from auto_round.export.formats.backends.auto_gptq import AutoGPTQFormat
from auto_round.export.formats.backends.llm_compressor import LLMCompressorFormat
from auto_round.export.formats.backends.mlx import MLXFormat
from auto_round.export.formats.backends.svdquant_nunchaku import SVDQuantNunchakuFormat
+from auto_round.export.formats.backends.svdquant_omni import SVDQuantOmniFormat
__all__ = [
"AutoAWQFormat",
@@ -32,4 +33,5 @@ __all__ = [
"LLMCompressorFormat",
"MLXFormat",
"SVDQuantNunchakuFormat",
+ "SVDQuantOmniFormat",
]
diff --git a/auto_round/export/formats/backends/svdquant_omni.py b/auto_round/export/formats/backends/svdquant_omni.py
new file mode 100644
index 000000000..8c130ba38
--- /dev/null
+++ b/auto_round/export/formats/backends/svdquant_omni.py
@@ -0,0 +1,72 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Standard format backend for vLLM-Omni SVDQuant NVFP4 checkpoints."""
+
+from __future__ import annotations
+
+from auto_round.export.formats.backends.svdquant_nunchaku import SVDQuantNunchakuFormat
+from auto_round.export.formats.base import OutputFormat
+
+
[email protected]("svdquant_omni")
+class SVDQuantOmniFormat(SVDQuantNunchakuFormat):
+ """Reuse full-model SVDQuant export handling, but select the Omni tensor ABI."""
+
+ support_schemes = ["NVFP4"]
+ format_name = "svdquant_omni"
+
+ @classmethod
+ def check_scheme_args(cls, scheme) -> bool:
+ from auto_round.export.svdquant_omni import validate_nvfp4_scheme
+
+ return validate_nvfp4_scheme(scheme)
+
+ def check_and_reset_format(self, scheme, ctx):
+ self.check_scheme_args(scheme)
+ self._validate_svd_layer_overrides(ctx.model, ctx.layer_config)
+ # Do not resolve Nunchaku architecture mappings or reset W4A4 to fake.
+ return None, scheme, ctx.layer_config, ctx.quant_block_list
+
+ def save_quantized(
+ self,
+ output_dir,
+ model=None,
+ tokenizer=None,
+ layer_config=None,
+ inplace=True,
+ device="cpu",
+ serialization_dict=None,
+ *,
+ adapter=None,
+ model_adapter=None,
+ **kwargs,
+ ):
+ if adapter is not None and model_adapter is not None:
+ raise TypeError("Pass only one of model_adapter and adapter.")
+ if output_dir is None:
+ return model
+ from auto_round.export.svdquant_omni import save_svdquant_omni
+
+ self._validate_svd_layer_overrides(model, layer_config)
+ if adapter is None:
+ adapter = model_adapter
+ # Nunchaku's named adapters and transform-selected mapping describe a
+ # different runtime ABI; do not resolve or inherit them for Omni.
+ if isinstance(adapter, str):
+ if adapter.strip().lower() not in {"auto", "identity"}:
+ raise ValueError("svdquant_omni requires an Omni-compatible adapter object, not Nunchaku adapter names")
+ adapter = None
+ save_svdquant_omni(model, output_dir, device=device, adapter=adapter)
+ return model
diff --git a/auto_round/export/svdquant_omni.py b/auto_round/export/svdquant_omni.py
new file mode 100644
index 000000000..b8037f64a
--- /dev/null
+++ b/auto_round/export/svdquant_omni.py
@@ -0,0 +1,284 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Convert logical SVDQuant records to vLLM-Omni's NVFP4 tensor layout.
+
+The default export fuses shared-input self-attention QKV for Omni's loaders.
+Tuned NVFP4 residuals retain their scales; untuned NVFP4 residuals use RTN.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import replace
+from pathlib import Path
+
+import torch
+
+from auto_round.data_type.nvfp import calculate_gparam, get_reciprocal, nv_fp4
+from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear, pack_fp4_to_uint8
+from auto_round.export.svdquant_nunchaku import (
+ IdentitySVDQuantModelAdapter,
+ SVDQuantModelAdapter,
+ _export_key,
+ _source_records,
+ _validate_adapter_provenance,
+)
+from auto_round.wrapper import WrapperWALayer
+
+
+def _fuse_qkv_records(model, records):
+ """Join already grouped QKV; never recompute their low-rank decomposition."""
+ by_name = {record.prefix: record for record in records}
+ consumed, fused = set(), []
+ for name, attention in model.named_modules():
+ if getattr(attention, "is_cross_attention", None) is not False:
+ continue
+ for names, target in ((("to_q", "to_k", "to_v"), "to_qkv"), (("q_proj", "k_proj", "v_proj"), "qkv_proj")):
+ prefixes = tuple(f"{name}.{suffix}" if name else suffix for suffix in names)
+ if not all(prefix in by_name for prefix in prefixes):
+ continue
+ group = tuple(by_name[prefix] for prefix in prefixes)
+ first = group[0]
+ if not all(
+ torch.equal(first.lora_down, record.lora_down) and torch.equal(first.smooth, record.smooth)
+ for record in group[1:]
+ ):
+ raise ValueError(f"{name}: Omni QKV export requires shared low-rank input and smoothing parameters")
+ bias = None
+ if any(record.bias is not None for record in group):
+ bias = torch.cat(
+ [
+ (
+ record.bias
+ if record.bias is not None
+ else record.residual_weight.new_zeros(record.residual_weight.shape[0])
+ )
+ for record in group
+ ]
+ )
+ fused.append(
+ replace(
+ first,
+ prefix=f"{name}.{target}" if name else target,
+ residual_weight=torch.cat([record.residual_weight for record in group]),
+ lora_up=torch.cat([record.lora_up for record in group]),
+ bias=bias,
+ sources=tuple(source for record in group for source in record.sources),
+ )
+ )
+ consumed.update(prefixes)
+ return tuple(record for record in records if record.prefix not in consumed) + tuple(fused)
+
+
+def validate_nvfp4_scheme(scheme) -> bool:
+ """Require the NVFP4 W4A4 scheme supported by Omni's SVDQuant runtime."""
+ aliases = frozenset({"nv_fp", "nv_fp4", "nv_fp4_with_static_gs"})
+ rules = dict(
+ data_type=aliases,
+ bits=4,
+ group_size=16,
+ sym=True,
+ act_data_type=aliases,
+ act_bits=4,
+ act_group_size=16,
+ act_sym=True,
+ act_dynamic=True,
+ )
+ for name, expected in rules.items():
+ actual = getattr(scheme, name, None)
+ if isinstance(expected, frozenset):
+ # Quantization wrappers persist the selected function's registry
+ # name; these RTN implementations retain the same NVFP4 format.
+ dtype = (
+ {
+ "rtn_nv_fp4_with_static_gs": "nv_fp4_with_static_gs",
+ "opt_rtn_nv_fp4": "nv_fp4",
+ }.get(actual, actual)
+ if isinstance(actual, str)
+ else actual
+ )
+ valid = isinstance(dtype, str) and dtype in expected
+ else:
+ valid = type(actual) is type(expected) and actual == expected
+ if not valid:
+ raise ValueError(
+ f"svdquant_omni only supports NVFP4 W4A4 group16: " f"got {name}={actual!r}, expected {expected!r}"
+ )
+ return True
+
+
+def _pack_residual(weight: torch.Tensor, *, scale=None, global_scale=None) -> dict[str, torch.Tensor]:
+ """Reuse the existing NVFP4 quantizer and FP4 encoder; only adapt layout."""
+ if weight.ndim != 2 or weight.shape[1] % 16 or 0 in weight.shape:
+ raise ValueError("NVFP4 residual must be a nonempty matrix with input width divisible by 16")
+ if not torch.isfinite(weight).all():
+ raise ValueError("NVFP4 residual must contain only finite values")
+ weight = weight.float()
+ n, k = weight.shape
+ if scale is not None:
+ if global_scale is None:
+ raise ValueError("tuned NVFP4 export requires weight_global_scale")
+ packed = QuantLinear(4, 16, k, n, False, data_type="nv_fp4", act_bits=16)
+ linear = torch.nn.Module()
+ linear.weight = weight
+ packed.pack(linear, scale, global_scale=global_scale, device=weight.device)
+ return {
+ "qweight": packed.weight_packed.view(torch.int8),
+ "wscales": packed.weight_scale.reshape(n, k // 16).T,
+ "wtscale": global_scale.float().reciprocal().reshape(1).bfloat16(),
+ "wcscales": torch.ones(n, dtype=torch.bfloat16, device=weight.device),
+ }
+ alpha = get_reciprocal(calculate_gparam(weight)).reshape(1).bfloat16()
+ if weight.count_nonzero() == 0:
+ alpha.fill_(1)
+ if not torch.isfinite(alpha.float()).all() or not (alpha > 0).all():
+ raise ValueError("NVFP4 tensor scale must be representable as positive finite BF16")
+ # Bound the temporary allocation in the existing FP4 encoder.
+ chunks, scales = [], []
+ for chunk in weight.split(128):
+ quantized, scale, _ = nv_fp4(chunk, global_scale=alpha.float().reciprocal())
+ denominator = scale.reshape(chunk.shape[0], k // 16, 1) * alpha.float()
+ normalized = quantized.reshape(chunk.shape[0], k // 16, 16) * get_reciprocal(denominator)
+ chunks.append(pack_fp4_to_uint8(normalized.reshape_as(chunk)))
+ scales.append(scale.reshape(chunk.shape[0], k // 16).to(torch.float8_e4m3fn))
+ return {
+ "qweight": torch.cat(chunks).view(torch.int8),
+ "wscales": torch.cat(scales).T,
+ "wtscale": alpha,
+ "wcscales": torch.ones(n, dtype=torch.bfloat16, device=weight.device),
+ }
+
+
+def collect_svdquant_omni_tensors(
+ model: torch.nn.Module, *, device: str = "cpu", adapter: SVDQuantModelAdapter | None = None
+) -> tuple[dict, dict]:
+ """Serialize adapter-selected projections without assuming a model architecture."""
+ fuse_qkv = adapter is None
+ adapter = adapter or IdentitySVDQuantModelAdapter()
+ sources = _source_records(model)
+ for source in sources:
+ validate_nvfp4_scheme(source.scheme)
+ records = tuple(adapter.map_modules(model, sources))
+ if fuse_qkv:
+ records = _fuse_qkv_records(model, records)
+ if not records:
+ raise ValueError("model adapter produced no SVDQuant export records")
+ _validate_adapter_provenance(sources, records)
+ adapter.validate_records(sources, records)
+ tensors, ranks = {}, set()
+ for record in records:
+ validate_nvfp4_scheme(record.scheme)
+ if record.omitted_suffixes - {"bias"}:
+ raise ValueError(f"{record.prefix}: cannot omit required Omni tensors")
+ weight, down, up = record.residual_weight, record.lora_down, record.lora_up
+ smooth = record.smooth.float()
+ n, k = weight.shape
+ rank = down.shape[0]
+ if rank <= 0 or down.shape != (rank, k) or up.shape != (n, rank) or smooth.shape != (k,):
+ raise ValueError(f"{record.prefix}: incompatible low-rank or smoothing dimensions")
+ if not torch.isfinite(smooth).all() or not (smooth > 0).all():
+ raise ValueError(f"{record.prefix}: smooth factors must be positive and finite")
+ if record.bias is not None and record.bias.shape != (n,):
+ raise ValueError(f"{record.prefix}: incompatible bias dimensions")
+ ranks.add(rank)
+ residuals = []
+ for source in record.sources:
+ residual = model.get_submodule(source.name).residual_linear
+ while isinstance(residual, WrapperWALayer):
+ residual = residual.orig_layer
+ residuals.append(residual)
+ scale = global_scale = None
+ if any(hasattr(residual, "scale") for residual in residuals):
+ if not all(
+ hasattr(residual, "scale") and hasattr(residual, "weight_global_scale") for residual in residuals
+ ):
+ raise ValueError("tuned Omni residuals require NVFP4 scale and weight_global_scale")
+ global_scale = residuals[0].weight_global_scale.to(device).float().reshape(1)
+ if not all(
+ torch.equal(global_scale, residual.weight_global_scale.to(device).float().reshape(1))
+ for residual in residuals
+ ) or not torch.equal(weight, torch.cat([source.residual_weight for source in record.sources])):
+ raise ValueError("adapter must preserve tuned residuals and their shared global scale")
+ scale = torch.cat([residual.scale.reshape(residual.weight.shape[0], -1) for residual in residuals]).to(
+ device
+ )
+ values = _pack_residual(weight.to(device), scale=scale, global_scale=global_scale)
+ values.update(
+ proj_down=(down.float() * smooth).T.bfloat16(),
+ proj_up=up.bfloat16(),
+ smooth_factor=smooth.reciprocal().bfloat16(),
+ )
+ if record.bias is not None:
+ values["bias"] = record.bias.bfloat16()
+ for suffix, value in values.items():
+ if suffix in record.omitted_suffixes:
+ continue
+ key = _export_key(record, suffix)
+ if key in tensors:
+ raise ValueError(f"duplicate Omni tensor key {key!r}")
+ value = value.detach().cpu().contiguous()
+ if value.is_floating_point() and not torch.isfinite(value.float()).all():
+ raise ValueError(f"{key}: nonfinite export tensor")
+ tensors[key] = value
+ if len(ranks) != 1:
+ raise ValueError("all exported projections must have the same rank")
+ prefixes = tuple(f"{source.name}." for source in sources)
+ extras = adapter.extra_tensors(model)
+ if not extras:
+ extras = {name: value for name, value in model.state_dict().items() if not name.startswith(prefixes)}
+ for name, value in extras.items():
+ if name in tensors:
+ raise ValueError(f"duplicate Omni tensor key {name!r}")
+ tensors[name] = value.detach().cpu().contiguous()
+ quantization = dict(
+ quant_method="svdquant",
+ precision="nvfp4",
+ rank=ranks.pop(),
+ act_unsigned=False,
+ modules_to_not_convert=[
+ name
+ for name, module in model.named_modules()
+ if isinstance(module, torch.nn.Linear) and not f"{name}.".startswith(prefixes)
+ ],
+ )
+ return tensors, quantization
+
+
+def save_svdquant_omni(
+ model: torch.nn.Module, output_dir: str | Path, *, device: str = "cpu", adapter: SVDQuantModelAdapter | None = None
+) -> str:
+ """Save a Diffusers-style component using the caller-selected model adapter."""
+ from safetensors.torch import save_file
+
+ output = Path(output_dir)
+ output.mkdir(parents=True, exist_ok=True)
+ filename = output / "diffusion_pytorch_model.safetensors"
+ if filename.exists():
+ raise FileExistsError(filename)
+ tensors, quantization = collect_svdquant_omni_tensors(model, device=device, adapter=adapter)
+ config = dict(model.config)
+ config["quantization_config"] = quantization
+ temporary = output / ".svdquant.tmp.safetensors"
+ try:
+ save_file(tensors, str(temporary), metadata={"format": "pt"})
+ os.replace(temporary, filename)
+ finally:
+ temporary.unlink(missing_ok=True)
+ for name, value in (("config.json", config), ("quantization_config.json", quantization)):
+ (output / name).write_text(json.dumps(value, indent=2, default=os.fspath) + "\n", encoding="utf-8")
+ return str(filename)
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index 6cc0fde63..645553003 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -723,6 +723,7 @@ class SupportedFormats:
"mlx",
"auto_round:mlx",
"svdquant_nunchaku",
+ "svdquant_omni",
)
self._gguf_format = tuple(sorted(GGUF_CONFIG.keys()))
self._support_list = self._support_format + self._gguf_format
diff --git a/test/unit/common/export/test_svdquant_omni.py b/test/unit/common/export/test_svdquant_omni.py
new file mode 100644
index 000000000..2409da38b
--- /dev/null
+++ b/test/unit/common/export/test_svdquant_omni.py
@@ -0,0 +1,249 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Coverage for the Omni-specific tensor layout, independent of model architecture."""
+
+import json
+from dataclasses import replace
+from types import SimpleNamespace
+
+import pytest
+import torch
+from safetensors.torch import load_file
+
+from auto_round.algorithms.composer import AlgorithmComposer, BlockContext
+from auto_round.algorithms.quantization.rtn.config import RTNConfig
+from auto_round.algorithms.transforms.svdquant import SVDQuantConfig
+from auto_round.algorithms.transforms.svdquant.wrapper import SVDQuantLinear
+from auto_round.data_type.nvfp import nv_fp4
+from auto_round.data_type.utils import update_block_global_scale_if_needed
+from auto_round.export.svdquant_nunchaku import IdentitySVDQuantModelAdapter
+from auto_round.export.svdquant_omni import save_svdquant_omni
+from auto_round.formats import get_formats
+from auto_round.schemes import PRESET_SCHEMES
+from auto_round.utils.device_manager import device_manager
+from auto_round.wrapper import WrapperWALayer
+
+
+def test_omni_tensor_layout_and_adapter_mapping(tmp_path):
+ model = torch.nn.Module()
+ model.config = {"_name_or_path": tmp_path}
+ residual = torch.nn.Linear(32, 16, dtype=torch.bfloat16)
+ for name, value in PRESET_SCHEMES["NVFP4"].to_dict().items():
+ setattr(residual, name, value)
+ down = torch.nn.Linear(32, 8, bias=False, dtype=torch.bfloat16)
+ up = torch.nn.Linear(8, 16, bias=False, dtype=torch.bfloat16)
+ model.projection = SVDQuantLinear(residual, down, up, torch.full((32,), 2, dtype=torch.bfloat16))
+ model.unquantized = torch.nn.Linear(32, 16)
+
+ class RenameAdapter(IdentitySVDQuantModelAdapter):
+ def map_modules(self, model, records):
+ return [replace(record, prefix="runtime.projection") for record in super().map_modules(model, records)]
+
+ for adapter, prefix in ((None, "projection"), (RenameAdapter(), "runtime.projection")):
+ if adapter is not None:
+ # Also exercise export of tuned residuals through the same ABI.
+ residual.data_type, residual.bits, residual.group_size = "nv_fp4", 4, 16
+ residual.weight_global_scale = torch.tensor([4.0])
+ quantized, residual.scale, _ = nv_fp4(residual.weight.float(), global_scale=residual.weight_global_scale)
+ residual.weight.data = quantized.bfloat16()
+ output = tmp_path / prefix
+ output_format = get_formats("svdquant_omni", SimpleNamespace(scheme="NVFP4"))[0]
+ assert output_format.save_quantized(output, model=model, adapter=adapter) is model
+ tensors = load_file(output / "diffusion_pytorch_model.safetensors")
+ assert set(tensors) == {
+ *(
+ f"{prefix}.{suffix}"
+ for suffix in (
+ "qweight",
+ "wscales",
+ "wtscale",
+ "wcscales",
+ "proj_down",
+ "proj_up",
+ "smooth_factor",
+ "bias",
+ )
+ ),
+ "unquantized.weight",
+ "unquantized.bias",
+ }
+ assert tensors[f"{prefix}.qweight"].shape == (16, 16)
+ assert tensors[f"{prefix}.qweight"].dtype == torch.int8
+ assert tensors[f"{prefix}.wscales"].shape == (2, 16)
+ assert tensors[f"{prefix}.wscales"].dtype == torch.float8_e4m3fn
+ assert tensors[f"{prefix}.wtscale"].shape == (1,)
+ assert tensors[f"{prefix}.wtscale"].dtype == torch.bfloat16
+ if adapter is not None:
+ torch.testing.assert_close(tensors[f"{prefix}.wtscale"], torch.tensor([0.25], dtype=torch.bfloat16))
+ torch.testing.assert_close(tensors[f"{prefix}.wscales"].float(), residual.scale.reshape(16, 2).T)
+ torch.testing.assert_close(tensors[f"{prefix}.wcscales"], torch.ones(16, dtype=torch.bfloat16))
+ torch.testing.assert_close(tensors[f"{prefix}.proj_down"], (down.weight.float() * 2).T.bfloat16())
+ torch.testing.assert_close(tensors[f"{prefix}.proj_up"], up.weight)
+ torch.testing.assert_close(tensors[f"{prefix}.smooth_factor"], torch.full((32,), 0.5, dtype=torch.bfloat16))
+ torch.testing.assert_close(tensors[f"{prefix}.bias"], residual.bias)
+ torch.testing.assert_close(tensors["unquantized.weight"], model.unquantized.weight)
+ config = json.loads((output / "config.json").read_text())
+ quantization = json.loads((output / "quantization_config.json").read_text())
+ assert config["quantization_config"] == quantization
+ assert quantization == {
+ "quant_method": "svdquant",
+ "precision": "nvfp4",
+ "rank": 8,
+ "act_unsigned": False,
+ "modules_to_not_convert": ["unquantized"],
+ }
+
+
+def test_omni_format_rejects_incompatible_schemes(tmp_path, monkeypatch):
+ output_format = get_formats("svdquant_omni", SimpleNamespace(scheme="NVFP4"))[0]
+ assert output_format.output_format == "svdquant_omni"
+ assert not output_format.is_supported_immediate_packing()
+ assert not output_format.is_supported_immediate_saving()
+ for preset in ("MXFP4", "W4A16", "NVFP4_E5M3"):
+ with pytest.raises(ValueError, match="NVFP4 W4A4 group16"):
+ get_formats("svdquant_omni", SimpleNamespace(scheme=preset))
+ for field, value in (("bits", 8), ("group_size", 32), ("act_bits", 16), ("act_dynamic", False)):
+ scheme = PRESET_SCHEMES["NVFP4"].copy()
+ setattr(scheme, field, value)
+ with pytest.raises(ValueError, match=field):
+ output_format.check_scheme_args(scheme)
+ model = torch.nn.Module()
+ residual = torch.nn.Linear(32, 16)
+ for name, value in PRESET_SCHEMES["MXFP4"].to_dict().items():
+ setattr(residual, name, value)
+ model.projection = SVDQuantLinear(
+ residual, torch.nn.Linear(32, 8, bias=False), torch.nn.Linear(8, 16, bias=False), torch.ones(32)
+ )
+ with pytest.raises(ValueError, match="NVFP4 W4A4 group16"):
+ save_svdquant_omni(model, tmp_path / "invalid")
+ with pytest.raises(ValueError, match="incompatible residual scheme"):
+ output_format._validate_svd_layer_overrides(model, {"projection": "MXFP4"})
+
+ from auto_round import AutoRound
+ from auto_round.algorithms.quantization.rtn.config import RTNConfig
+ from auto_round.algorithms.transforms.svdquant import SVDQuantConfig
+
+ created = {}
+
+ class FakeCompressor:
+ def __init__(self, config, **kwargs):
+ created.update(kwargs)
+
+ monkeypatch.setattr("auto_round.autoround._build_model_type_ctor_kwargs", lambda *args, **kwargs: ("llm", {}))
+ monkeypatch.setattr("auto_round.autoround._get_compressor_class", lambda model_type, base_cls: FakeCompressor)
+ for format in ("svdquant_omni", "svdquant_omni,fake"):
+ AutoRound(
+ "dummy-model",
+ scheme="NVFP4",
+ format=format,
+ alg_configs=[SVDQuantConfig(smooth_enabled=False), RTNConfig(disable_opt_rtn=True)],
+ )
+ assert created["format"] == format
+
+
+def test_omni_zero_shot_rtn_fuses_qkv_with_shared_scale(tmp_path, monkeypatch):
+ monkeypatch.setattr(device_manager, "_device_map", "cpu")
+ monkeypatch.setattr(device_manager, "_device_list", ["cpu"])
+ monkeypatch.setattr(device_manager, "_major_device", "cpu")
+
+ class SelfAttention(torch.nn.Module):
+ is_cross_attention = False
+
+ def __init__(self):
+ super().__init__()
+ self.to_q = torch.nn.Linear(32, 32, bias=False)
+ self.to_k = torch.nn.Linear(32, 32, bias=False)
+ self.to_v = torch.nn.Linear(32, 32, bias=False)
+
+ class WanBlock(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.attn = SelfAttention()
+
+ model = torch.nn.Module()
+ model.blocks = torch.nn.ModuleList([WanBlock()])
+ model.config = {"_name_or_path": str(tmp_path)}
+ scheme = PRESET_SCHEMES["NVFP4"].copy()
+ for name, module in model.named_modules():
+ module.global_name = name
+ if isinstance(module, torch.nn.Linear):
+ for key, value in scheme.to_dict().items():
+ setattr(module, key, value)
+ module.scale_dtype = torch.float32
+
+ block_name = "blocks.0"
+ orchestrator = SimpleNamespace(
+ model_context=SimpleNamespace(
+ model=model,
+ amp=False,
+ amp_dtype=torch.float32,
+ is_diffusion=True,
+ is_moe_model=False,
+ output_config=None,
+ ),
+ compress_context=SimpleNamespace(
+ enable_torch_compile=False,
+ low_gpu_mem_usage=False,
+ cache_device=torch.device("cpu"),
+ clear_memory=lambda: None,
+ formats=[SimpleNamespace(format_name="svdquant_omni")],
+ ),
+ calibration_context=SimpleNamespace(batch_size=1, batch_dim=0),
+ scheme_context=scheme,
+ scale_dtype=torch.float32,
+ nblocks=1,
+ quant_block_list=[[block_name]],
+ data_type=scheme.data_type,
+ batch_dim=0,
+ batch_size=1,
+ cache_device="cpu",
+ amp=False,
+ amp_dtype=torch.float32,
+ shared_cache_keys=(),
+ )
+ composer = AlgorithmComposer(
+ [
+ SVDQuantConfig(rank=2, smooth_enabled=False, residual_iters=1, low_rank_dtype="fp32"),
+ RTNConfig(disable_opt_rtn=True),
+ ],
+ orchestrator=orchestrator,
+ )
+ composer.prepare_run()
+
+ block = model.blocks[0]
+ # Match the Orchestrator's initial NVFP4 scale setup before SVD replacement.
+ update_block_global_scale_if_needed(block, scheme.data_type, scheme.group_size)
+ ctx = BlockContext(
+ model=model,
+ block_names=[block_name],
+ block_name=block_name,
+ block_index=0,
+ block_cnt=1,
+ )
+ composer.compress_block(block, fp_inputs=None, input_others={}, block_ctx=ctx)
+
+ residual_scales = []
+ for name in ("to_q", "to_k", "to_v"):
+ residual = getattr(block.attn, name).residual_linear
+ while isinstance(residual, WrapperWALayer):
+ residual = residual.orig_layer
+ residual_scales.append(residual.weight_global_scale)
+ torch.testing.assert_close(residual_scales[0], residual_scales[1])
+ torch.testing.assert_close(residual_scales[0], residual_scales[2])
+
+ output_dir = tmp_path / "svdquant_omni"
+ save_svdquant_omni(model, output_dir)
+ tensors = load_file(output_dir / "diffusion_pytorch_model.safetensors")
+ assert "blocks.0.attn.to_qkv.wtscale" in tensors