Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Support SVDQuant (#2095)

Wang, Changccb365886a5c
diff --git a/auto_round/algorithms/block_runner.py b/auto_round/algorithms/block_runner.py
index b2771011d..6b735c9db 100644
--- a/auto_round/algorithms/block_runner.py
+++ b/auto_round/algorithms/block_runner.py
@@ -224,41 +224,36 @@ class BlockForwardRunner:
if is_returned_list and self.batch_size != 1: # split it to 1
if batch_output_dict:
for key, value in batch_output_dict.items():
- output_dict.setdefault(key, []).extend(self.split_outputs(value))
+ output_dict.setdefault(key, []).extend(
+ item.to(out_device) for item in self.split_outputs(value)
+ )
output = self.split_outputs(output)
else:
if batch_output_dict:
for key, value in batch_output_dict.items():
- output_dict.setdefault(key, []).append(value)
+ output_dict.setdefault(key, []).append(value.to(out_device))
output = [output]
- outputs.extend(output)
+ outputs.extend(item.to(out_device) for item in output)
+ del raw_output, batch_output_dict, output, batch_inputs, batch_others
if not outputs:
raise RuntimeError("BlockForwardRunner.forward: no outputs collected.")
if is_returned_list:
- result = [o.to(out_device) for o in outputs]
+ result = outputs
if output_dict:
- self.last_output_dict = {key: [o.to(out_device) for o in values] for key, values in output_dict.items()}
+ self.last_output_dict = output_dict
self.last_output_dict["hidden_states"] = result
return result
else:
- if self.batch_size == 1:
- outputs = [output.unsqueeze(dim=self.batch_dim).to(out_device) for output in outputs]
- if output_dict:
- output_dict = {
- key: [value.unsqueeze(dim=self.batch_dim).to(out_device) for value in values]
- for key, values in output_dict.items()
- }
-
- outputs = torch.cat(outputs, dim=self.batch_dim).to(out_device)
+ outputs = torch.cat(outputs, dim=self.batch_dim)
if output_dict:
self.last_output_dict = {
- key: torch.cat(values, dim=self.batch_dim).to(out_device) for key, values in output_dict.items()
+ key: torch.cat(values, dim=self.batch_dim) for key, values in output_dict.items()
}
self.last_output_dict["hidden_states"] = outputs
- return outputs.to(out_device)
+ return outputs
# ── Input selection ──────────────────────────────────────────────────────
diff --git a/auto_round/algorithms/registry.py b/auto_round/algorithms/registry.py
index 081b09f0f..b79647e7a 100644
--- a/auto_round/algorithms/registry.py
+++ b/auto_round/algorithms/registry.py
@@ -47,6 +47,7 @@ def _ensure_pipeline_members_registered() -> None:
"auto_round.algorithms.quantization.sign_roundv2.quantizer",
"auto_round.algorithms.quantization.adam_round.adam",
"auto_round.algorithms.transforms.awq.base",
+ "auto_round.algorithms.transforms.svdquant.apply",
):
importlib.import_module(module_name)
_pipeline_members_registered = True
diff --git a/auto_round/algorithms/transforms/svdquant/__init__.py b/auto_round/algorithms/transforms/svdquant/__init__.py
new file mode 100644
index 000000000..14099b8a1
--- /dev/null
+++ b/auto_round/algorithms/transforms/svdquant/__init__.py
@@ -0,0 +1,18 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round.algorithms.transforms.svdquant.config import SVDQuantConfig
+from auto_round.algorithms.transforms.svdquant.wrapper import SVDQuantLinear
+
+__all__ = ["SVDQuantConfig", "SVDQuantLinear"]
diff --git a/auto_round/algorithms/transforms/svdquant/apply.py b/auto_round/algorithms/transforms/svdquant/apply.py
new file mode 100644
index 000000000..1bc582f95
--- /dev/null
+++ b/auto_round/algorithms/transforms/svdquant/apply.py
@@ -0,0 +1,715 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import math
+import random
+from dataclasses import dataclass, field
+from functools import partial
+from typing import Any
+
+import torch
+
+import auto_round.algorithms.transforms.svdquant.residual as residual_module
+from auto_round.algorithms.registry import register_pipeline_member
+from auto_round.algorithms.transforms.base import BasePreprocessor
+from auto_round.algorithms.transforms.svdquant.config import SVDQuantConfig
+from auto_round.algorithms.transforms.svdquant.residual import (
+ ActivationQuantScheme,
+ ResidualQuantScheme,
+ iterate_residual_decomposition,
+ truncated_svd,
+)
+from auto_round.algorithms.transforms.svdquant.smooth import (
+ SmoothCandidate,
+ absmax_channel_span,
+ build_alpha_beta_candidates,
+ build_smooth_scale,
+ select_best_layer_candidate,
+ summarize_smooth_scale,
+ validate_smooth_scale_for_deployment,
+)
+from auto_round.algorithms.transforms.svdquant.smooth_adapters import SmoothSearchGroup, discover_svdquant_groups
+from auto_round.algorithms.transforms.svdquant.wrapper import SVDQuantLinear
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme
+from auto_round.utils.model import map_nested_tensors
+
+_SCHEME_ATTRS = set(QuantizationScheme.get_attributes())
+_RUNTIME_QUANT_ATTRS = {"scale_dtype", "weight_global_scale", "tuning_device"}
+_MXFP4_ALIASES = frozenset({"mx_fp", "mx_fp4", "mx_fp4e2m1"})
+
+
+def _detach_to_cpu(value: Any) -> Any:
+ return map_nested_tensors(value, lambda tensor: tensor.detach().to("cpu", copy=True))
+
+
+def _move_to_device(value: Any, device: torch.device, dtype: torch.dtype | None = None) -> Any:
+ def move(tensor: torch.Tensor) -> torch.Tensor:
+ target_dtype = dtype if dtype is not None and tensor.is_floating_point() else tensor.dtype
+ return tensor.to(device=device, dtype=target_dtype)
+
+ return map_nested_tensors(value, move)
+
+
+@dataclass
+class CapturedEvaluation:
+ args: tuple[Any, ...]
+ kwargs: dict[str, Any]
+ output: Any
+
+
+@dataclass
+class SmoothGroupCalibration:
+ group: SmoothSearchGroup
+ limit: int
+ projection_inputs: list[torch.Tensor] = field(default_factory=list)
+ evaluation_calls: list[CapturedEvaluation] = field(default_factory=list)
+ seen_calls: int = 0
+ pending_slot: int | None = None
+ pending_input: torch.Tensor | None = None
+ random: random.Random = field(default_factory=lambda: random.Random(0))
+
+ def begin_call(self, inputs: torch.Tensor) -> None:
+ self.seen_calls += 1
+ if len(self.projection_inputs) < self.limit:
+ slot = len(self.projection_inputs)
+ else:
+ candidate = self.random.randrange(self.seen_calls)
+ slot = candidate if candidate < self.limit else None
+ self.pending_slot = slot
+ self.pending_input = _detach_to_cpu(inputs) if slot is not None else None
+
+ def finish_call(self, args: tuple[Any, ...], kwargs: dict[str, Any], output: Any) -> None:
+ slot = self.pending_slot
+ captured_input = self.pending_input
+ self.pending_slot = None
+ self.pending_input = None
+ if slot is None or captured_input is None:
+ return
+ captured = CapturedEvaluation(_detach_to_cpu(args), _detach_to_cpu(kwargs), _detach_to_cpu(output))
+ if slot == len(self.projection_inputs):
+ self.projection_inputs.append(captured_input)
+ self.evaluation_calls.append(captured)
+ else:
+ self.projection_inputs[slot] = captured_input
+ self.evaluation_calls[slot] = captured
+
+
+@register_pipeline_member(SVDQuantConfig)
+class SVDQuantTransform(BasePreprocessor):
+ """Split target linears into a quantized residual and an FP low-rank branch."""
+
+ def __init__(self, config: SVDQuantConfig) -> None:
+ super().__init__(config)
+ self._configured_block_names: tuple[str, ...] = ()
+ self._block_groups: dict[str, list[SmoothSearchGroup]] = {}
+ self._smooth_calibration: dict[str, SmoothGroupCalibration] = {}
+ self._target_modules = config.target_modules
+ if self._target_modules is None and config.model_adapter == "flux":
+ from auto_round.export.svdquant_adapters import FLUX_SVDQUANT_TARGET_MODULES
+
+ self._target_modules = FLUX_SVDQUANT_TARGET_MODULES
+
+ def bind(self, orchestrator) -> None:
+ super().bind(orchestrator)
+ nblocks = getattr(orchestrator, "nblocks", 1)
+ if nblocks != 1:
+ raise ValueError(f"SVDQuant requires nblocks=1, got nblocks={nblocks}.")
+ quant_block_list = getattr(orchestrator, "quant_block_list", None) or ()
+ self._configured_block_names = tuple(
+ block_name for block_group in quant_block_list for block_name in block_group
+ )
+
+ def prepare_run(self, composer=None) -> None:
+ self._block_groups.clear()
+ if self.model is None:
+ return
+ self._resolve_model_adapter(self.model)
+ for block_name in self._configured_block_names:
+ block = self.model.get_submodule(block_name)
+ self._block_groups[block_name] = discover_svdquant_groups(block, self._is_target)
+ logger.info(
+ "SVDQuant: resolved %d projection groups across %d blocks.",
+ sum(len(groups) for groups in self._block_groups.values()),
+ len(self._block_groups),
+ )
+
+ def _resolve_model_adapter(self, model: torch.nn.Module | None, block: torch.nn.Module | None = None) -> str:
+ model_adapter = self.config.model_adapter or "auto"
+ if model_adapter == "auto":
+ model_config = getattr(model, "config", None)
+ class_name = (
+ model_config.get("_class_name", type(model).__name__)
+ if hasattr(model_config, "get")
+ else type(model).__name__
+ )
+ if "fluxtransformer" in str(class_name).lower():
+ model_adapter = "flux"
+ elif block is not None and block.__class__.__name__ in {
+ "FluxTransformerBlock",
+ "FluxSingleTransformerBlock",
+ }:
+ model_adapter = "flux"
+ if model is not None:
+ model._autoround_svdquant_model_adapter = model_adapter
+ if model_adapter == "flux":
+ from auto_round.algorithms.transforms.svdquant.smooth_adapters.flux import warn_if_unverified_flux_model
+
+ if model is not None:
+ warn_if_unverified_flux_model(model)
+ self._target_modules = self.config.target_modules
+ if self._target_modules is None and model_adapter == "flux":
+ from auto_round.export.svdquant_adapters import FLUX_SVDQUANT_TARGET_MODULES
+
+ self._target_modules = FLUX_SVDQUANT_TARGET_MODULES
+ return model_adapter
+
+ def register_fp_input_forward_hooks(self, block) -> list:
+ if not self.config.smooth_enabled:
+ return []
+ self._resolve_model_adapter(self.model, block)
+ self._clear_smooth_calibration()
+ block_name = str(getattr(block, "global_name", ""))
+ groups = self._block_groups.get(block_name)
+ if groups is None:
+ groups = discover_svdquant_groups(block, self._is_target)
+ self._block_groups[block_name] = groups
+ self._smooth_calibration = {
+ group.key: SmoothGroupCalibration(group, self.config.smooth_max_calibration_calls) for group in groups
+ }
+
+ projection_owners = {}
+ evaluation_owners = {}
+ modules = {}
+ for calibration in self._smooth_calibration.values():
+ projection_module = calibration.group.projection_input_module
+ evaluation_module = calibration.group.evaluation_module
+ projection_owners.setdefault(id(projection_module), []).append(calibration)
+ evaluation_owners.setdefault(id(evaluation_module), []).append(calibration)
+ modules[id(projection_module)] = projection_module
+ modules[id(evaluation_module)] = evaluation_module
+
+ def collect_calibration(module, inputs, kwargs, output):
+ for calibration in projection_owners.get(id(module), ()):
+ if inputs and torch.is_tensor(inputs[0]):
+ value = inputs[0]
+ if value.shape[-1] == calibration.group.projections[0].in_features:
+ calibration.begin_call(value)
+ for calibration in evaluation_owners.get(id(module), ()):
+ calibration.finish_call(inputs, kwargs, output)
+
+ return [module.register_forward_hook(collect_calibration, with_kwargs=True) for module in modules.values()]
+
+ @torch.no_grad()
+ def pre_quantize_block(self, ctx) -> None:
+ if len(ctx.block_names) != 1:
+ raise ValueError(f"SVDQuant requires one block at a time, got {ctx.block_names!r}.")
+ block_name = ctx.block_name
+ block = ctx.model.get_submodule(block_name)
+ self._resolve_model_adapter(ctx.model, block)
+ groups = self._block_groups.get(block_name)
+ if groups is None:
+ groups = discover_svdquant_groups(block, self._is_target)
+ self._block_groups[block_name] = groups
+
+ if self.config.smooth_enabled:
+ self._pre_quantize_smoothed_block(block, groups)
+ return
+
+ local_names = {id(module): name for name, module in block.named_modules() if name}
+ replacements = []
+ for group in groups:
+ wrappers = self._decompose_group(group)
+ for projection, wrapper in zip(group.projections, wrappers):
+ local_name = local_names.get(id(projection))
+ if local_name is None:
+ raise ValueError(f"SVDQuant could not locate projection {self._module_name(projection)!r}.")
+ replacements.append((local_name, wrapper))
+
+ for local_name, wrapper in replacements:
+ _set_child_module(block, local_name, wrapper)
+
+ def post_quantize_block(self, ctx) -> None:
+ self._clear_smooth_calibration()
+ self._block_groups.pop(ctx.block_name, None)
+
+ def finalize_run(self) -> None:
+ self._clear_smooth_calibration()
+ self._block_groups.clear()
+
+ def _clear_smooth_calibration(self) -> None:
+ self._smooth_calibration.clear()
+
+ def _pre_quantize_smoothed_block(self, block: torch.nn.Module, groups: list[SmoothSearchGroup]) -> None:
+ if not self._smooth_calibration:
+ raise ValueError("SVDQuant smooth calibration inputs are missing for the current block.")
+ local_names = {id(module): name for name, module in block.named_modules() if name}
+ try:
+ selected_scales = {}
+ for group in groups:
+ calibration = self._smooth_calibration.get(group.key)
+ if calibration is None or not calibration.projection_inputs or not calibration.evaluation_calls:
+ raise ValueError(f"SVDQuant smooth calibration inputs are missing for group {group.key!r}.")
+ selected_scales[group.key] = self._search_group_scale(calibration, block, local_names)
+
+ replacements = []
+ for group in groups:
+ calibration = self._smooth_calibration[group.key]
+ wrappers = self._decompose_smoothed_group(calibration, selected_scales[group.key], block, local_names)
+ for projection, wrapper in zip(group.projections, wrappers):
+ local_name = local_names.get(id(projection))
+ if local_name is None:
+ raise ValueError(f"SVDQuant could not locate projection {self._module_name(projection)!r}.")
+ replacements.append((local_name, wrapper))
+ for local_name, wrapper in replacements:
+ _set_child_module(block, local_name, wrapper)
+ finally:
+ self._clear_smooth_calibration()
+
+ def _search_group_scale(
+ self,
+ calibration: SmoothGroupCalibration,
+ block: torch.nn.Module,
+ local_names: dict[int, str],
+ ) -> torch.Tensor:
+ group = calibration.group
+ device = group.projections[0].weight.device
+ x_span = torch.stack([absmax_channel_span(inputs, -1) for inputs in calibration.projection_inputs], dim=0).amax(
+ dim=0
+ )
+ weights = [
+ projection.weight.detach().to(device=device, dtype=torch.float32) for projection in group.projections
+ ]
+ w_span = absmax_channel_span(torch.cat(weights, dim=0), 1).cpu()
+ scored = []
+ for alpha, beta in build_alpha_beta_candidates(self.config.smooth_num_grids):
+ scale = build_smooth_scale(x_span, w_span, alpha, beta, eps=self.config.smooth_eps)
+ try:
+ scale = validate_smooth_scale_for_deployment(
+ scale, dtype=group.projections[0].weight.dtype, module_name=group.key
+ ).to(torch.float32)
+ error = self._score_group_wrappers(
+ calibration, self._candidate_group_wrappers(group, scale), block, local_names
+ )
+ except (RuntimeError, ValueError, TypeError) as exc:
+ logger.debug("Skipping SVDQuant smooth candidate (%s, %s) for %s: %s", alpha, beta, group.key, exc)
+ error = float("inf")
+ candidate = SmoothCandidate(alpha, beta, scale)
+ scored.append((candidate, error))
+ selected = select_best_layer_candidate(scored, module_name=group.key)
+ error = next(error for candidate, error in reversed(scored) if candidate is selected)
+ self._log_selected_smooth_candidate(group.key, selected, error)
+ return selected.scale
+
+ @staticmethod
+ def _log_selected_smooth_candidate(module_name: str, candidate: SmoothCandidate, error: float) -> None:
+ stats = summarize_smooth_scale(candidate.scale)
+ logger.info(
+ "SVDQuant smooth selected for %s: alpha=%.6g beta=%.6g error=%.6g "
+ "scale_min=%.6g scale_max=%.6g scale_ratio=%.6g below_1e-3=%d above_20=%d",
+ module_name,
+ candidate.alpha,
+ candidate.beta,
+ error,
+ stats.minimum,
+ stats.maximum,
+ stats.ratio,
+ stats.below_min_count,
+ stats.above_max_count,
+ )
+ if stats.below_min_count or stats.above_max_count:
+ logger.warning(
+ "SVDQuant smooth scale for %s contains extreme values: below_1e-3=%d above_20=%d",
+ module_name,
+ stats.below_min_count,
+ stats.above_max_count,
+ )
+
+ def _score_group_wrappers(
+ self,
+ calibration: SmoothGroupCalibration,
+ wrappers: list[SVDQuantLinear],
+ block: torch.nn.Module,
+ local_names: dict[int, str],
+ ) -> float:
+ group = calibration.group
+ replacements = []
+ for projection, wrapper in zip(group.projections, wrappers):
+ local_name = local_names.get(id(projection))
+ if local_name is None:
+ raise ValueError(f"SVDQuant could not locate projection {self._module_name(projection)!r}.")
+ replacements.append((local_name, projection, wrapper))
+ try:
+ for local_name, _, wrapper in replacements:
+ _set_child_module(block, local_name, wrapper)
+ error = torch.zeros((), dtype=torch.float64)
+ for call in calibration.evaluation_calls:
+ evaluation_module = group.evaluation_module
+ if len(group.projections) == 1 and evaluation_module is group.projections[0]:
+ evaluation_module = wrappers[0]
+ device = group.projections[0].weight.device
+ dtype = group.projections[0].weight.dtype
+ args = _move_to_device(call.args, device, dtype)
+ kwargs = group.filter_evaluation_kwargs(_move_to_device(call.kwargs, device, dtype))
+ actual = group.normalize_output(evaluation_module(*args, **kwargs))
+ reference = tuple(tensor.to(device) for tensor in group.normalize_output(call.output))
+ if len(actual) != len(reference):
+ raise ValueError("SVDQuant smooth output tensor count changed.")
+ for actual_tensor, reference_tensor in zip(actual, reference):
+ if actual_tensor.shape != reference_tensor.shape:
+ raise ValueError("SVDQuant smooth output tensor shape changed.")
+ error += torch.sum((actual_tensor.float() - reference_tensor.float()).square()).double().cpu()
+ return error.item()
+ finally:
+ for local_name, projection, _ in replacements:
+ _set_child_module(block, local_name, projection)
+
+ def _candidate_group_wrappers(self, group: SmoothSearchGroup, scale: torch.Tensor) -> list[SVDQuantLinear]:
+ weights = [
+ projection.weight.detach().to(torch.float32) * scale.to(projection.weight.device)
+ for projection in group.projections
+ ]
+ stacked = torch.cat(weights, dim=0)
+ output_sizes = [projection.out_features for projection in group.projections]
+ rank = min(self.config.rank, *stacked.shape)
+ _, down, up = truncated_svd(stacked, rank)
+ low_rank_dtype = self._resolve_low_rank_dtype(group.projections[0].weight.dtype)
+ deployed_down = down.to(low_rank_dtype)
+ deployed_up = up.to(low_rank_dtype)
+ low_rank_parts = (deployed_up.float() @ deployed_down.float()).split(output_sizes, dim=0)
+ residuals = []
+ for projection, weight, low_rank in zip(group.projections, weights, low_rank_parts):
+ residual = (weight - low_rank).to(projection.weight.dtype)
+ residuals.append(residual_module.rtn_qdq_residual(residual, self._residual_quant_scheme(projection)))
+ return self._build_group_wrappers(
+ group,
+ residuals,
+ deployed_down,
+ deployed_up.split(output_sizes, dim=0),
+ scale,
+ activation_scheme=self._group_activation_quant_scheme(group),
+ )
+
+ def _decompose_smoothed_group(
+ self,
+ calibration: SmoothGroupCalibration,
+ scale: torch.Tensor,
+ block: torch.nn.Module,
+ local_names: dict[int, str],
+ ) -> list[SVDQuantLinear]:
+ group = calibration.group
+ weights = [
+ projection.weight.detach().to(torch.float32) * scale.to(projection.weight.device)
+ for projection in group.projections
+ ]
+ stacked = torch.cat(weights, dim=0)
+ output_sizes = [projection.out_features for projection in group.projections]
+ rank = min(self.config.rank, *stacked.shape)
+ low_rank_dtype = self._resolve_low_rank_dtype(group.projections[0].weight.dtype)
+ if self.config.residual_iters == 1:
+ _, down, up = truncated_svd(stacked, rank)
+ deployed_down = down.to(low_rank_dtype)
+ deployed_up = up.to(low_rank_dtype)
+ low_rank = deployed_up.float() @ deployed_down.float()
+ residuals = [
+ residual.to(projection.weight.dtype)
+ for residual, projection in zip(stacked.sub(low_rank).split(output_sizes, dim=0), group.projections)
+ ]
+ else:
+ residuals, deployed_down, deployed_up = self._iterate_smoothed_group_residual(
+ calibration, block, local_names, stacked, rank, low_rank_dtype, scale
+ )
+ return self._build_group_wrappers(
+ group, residuals, deployed_down, deployed_up.split(output_sizes, dim=0), scale
+ )
+
+ def _iterate_smoothed_group_residual(
+ self,
+ calibration: SmoothGroupCalibration,
+ block: torch.nn.Module,
+ local_names: dict[int, str],
+ stacked: torch.Tensor,
+ rank: int,
+ low_rank_dtype: torch.dtype,
+ scale: torch.Tensor,
+ ) -> tuple[list[torch.Tensor], torch.Tensor, torch.Tensor]:
+ group = calibration.group
+ output_sizes = [projection.out_features for projection in group.projections]
+ quantized_residual = torch.zeros_like(stacked)
+ best = None
+ best_error = float("inf")
+ activation_scheme = self._group_activation_quant_scheme(group)
+ for iteration in range(1, self.config.residual_iters + 1):
+ _, down, up = truncated_svd(stacked - quantized_residual, rank)
+ deployed_down = down.to(low_rank_dtype)
+ deployed_up = up.to(low_rank_dtype)
+ low_rank = deployed_up.float() @ deployed_down.float()
+ residual_parts = (stacked - low_rank).split(output_sizes, dim=0)
+ qdq_residuals = [
+ residual_module.rtn_qdq_residual(
+ residual.to(projection.weight.dtype), self._residual_quant_scheme(projection)
+ )
+ for residual, projection in zip(residual_parts, group.projections)
+ ]
+ quantized_residual = torch.cat([residual.float() for residual in qdq_residuals], dim=0)
+ wrappers = self._build_group_wrappers(
+ group,
+ qdq_residuals,
+ deployed_down,
+ deployed_up.split(output_sizes, dim=0),
+ scale,
+ activation_scheme=activation_scheme,
+ )
+ error = self._score_group_wrappers(calibration, wrappers, block, local_names)
+ accepted = math.isfinite(error) and error <= best_error
+ if accepted:
+ best = (deployed_down.clone(), deployed_up.clone(), iteration)
+ best_error = error
+ elif self.config.residual_early_stop and best is not None:
+ logger.info(
+ "SVDQuant residual early stop for %s at iteration %d: output error %.6g > best %.6g",
+ group.key,
+ iteration,
+ error,
+ best_error,
+ )
+ break
+ if best is None:
+ raise ValueError(f"SVDQuant residual iteration failed for group {group.key!r}.")
+ deployed_down, deployed_up, selected_iteration = best
+ logger.info(
+ "SVDQuant residual selected iteration %d/%d for %s: output error %.6g",
+ selected_iteration,
+ self.config.residual_iters,
+ group.key,
+ best_error,
+ )
+ low_rank = deployed_up.float() @ deployed_down.float()
+ residuals = [
+ residual.to(projection.weight.dtype)
+ for residual, projection in zip((stacked - low_rank).split(output_sizes, dim=0), group.projections)
+ ]
+ return residuals, deployed_down, deployed_up
+
+ def _decompose_group(self, group: SmoothSearchGroup) -> list[SVDQuantLinear]:
+ weights = [projection.weight.detach().to(torch.float32) for projection in group.projections]
+ stacked = torch.cat(weights, dim=0)
+ output_sizes = [projection.out_features for projection in group.projections]
+ rank = min(self.config.rank, *stacked.shape)
+ low_rank_dtype = self._resolve_low_rank_dtype(group.projections[0].weight.dtype)
+
+ if self.config.residual_iters == 1:
+ _, down, up = truncated_svd(stacked, rank)
+ deployed_down = down.to(low_rank_dtype)
+ deployed_up = up.to(low_rank_dtype)
+ deployed_low_rank = deployed_up.float() @ deployed_down.float()
+ residuals = [
+ residual.to(projection.weight.dtype)
+ for residual, projection in zip(
+ (stacked - deployed_low_rank).split(output_sizes, dim=0), group.projections
+ )
+ ]
+ else:
+ scheme = self._shared_residual_scheme(group)
+ result = iterate_residual_decomposition(
+ stacked,
+ rank=rank,
+ scheme=scheme,
+ iterations=self.config.residual_iters,
+ early_stop=self.config.residual_early_stop,
+ residual_dtype=group.projections[0].weight.dtype,
+ low_rank_dtype=low_rank_dtype,
+ )
+ deployed_down = result.down
+ deployed_up = result.up
+ residuals = [
+ residual.to(projection.weight.dtype)
+ for residual, projection in zip(result.residual.split(output_sizes, dim=0), group.projections)
+ ]
+ logger.info(
+ "SVDQuant residual selected iteration %d/%d for %s: weight error %.6g",
+ result.selected_iteration,
+ self.config.residual_iters,
+ group.key,
+ result.error,
+ )
+
+ up_parts = deployed_up.split(output_sizes, dim=0)
+ smooth = torch.ones(stacked.shape[1], device=stacked.device, dtype=torch.float32)
+ return self._build_group_wrappers(group, residuals, deployed_down, up_parts, smooth)
+
+ def _build_group_wrappers(
+ self,
+ group: SmoothSearchGroup,
+ residuals: list[torch.Tensor],
+ down: torch.Tensor,
+ up_parts: tuple[torch.Tensor, ...],
+ smooth: torch.Tensor,
+ activation_scheme: ActivationQuantScheme | None = None,
+ ) -> list[SVDQuantLinear]:
+ wrappers = []
+ rank = down.shape[0]
+ input_smooth = smooth.reciprocal()
+ for projection, residual_weight, up in zip(group.projections, residuals, up_parts):
+ residual = self._new_linear_like(projection, residual_weight, projection.bias)
+ lora_down = torch.nn.Linear(
+ projection.in_features,
+ rank,
+ bias=False,
+ dtype=down.dtype,
+ device=projection.weight.device,
+ )
+ lora_up = torch.nn.Linear(
+ rank,
+ projection.out_features,
+ bias=False,
+ dtype=up.dtype,
+ device=projection.weight.device,
+ )
+ lora_down.weight.copy_(down)
+ lora_up.weight.copy_(up)
+ self._mark_unquantized(lora_down)
+ self._mark_unquantized(lora_up)
+ self._copy_quant_attrs(projection, residual, suffix=".residual_linear")
+ activation_qdq = (
+ None
+ if activation_scheme is None
+ else partial(residual_module.rtn_qdq_activation, scheme=activation_scheme)
+ )
+ wrappers.append(
+ SVDQuantLinear(
+ residual,
+ lora_down,
+ lora_up,
+ input_smooth.to(projection.weight.dtype),
+ activation_qdq=activation_qdq,
+ )
+ )
+ return wrappers
+
+ def _shared_residual_scheme(self, group: SmoothSearchGroup) -> ResidualQuantScheme:
+ schemes = tuple(self._residual_quant_scheme(projection) for projection in group.projections)
+ if any(scheme != schemes[0] for scheme in schemes[1:]):
+ raise ValueError(f"SVDQuant group {group.key!r} has inconsistent residual quantization schemes.")
+ return schemes[0]
+
+ def _group_activation_quant_scheme(self, group: SmoothSearchGroup) -> ActivationQuantScheme:
+ schemes = tuple(self._activation_quant_scheme(projection) for projection in group.projections)
+ if any(scheme != schemes[0] for scheme in schemes[1:]):
+ raise ValueError(f"SVDQuant group {group.key!r} has inconsistent activation quantization schemes.")
+ return schemes[0]
+
+ def _activation_quant_scheme(self, module: torch.nn.Linear) -> ActivationQuantScheme:
+ attributes = {
+ "data_type": "act_data_type",
+ "bits": "act_bits",
+ "group_size": "act_group_size",
+ "sym": "act_sym",
+ }
+ missing = [
+ source for source in attributes.values() if not hasattr(module, source) or getattr(module, source) is None
+ ]
+ if missing:
+ raise ValueError(
+ f"SVDQuant smooth calibration requires a complete activation scheme for "
+ f"{self._module_name(module)!r}; missing: {', '.join(missing)}."
+ )
+ return ActivationQuantScheme(**{target: getattr(module, source) for target, source in attributes.items()})
+
+ def _residual_quant_scheme(self, module: torch.nn.Linear) -> ResidualQuantScheme:
+ required = ("data_type", "bits", "group_size", "sym")
+ missing = [attr for attr in required if not hasattr(module, attr) or getattr(module, attr) is None]
+ if missing:
+ raise ValueError(
+ f"SVDQuant residual iteration requires a complete quantization scheme for "
+ f"{self._module_name(module)!r}; missing: {', '.join(missing)}."
+ )
+ return ResidualQuantScheme(**{attr: getattr(module, attr) for attr in required})
+
+ def _is_target(self, name: str, module: torch.nn.Module) -> bool:
+ if not isinstance(module, torch.nn.Linear):
+ return False
+ full_name = str(getattr(module, "global_name", name))
+ if self._target_modules and not any(
+ pattern in name or pattern in full_name for pattern in self._target_modules
+ ):
+ return False
+ if self.config.exclude_modules and any(
+ pattern in name or pattern in full_name for pattern in self.config.exclude_modules
+ ):
+ return False
+ return True
+
+ @staticmethod
+ def _new_linear_like(module: torch.nn.Linear, weight: torch.Tensor, bias: torch.Tensor | None):
+ residual = torch.nn.Linear(
+ module.in_features,
+ module.out_features,
+ bias=bias is not None,
+ dtype=module.weight.dtype,
+ device=module.weight.device,
+ )
+ residual.weight.copy_(weight.to(module.weight.dtype))
+ if bias is not None:
+ residual.bias.copy_(bias.detach().to(module.weight.dtype))
+ return residual
+
+ @staticmethod
+ def _mark_unquantized(module: torch.nn.Module) -> None:
+ module.bits = 16
+ module.act_bits = 16
+
+ @staticmethod
+ def _copy_quant_attrs(src: torch.nn.Module, dst: torch.nn.Module, suffix: str) -> None:
+ for attr in _SCHEME_ATTRS | _RUNTIME_QUANT_ATTRS:
+ if hasattr(src, attr):
+ setattr(dst, attr, getattr(src, attr))
+ if getattr(dst, "bits", None) == 4 and getattr(dst, "data_type", None) in _MXFP4_ALIASES:
+ dst.data_type = f"{dst.data_type}_rceil"
+ if getattr(dst, "act_bits", None) == 4 and getattr(dst, "act_data_type", None) in _MXFP4_ALIASES:
+ dst.act_data_type = f"{dst.act_data_type}_rceil"
+ if hasattr(src, "global_name"):
+ dst.global_name = f"{src.global_name}{suffix}"
+
+ def _resolve_low_rank_dtype(self, fallback: torch.dtype) -> torch.dtype:
+ dtype = str(self.config.low_rank_dtype).lower()
+ if dtype in {"bf16", "bfloat16"}:
+ return torch.bfloat16
+ if dtype in {"fp16", "float16"}:
+ return torch.float16
+ if dtype in {"fp32", "float32"}:
+ return torch.float32
+ return fallback
+
+ @staticmethod
+ def _module_name(module: torch.nn.Module) -> str:
+ return str(getattr(module, "global_name", module.__class__.__name__))
+
+
+def _set_child_module(root: torch.nn.Module, name: str, module: torch.nn.Module) -> None:
+ parts = name.split(".")
+ parent = root
+ for part in parts[:-1]:
+ parent = (
+ parent[int(part)]
+ if part.isdigit() and isinstance(parent, (torch.nn.ModuleList, torch.nn.Sequential))
+ else getattr(parent, part)
+ )
+ leaf = parts[-1]
+ if leaf.isdigit() and isinstance(parent, (torch.nn.ModuleList, torch.nn.Sequential)):
+ parent[int(leaf)] = module
+ else:
+ setattr(parent, leaf, module)
diff --git a/auto_round/algorithms/transforms/svdquant/config.py b/auto_round/algorithms/transforms/svdquant/config.py
new file mode 100644
index 000000000..cb929cc08
--- /dev/null
+++ b/auto_round/algorithms/transforms/svdquant/config.py
@@ -0,0 +1,115 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from auto_round.algorithms.quantization.config import QuantizationConfig
+
+
+class SVDQuantConfig(QuantizationConfig):
+ """Configuration for the SVDQuant structural preprocessor.
+
+ Args:
+ rank: Rank of the floating-point low-rank branch.
+ smooth_enabled: Whether to search activation-aware input smoothing factors.
+ smooth_num_grids: Grid resolution for Alpha/Beta smooth candidates.
+ smooth_max_calibration_calls: Maximum retained calls per smooth group.
+ target_modules: Module-name substrings eligible for decomposition.
+ exclude_modules: Module-name substrings excluded from decomposition.
+ low_rank_dtype: Deployment dtype for the low-rank factors.
+ smooth_eps: Positive floor used to construct finite smooth factors.
+ residual_iters: Number of alternating low-rank/residual fitting iterations.
+ residual_early_stop: Stop when the reconstruction objective stops improving.
+ model_adapter: Architecture adapter used for grouping and export.
+ **kwargs: Shared :class:`QuantizationConfig` options.
+ """
+
+ def __init__(
+ self,
+ *,
+ rank: int = 32,
+ smooth_enabled: bool = False,
+ smooth_num_grids: int = 20,
+ smooth_max_calibration_calls: int = 128,
+ target_modules: list[str] | tuple[str, ...] | str | None = None,
+ exclude_modules: list[str] | tuple[str, ...] | str | None = None,
+ low_rank_dtype: str = "bf16",
+ smooth_eps: float = 1e-6,
+ residual_iters: int = 1,
+ residual_early_stop: bool = False,
+ model_adapter: str | None = None,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ if type(smooth_enabled) is not bool:
+ raise ValueError(f"`smooth_enabled` must be a bool, got {smooth_enabled!r}")
+ if not isinstance(rank, int) or isinstance(rank, bool) or rank < 0:
+ raise ValueError(f"`rank` must be a non-negative integer, got {rank!r}")
+ if type(smooth_num_grids) is not int or smooth_num_grids < 2:
+ raise ValueError(
+ f"`smooth_num_grids` must be an integer greater than or equal to 2, got {smooth_num_grids!r}"
+ )
+ if type(smooth_max_calibration_calls) is not int or smooth_max_calibration_calls < 1:
+ raise ValueError(
+ f"`smooth_max_calibration_calls` must be a positive integer, got {smooth_max_calibration_calls!r}"
+ )
+ if smooth_eps <= 0:
+ raise ValueError(f"`smooth_eps` must be positive, got {smooth_eps!r}")
+ if not isinstance(low_rank_dtype, str) or low_rank_dtype.lower() not in {
+ "bf16",
+ "bfloat16",
+ "fp16",
+ "float16",
+ "fp32",
+ "float32",
+ }:
+ raise ValueError(
+ "`low_rank_dtype` must be one of bf16, bfloat16, fp16, float16, fp32, or float32, "
+ f"got {low_rank_dtype!r}"
+ )
+ if type(residual_iters) is not int or residual_iters < 1:
+ raise ValueError(f"`residual_iters` must be a positive integer, got {residual_iters!r}")
+ if type(residual_early_stop) is not bool:
+ raise ValueError(f"`residual_early_stop` must be a bool, got {residual_early_stop!r}")
+ self.rank = rank
+ self.smooth_enabled = smooth_enabled
+ self.smooth_num_grids = smooth_num_grids
+ self.smooth_max_calibration_calls = smooth_max_calibration_calls
+ self.target_modules = _normalize_patterns(target_modules)
+ self.exclude_modules = _normalize_patterns(exclude_modules)
+ self.low_rank_dtype = low_rank_dtype
+ self.smooth_eps = smooth_eps
+ self.residual_iters = residual_iters
+ self.residual_early_stop = residual_early_stop
+ self.model_adapter = model_adapter
+ self.need_calib = smooth_enabled
+
+ def __repr__(self) -> str:
+ return (
+ f"SVDQuantConfig(rank={self.rank}, smooth_enabled={self.smooth_enabled!r}, "
+ f"smooth_num_grids={self.smooth_num_grids}, "
+ f"smooth_max_calibration_calls={self.smooth_max_calibration_calls}, "
+ f"low_rank_dtype={self.low_rank_dtype!r}, "
+ f"target_modules={self.target_modules}, exclude_modules={self.exclude_modules}, "
+ f"residual_iters={self.residual_iters}, residual_early_stop={self.residual_early_stop!r}, "
+ f"model_adapter={self.model_adapter!r})"
+ )
+
+
+def _normalize_patterns(value):
+ if value is None:
+ return None
+ if isinstance(value, str):
+ return [item.strip() for item in value.split(",") if item.strip()]
+ return list(value)
diff --git a/auto_round/algorithms/transforms/svdquant/residual.py b/auto_round/algorithms/transforms/svdquant/residual.py
new file mode 100644
index 000000000..b9f265c2d
--- /dev/null
+++ b/auto_round/algorithms/transforms/svdquant/residual.py
@@ -0,0 +1,239 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Residual low-rank decomposition inspired by SVDQuant.
+
+Reference: Li et al., "SVDQuant: Absorbing Outliers by Low-Rank Components for
+4-Bit Diffusion Models", ICLR 2025, https://arxiv.org/abs/2411.05007.
+
+This module is an AutoRound implementation and does not copy source code from
+the reference implementation.
+"""
+
+import math
+from dataclasses import dataclass
+
+import torch
+
+from auto_round.data_type.utils import get_quant_func
+
+_FIXED_MXFP4_DTYPES = frozenset({"mx_fp4", "mx_fp4e2m1"})
+_MXFP4_ALIASES = frozenset({"mx_fp", *_FIXED_MXFP4_DTYPES})
+
+
+def _validate_scheme_values(scheme):
+ values = {}
+ for field in ("data_type", "bits", "group_size", "sym"):
+ try:
+ values[field] = getattr(scheme, field)
+ except AttributeError as exc:
+ raise ValueError(f"Residual quantization scheme is missing required value {field!r}.") from exc
+
+ if not isinstance(values["data_type"], str) or not values["data_type"].strip():
+ raise ValueError("Residual quantization scheme data_type must be a non-empty string.")
+ if not isinstance(values["bits"], int) or isinstance(values["bits"], bool) or values["bits"] <= 0:
+ raise ValueError("Residual quantization scheme bits must be a positive integer.")
+ if values["data_type"] in _FIXED_MXFP4_DTYPES and values["bits"] != 4:
+ raise ValueError(
+ f"Residual quantization scheme data_type={values['data_type']!r} requires bits=4; "
+ f"got bits={values['bits']}."
+ )
+
+ group_size = values["group_size"]
+ scalar_group_size = isinstance(group_size, int) and not isinstance(group_size, bool) and group_size >= -1
+ block_group_size = (
+ isinstance(group_size, tuple)
+ and len(group_size) == 2
+ and all(isinstance(size, int) and not isinstance(size, bool) and size > 0 for size in group_size)
+ )
+ if not scalar_group_size and not block_group_size:
+ raise ValueError(
+ "Residual quantization scheme group_size must be -1, 0, a positive integer, or a pair of positive integers."
+ )
+ if not isinstance(values["sym"], bool):
+ raise ValueError("Residual quantization scheme sym must be a boolean.")
+ return values
+
+
+@dataclass(frozen=True)
+class ResidualQuantScheme:
+ """Weight quantization settings for stateless residual QDQ."""
+
+ data_type: str | None = None
+ bits: int | None = None
+ group_size: int | tuple[int, int] | None = None
+ sym: bool | None = None
+
+ def __post_init__(self) -> None:
+ _validate_scheme_values(self)
+
+
+@dataclass(frozen=True)
+class ActivationQuantScheme:
+ """Activation quantization settings for stateless calibration QDQ."""
+
+ data_type: str | None = None
+ bits: int | None = None
+ group_size: int | tuple[int, int] | None = None
+ sym: bool | None = None
+
+ def __post_init__(self) -> None:
+ _validate_scheme_values(self)
+
+
+@dataclass(frozen=True)
+class ResidualDecomposition:
+ """Best deployment-materialized candidate from residual outer iteration."""
+
+ residual: torch.Tensor
+ down: torch.Tensor
+ up: torch.Tensor
+ selected_iteration: int
+ error: float
+
+
+def _rtn_qdq_tensor(tensor: torch.Tensor, scheme, *, tensor_name: str) -> torch.Tensor:
+ values = _validate_scheme_values(scheme)
+ requested_dtype = values["data_type"]
+ if values["bits"] == 4 and requested_dtype in _MXFP4_ALIASES:
+ requested_dtype = f"{requested_dtype}_rceil"
+ quant_func, resolved_dtype = get_quant_func(
+ dtype=requested_dtype,
+ bits=values["bits"],
+ sym=values["sym"],
+ disable_opt_rtn=True,
+ group_size=values["group_size"],
+ iters=0,
+ )
+ logical_dtype = resolved_dtype.removeprefix("rtn_")
+ resolved_base_dtype = logical_dtype.removesuffix("_rceil")
+ if (
+ resolved_base_dtype in _MXFP4_ALIASES
+ and values["bits"] == 4
+ and (
+ not isinstance(values["group_size"], int)
+ or isinstance(values["group_size"], bool)
+ or values["group_size"] != 32
+ )
+ ):
+ raise ValueError(
+ f"Deployable MXFP4 {tensor_name} QDQ requires scalar group_size=32; "
+ f"got group_size={values['group_size']!r}."
+ )
+
+ qdq, _, _ = quant_func(
+ tensor=tensor,
+ bits=values["bits"],
+ group_size=values["group_size"],
+ data_type=logical_dtype,
+ )
+ if qdq.shape != tensor.shape or qdq.dtype != tensor.dtype:
+ raise ValueError(
+ f"{tensor_name.capitalize()} RTN QDQ must preserve the input shape and dtype; "
+ f"got shape={tuple(qdq.shape)}, dtype={qdq.dtype}."
+ )
+ if qdq.device != tensor.device:
+ raise ValueError(
+ f"{tensor_name.capitalize()} RTN QDQ must preserve the input device; "
+ f"got input device={tensor.device}, output device={qdq.device}."
+ )
+ if not torch.isfinite(qdq).all():
+ raise ValueError(f"{tensor_name.capitalize()} RTN QDQ produced non-finite values.")
+ return qdq
+
+
+def rtn_qdq_residual(weight: torch.Tensor, scheme: ResidualQuantScheme) -> torch.Tensor:
+ """Apply the registered RTN quantize-dequantize function to a residual."""
+ return _rtn_qdq_tensor(weight, scheme, tensor_name="residual")
+
+
+def rtn_qdq_activation(activation: torch.Tensor, scheme: ActivationQuantScheme) -> torch.Tensor:
+ """Apply deployment-compatible dynamic activation quantize-dequantize."""
+ return _rtn_qdq_tensor(activation, scheme, tensor_name="activation")
+
+
+def truncated_svd(weight: torch.Tensor, rank: int) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Return a rank-limited reconstruction and its shared down/up factors."""
+ if weight.ndim != 2:
+ raise ValueError(f"SVDQuant expects a two-dimensional weight matrix, got shape={tuple(weight.shape)}.")
+ max_rank = min(weight.shape)
+ if not isinstance(rank, int) or isinstance(rank, bool) or not 0 <= rank <= max_rank:
+ raise ValueError(f"SVDQuant rank must be an integer in [0, {max_rank}], got {rank!r}.")
+
+ out_features, in_features = weight.shape
+ if rank == 0:
+ low_rank = torch.zeros_like(weight)
+ down_weight = torch.empty((0, in_features), dtype=weight.dtype, device=weight.device)
+ up_weight = torch.empty((out_features, 0), dtype=weight.dtype, device=weight.device)
+ return low_rank, down_weight, up_weight
+
+ u, s, vh = torch.linalg.svd(weight, full_matrices=False)
+ down_weight = vh[:rank, :]
+ up_weight = u[:, :rank] * s[:rank].reshape(1, -1)
+ return up_weight @ down_weight, down_weight, up_weight
+
+
+def iterate_residual_decomposition(
+ weight: torch.Tensor,
+ *,
+ rank: int,
+ scheme: ResidualQuantScheme,
+ iterations: int,
+ early_stop: bool,
+ residual_dtype: torch.dtype,
+ low_rank_dtype: torch.dtype,
+) -> ResidualDecomposition:
+ """Select the lowest weight-MSE residual/low-rank candidate after deployment casting."""
+ if type(iterations) is not int or iterations < 1:
+ raise ValueError(f"SVDQuant residual iterations must be a positive integer, got {iterations!r}.")
+
+ quantized_residual = torch.zeros_like(weight)
+ best_down = None
+ best_up = None
+ best_error = float("inf")
+ best_iteration = None
+
+ for iteration in range(1, iterations + 1):
+ low_rank, down, up = truncated_svd(weight - quantized_residual, rank)
+ if not all(torch.isfinite(tensor).all() for tensor in (low_rank, down, up)):
+ break
+
+ residual = (weight - low_rank).to(residual_dtype)
+ quantized_residual = rtn_qdq_residual(residual, scheme).to(weight.dtype)
+ deployed_down = down.to(low_rank_dtype)
+ deployed_up = up.to(low_rank_dtype)
+ deployed_low_rank = deployed_up.to(weight.dtype) @ deployed_down.to(weight.dtype)
+ error = torch.sum((weight - (quantized_residual + deployed_low_rank)).square()).item()
+ accepted = math.isfinite(error) and error <= best_error
+
+ if accepted:
+ best_down = deployed_down.clone()
+ best_up = deployed_up.clone()
+ best_error = error
+ best_iteration = iteration
+ elif early_stop and best_iteration is not None:
+ break
+
+ if best_down is None or best_up is None or best_iteration is None:
+ raise ValueError("SVDQuant residual iteration did not produce a finite candidate.")
+
+ deployed_low_rank = best_up.to(weight.dtype) @ best_down.to(weight.dtype)
+ residual = weight - deployed_low_rank
+ if not torch.isfinite(residual).all():
+ raise ValueError("SVDQuant residual iteration produced a non-finite residual.")
+ return ResidualDecomposition(residual, best_down, best_up, best_iteration, best_error)
diff --git a/auto_round/algorithms/transforms/svdquant/smooth.py b/auto_round/algorithms/transforms/svdquant/smooth.py
new file mode 100644
index 000000000..41d6b4c0c
--- /dev/null
+++ b/auto_round/algorithms/transforms/svdquant/smooth.py
@@ -0,0 +1,156 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import Iterable, TypeVar
+
+import torch
+
+
+@dataclass(frozen=True)
+class SmoothCandidate:
+ alpha: float
+ beta: float
+ scale: torch.Tensor
+
+
+@dataclass(frozen=True)
+class SmoothScaleStats:
+ minimum: float
+ maximum: float
+ ratio: float
+ below_min_count: int
+ above_max_count: int
+
+
+def build_alpha_beta_candidates(num_grids: int) -> list[tuple[float, float]]:
+ """Build identity, activation-only, and activation/weight balanced candidates."""
+ if type(num_grids) is not int or num_grids < 2:
+ raise ValueError(f"`num_grids` must be an integer greater than or equal to 2, got {num_grids!r}")
+ choices = [index / num_grids for index in range(1, num_grids)]
+ return [(0.0, 0.0), *[(alpha, 0.0) for alpha in choices], *[(alpha, 1.0 - alpha) for alpha in choices]]
+
+
+def absmax_channel_span(tensor: torch.Tensor, channels_dim: int) -> torch.Tensor:
+ """Return per-channel absolute maxima over every non-channel dimension."""
+ if tensor.ndim == 0:
+ raise ValueError("Cannot calculate a channel span for a scalar tensor.")
+ channels_dim %= tensor.ndim
+ moved = tensor.detach().movedim(channels_dim, -1)
+ return moved.abs().reshape(-1, moved.shape[-1]).amax(dim=0).to(torch.float32)
+
+
+def build_smooth_scale(
+ x_span: torch.Tensor,
+ w_span: torch.Tensor,
+ alpha: float,
+ beta: float,
+ eps: float | None = None,
+) -> torch.Tensor:
+ """Construct the factor ``x_span**alpha / w_span**beta`` with safe fallbacks."""
+ if not 0.0 <= alpha <= 1.0 or not 0.0 <= beta <= 1.0:
+ raise ValueError(f"Smooth alpha and beta must be in [0, 1], got alpha={alpha!r}, beta={beta!r}")
+ if x_span.shape != w_span.shape:
+ raise ValueError(f"Smooth spans must have matching shapes, got {x_span.shape} and {w_span.shape}")
+
+ x_span = x_span.to(torch.float32)
+ w_span = w_span.to(device=x_span.device, dtype=torch.float32)
+ x_zero = x_span == 0
+ w_zero = w_span == 0
+ if eps is not None:
+ if eps <= 0:
+ raise ValueError(f"`eps` must be positive, got {eps!r}")
+ x_span = torch.where(x_zero, eps, x_span)
+ w_span = torch.where(w_zero, eps, w_span)
+ if alpha == 0.0 and beta == 0.0:
+ return torch.ones_like(x_span)
+
+ if alpha > 0.0:
+ scale = x_span.pow(alpha)
+ if beta > 0.0:
+ scale = scale / w_span.pow(beta)
+ else:
+ scale = w_span.pow(-beta)
+
+ scale = scale.clone()
+ if beta > 0.0 and bool(w_zero.any()):
+ scale.fill_(1)
+ elif alpha > 0.0:
+ scale[x_zero] = 1
+ scale[scale == 0] = 1
+ if not torch.isfinite(scale).all():
+ scale.fill_(1)
+ return scale
+
+
+def validate_smooth_scale_for_deployment(
+ scale: torch.Tensor,
+ *,
+ dtype: torch.dtype,
+ module_name: str,
+) -> torch.Tensor:
+ """Validate a smooth scale after materialization in its deployment dtype."""
+ deployed = scale.to(dtype=dtype)
+ reciprocal = deployed.reciprocal()
+ if (
+ not bool(torch.isfinite(deployed).all())
+ or not bool((deployed > 0).all())
+ or not bool(torch.isfinite(reciprocal).all())
+ or not bool((reciprocal > 0).all())
+ ):
+ raise ValueError(f"SVDQuant smooth scale is not deployable for {module_name!r} in dtype {dtype}.")
+ return deployed
+
+
+def summarize_smooth_scale(
+ scale: torch.Tensor,
+ *,
+ low_threshold: float = 1e-3,
+ high_threshold: float = 20.0,
+) -> SmoothScaleStats:
+ """Summarize factor range and deployment-risk threshold counts."""
+ values = scale.detach().to(device="cpu", dtype=torch.float32)
+ minimum = values.amin().item()
+ maximum = values.amax().item()
+ return SmoothScaleStats(
+ minimum=minimum,
+ maximum=maximum,
+ ratio=maximum / minimum,
+ below_min_count=int((values < low_threshold).sum().item()),
+ above_max_count=int((values > high_threshold).sum().item()),
+ )
+
+
+_CandidateT = TypeVar("_CandidateT")
+
+
+def select_best_layer_candidate(
+ candidates: Iterable[tuple[_CandidateT, float | torch.Tensor]],
+ *,
+ module_name: str,
+) -> _CandidateT:
+ """Select the lowest finite-error candidate, preferring the later exact tie."""
+ best_candidate = None
+ best_error = float("inf")
+ for candidate, error in candidates:
+ error_value = error.item() if torch.is_tensor(error) else float(error)
+ if math.isfinite(error_value) and error_value <= best_error:
+ best_candidate = candidate
+ best_error = error_value
+ if best_candidate is None:
+ raise ValueError(f"SVDQuant smooth search produced no finite candidate for {module_name!r}.")
+ return best_candidate
diff --git a/auto_round/algorithms/transforms/svdquant/smooth_adapters/__init__.py b/auto_round/algorithms/transforms/svdquant/smooth_adapters/__init__.py
new file mode 100644
index 000000000..c8613c305
--- /dev/null
+++ b/auto_round/algorithms/transforms/svdquant/smooth_adapters/__init__.py
@@ -0,0 +1,36 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import torch
+
+from auto_round.algorithms.transforms.svdquant.smooth_adapters.base import (
+ SmoothSearchGroup,
+ TargetPredicate,
+ generic_linear_groups,
+)
+from auto_round.algorithms.transforms.svdquant.smooth_adapters.flux import discover_flux_groups, supports_flux_block
+
+
+def discover_svdquant_groups(block: torch.nn.Module, is_target: TargetPredicate) -> list[SmoothSearchGroup]:
+ """Discover shared-input projection groups for one quantization block."""
+ if supports_flux_block(block):
+ return discover_flux_groups(block, is_target)
+ return generic_linear_groups(block, is_target)
+
+
+discover_smooth_search_groups = discover_svdquant_groups
+
+__all__ = ["SmoothSearchGroup", "discover_smooth_search_groups", "discover_svdquant_groups"]
diff --git a/auto_round/algorithms/transforms/svdquant/smooth_adapters/base.py b/auto_round/algorithms/transforms/svdquant/smooth_adapters/base.py
new file mode 100644
index 000000000..9380fda9e
--- /dev/null
+++ b/auto_round/algorithms/transforms/svdquant/smooth_adapters/base.py
@@ -0,0 +1,134 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import inspect
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+import torch
+
+TargetPredicate = Callable[[str, torch.nn.Module], bool]
+
+
+def normalize_tensors(output: Any, indices: tuple[int, ...] | None = None) -> tuple[torch.Tensor, ...]:
+ """Return floating-point tensors used by the smooth output-error objective."""
+ if indices is not None:
+ if not isinstance(output, (tuple, list)):
+ raise TypeError("Indexed SVDQuant smooth output must be a tuple or list.")
+ output = tuple(output[index] for index in indices)
+
+ tensors = []
+
+ def collect(value):
+ if torch.is_tensor(value):
+ if value.is_floating_point():
+ tensors.append(value)
+ elif isinstance(value, Mapping):
+ for item in value.values():
+ collect(item)
+ elif isinstance(value, (tuple, list)):
+ for item in value:
+ collect(item)
+
+ collect(output)
+ if not tensors:
+ raise ValueError("SVDQuant smooth evaluation produced no floating-point tensors.")
+ return tuple(tensors)
+
+
+def filter_supported_kwargs(module: torch.nn.Module, kwargs: Mapping[str, Any]) -> dict[str, Any]:
+ parameters = inspect.signature(module.forward).parameters
+ if any(parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters.values()):
+ return dict(kwargs)
+ return {name: value for name, value in kwargs.items() if name in parameters}
+
+
+@dataclass(frozen=True)
+class SmoothSearchGroup:
+ """Projection group that shares an input scale and low-rank down factor."""
+
+ key: str
+ projection_names: tuple[str, ...]
+ projections: tuple[torch.nn.Linear, ...]
+ projection_input_key: str
+ projection_input_module: torch.nn.Module
+ evaluation_input_key: str
+ evaluation_module: torch.nn.Module
+ output_indices: tuple[int, ...] | None = None
+
+ def __post_init__(self) -> None:
+ if not self.projections:
+ raise ValueError(f"SVDQuant group {self.key!r} has no projections.")
+ if len(self.projection_names) != len(self.projections):
+ raise ValueError(f"SVDQuant group {self.key!r} has mismatched names and projections.")
+ if len({projection.in_features for projection in self.projections}) != 1:
+ raise ValueError(f"SVDQuant group {self.key!r} projections must share an input width.")
+
+ def filter_evaluation_kwargs(self, kwargs: Mapping[str, Any]) -> dict[str, Any]:
+ return filter_supported_kwargs(self.evaluation_module, kwargs)
+
+ def normalize_output(self, output: Any) -> tuple[torch.Tensor, ...]:
+ return normalize_tensors(output, self.output_indices)
+
+
+def module_global_name(block: torch.nn.Module, local_name: str) -> str:
+ prefix = str(getattr(block, "global_name", block.__class__.__name__))
+ return f"{prefix}.{local_name}" if local_name else prefix
+
+
+def resolve_module(root: torch.nn.Module, path: str) -> torch.nn.Module | None:
+ module = root
+ for part in path.split("."):
+ if part.isdigit() and isinstance(module, (torch.nn.ModuleList, torch.nn.Sequential)):
+ index = int(part)
+ if index >= len(module):
+ return None
+ module = module[index]
+ else:
+ candidate = getattr(module, part, None)
+ if not isinstance(candidate, torch.nn.Module):
+ return None
+ module = candidate
+ return module
+
+
+def generic_linear_groups(
+ block: torch.nn.Module,
+ is_target: TargetPredicate,
+ *,
+ consumed: set[int] | None = None,
+) -> list[SmoothSearchGroup]:
+ consumed = consumed or set()
+ groups = []
+ for local_name, module in block.named_modules():
+ if not local_name or id(module) in consumed or not is_target(local_name, module):
+ continue
+ if not isinstance(module, torch.nn.Linear):
+ continue
+ name = module_global_name(block, local_name)
+ groups.append(
+ SmoothSearchGroup(
+ key=name,
+ projection_names=(name,),
+ projections=(module,),
+ projection_input_key=name,
+ projection_input_module=module,
+ evaluation_input_key=name,
+ evaluation_module=module,
+ )
+ )
+ return groups
diff --git a/auto_round/algorithms/transforms/svdquant/smooth_adapters/flux.py b/auto_round/algorithms/transforms/svdquant/smooth_adapters/flux.py
new file mode 100644
index 000000000..dfa8cf891
--- /dev/null
+++ b/auto_round/algorithms/transforms/svdquant/smooth_adapters/flux.py
@@ -0,0 +1,124 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import torch
+
+from auto_round.algorithms.transforms.svdquant.smooth_adapters.base import (
+ SmoothSearchGroup,
+ TargetPredicate,
+ generic_linear_groups,
+ module_global_name,
+ resolve_module,
+)
+from auto_round.logger import logger
+
+_VERIFIED_FLUX_MODEL_IDS = ("black-forest-labs/flux.1-dev", "flux.1-dev")
+
+
+@dataclass(frozen=True)
+class FluxSmoothGroupSpec:
+ """Declarative FLUX projection grouping and output-error evaluation route."""
+
+ key: str
+ projection_paths: tuple[str, ...]
+ evaluation_path: str
+ output_indices: tuple[int, ...] | None
+
+
+_DOUBLE_GROUPS = (
+ FluxSmoothGroupSpec("attn.qkv", ("attn.to_q", "attn.to_k", "attn.to_v"), "attn", (0,)),
+ FluxSmoothGroupSpec("attn.add_qkv", ("attn.add_q_proj", "attn.add_k_proj", "attn.add_v_proj"), "attn", (1,)),
+ FluxSmoothGroupSpec("attn.to_out.0", ("attn.to_out.0",), "attn.to_out.0", None),
+ FluxSmoothGroupSpec("attn.to_add_out", ("attn.to_add_out",), "attn.to_add_out", None),
+ FluxSmoothGroupSpec("ff.net.0.proj", ("ff.net.0.proj",), "ff.net.0.proj", None),
+ FluxSmoothGroupSpec("ff.net.2", ("ff.net.2",), "ff.net.2", None),
+ FluxSmoothGroupSpec("ff_context.net.0.proj", ("ff_context.net.0.proj",), "ff_context.net.0.proj", None),
+ FluxSmoothGroupSpec("ff_context.net.2", ("ff_context.net.2",), "ff_context.net.2", None),
+)
+
+_SINGLE_GROUPS = (
+ FluxSmoothGroupSpec("parallel_qkv_mlp", ("attn.to_q", "attn.to_k", "attn.to_v", "proj_mlp"), "", None),
+ FluxSmoothGroupSpec("proj_out", ("proj_out",), "proj_out", None),
+)
+
+
+def supports_flux_block(block: torch.nn.Module) -> bool:
+ return block.__class__.__name__ in {"FluxTransformerBlock", "FluxSingleTransformerBlock"}
+
+
+def warn_if_unverified_flux_model(model: torch.nn.Module) -> None:
+ """Warn when FLUX grouping is used outside the model validated by this adapter."""
+ config = getattr(model, "config", None)
+ if hasattr(config, "get"):
+ model_id = config.get("_name_or_path", "")
+ else:
+ model_id = getattr(config, "_name_or_path", "")
+ normalized = str(model_id).lower()
+ if not any(verified_id in normalized for verified_id in _VERIFIED_FLUX_MODEL_IDS):
+ logger.warning_once(
+ "SVDQuant FLUX grouping has been validated with FLUX.1-dev only; "
+ "model %r is unverified and may require a dedicated adapter.",
+ model_id or type(model).__name__,
+ )
+
+
+def _make_group(
+ block: torch.nn.Module,
+ specification: FluxSmoothGroupSpec,
+ is_target: TargetPredicate,
+) -> SmoothSearchGroup | None:
+ selected = []
+ for path in specification.projection_paths:
+ module = resolve_module(block, path)
+ if isinstance(module, torch.nn.Linear) and is_target(path, module):
+ selected.append((path, module))
+ if not selected:
+ return None
+
+ evaluation_module = (
+ block if not specification.evaluation_path else resolve_module(block, specification.evaluation_path)
+ )
+ if evaluation_module is None:
+ raise ValueError(
+ f"Flux SVDQuant group {module_global_name(block, specification.key)!r} has no evaluation module."
+ )
+ names = tuple(module_global_name(block, path) for path, _ in selected)
+ projections = tuple(module for _, module in selected)
+ return SmoothSearchGroup(
+ key=module_global_name(block, specification.key),
+ projection_names=names,
+ projections=projections,
+ projection_input_key=names[0],
+ projection_input_module=projections[0],
+ evaluation_input_key=module_global_name(block, specification.evaluation_path),
+ evaluation_module=evaluation_module,
+ output_indices=specification.output_indices,
+ )
+
+
+def discover_flux_groups(block: torch.nn.Module, is_target: TargetPredicate) -> list[SmoothSearchGroup]:
+ specifications = _SINGLE_GROUPS if block.__class__.__name__ == "FluxSingleTransformerBlock" else _DOUBLE_GROUPS
+
+ groups = []
+ for specification in specifications:
+ group = _make_group(block, specification, is_target)
+ if group is not None:
+ groups.append(group)
+ consumed = {id(projection) for group in groups for projection in group.projections}
+ groups.extend(generic_linear_groups(block, is_target, consumed=consumed))
+ return groups
diff --git a/auto_round/algorithms/transforms/svdquant/wrapper.py b/auto_round/algorithms/transforms/svdquant/wrapper.py
new file mode 100644
index 000000000..7dda281ba
--- /dev/null
+++ b/auto_round/algorithms/transforms/svdquant/wrapper.py
@@ -0,0 +1,42 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections.abc import Callable
+
+import torch
+
+
+class SVDQuantLinear(torch.nn.Module):
+ """Linear decomposed into a quantizable residual branch and an FP low-rank branch."""
+
+ def __init__(
+ self,
+ residual_linear: torch.nn.Linear,
+ lora_down: torch.nn.Linear,
+ lora_up: torch.nn.Linear,
+ smooth: torch.Tensor,
+ activation_qdq: Callable[[torch.Tensor], torch.Tensor] | None = None,
+ ):
+ super().__init__()
+ self.residual_linear = residual_linear
+ self.lora_down = lora_down
+ self.lora_up = lora_up
+ self.activation_qdq = activation_qdq
+ self.register_buffer("smooth", smooth.detach().clone())
+
+ def forward(self, x):
+ smooth = self.smooth.to(device=x.device, dtype=x.dtype)
+ smoothed = x * smooth
+ residual_input = smoothed if self.activation_qdq is None else self.activation_qdq(smoothed)
+ return self.residual_linear(residual_input) + self.lora_up(self.lora_down(smoothed))
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 4f2bee857..f3dfe570f 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -634,6 +634,9 @@ class _CompressorBuilder(object):
alg_configs = normalize_algorithm_config(alg_configs)
configs_for_routing = alg_configs if isinstance(alg_configs, list) else [alg_configs]
preprocessor_configs, _, quant_config = _resolve_quant_config_for_routing(configs_for_routing)
+ is_svdquant = any(type(config).__name__ == "SVDQuantConfig" for config in preprocessor_configs)
+ if is_svdquant:
+ format = "svdquant_nunchaku"
# Model-free routing is now supported directly by the new entry path.
model_free_iters = 0 if isinstance(quant_config, RTNConfig) else getattr(quant_config, "iters", None)
@@ -643,6 +646,10 @@ class _CompressorBuilder(object):
# route predicate; otherwise the fast path silently drops them and
# cannot emit the required export metadata.
route_decision_kwargs = dict(base_kwargs, **route_kwargs, format=format)
+ is_svdquant_rtn = type(quant_config) is RTNConfig and is_svdquant
+ if is_svdquant_rtn and not route_kwargs.get("model_free", False):
+ # SVDQuant must run before plain RTN on the regular blockwise path.
+ route_decision_kwargs["disable_model_free"] = True
route_scheme = (
scheme
if hasattr(scheme, "options") and hasattr(scheme, "avg_bits")
diff --git a/auto_round/calibration/diffusion.py b/auto_round/calibration/diffusion.py
index 688be5936..7a37d7c33 100644
--- a/auto_round/calibration/diffusion.py
+++ b/auto_round/calibration/diffusion.py
@@ -35,6 +35,19 @@ from auto_round.utils.device_manager import device_manager
from auto_round.utils.model import wrap_block_forward_positional_to_kwargs
+def _prepare_pipeline_for_calibration(pipe, target_device, *, low_gpu_mem_usage: bool) -> str | None:
+ """Place a diffusion pipeline for calibration without exceeding one GPU."""
+ resolved_device = torch.device(target_device)
+ if low_gpu_mem_usage:
+ enable_model_cpu_offload = getattr(pipe, "enable_model_cpu_offload", None)
+ if not callable(enable_model_cpu_offload):
+ raise ValueError("The diffusion pipeline does not support component-level model CPU offload.")
+ enable_model_cpu_offload(device=resolved_device)
+ return "model"
+ pipe.to(resolved_device)
+ return None
+
+
@register_calibrator("diffusion")
class DiffusionCalibrator(LLMCalibrator):
"""Calibrator for diffusion models (Stable Diffusion / FLUX / ...)."""
@@ -120,7 +133,11 @@ class DiffusionCalibrator(LLMCalibrator):
exit(-1)
target_device = device_manager.device
- pipe.to(torch.device(target_device))
+ self._cpu_offload_mode = _prepare_pipeline_for_calibration(
+ pipe,
+ target_device,
+ low_gpu_mem_usage=self.low_gpu_mem_usage,
+ )
pipeline_fn = getattr(pipe, "_autoround_pipeline_fn", None)
# Check if this is an I2V pipeline (needs calibration image)
requires_image = False
diff --git a/auto_round/cli/algorithms.py b/auto_round/cli/algorithms.py
index 4a38c6e6d..fb07b5263 100644
--- a/auto_round/cli/algorithms.py
+++ b/auto_round/cli/algorithms.py
@@ -253,6 +253,89 @@ class AWQ(AlgorithmHandler):
)
+class SVDQuant(AlgorithmHandler):
+ name = "svdquant"
+ aliases = ("svdquant",)
+ summary = "SVD low-rank decomposition before residual quantization."
+ config_factory = None
+
+ def register(self, group) -> None:
+ group.add_argument("--svdquant-rank", default=32, type=int, help="SVDQuant low-rank size.")
+ group.add_argument(
+ "--enable-svdquant-smooth",
+ dest="svdquant_smooth_enabled",
+ default=False,
+ action="store_true",
+ help="Enable SVDQuant activation-aware smoothing.",
+ )
+ group.add_argument(
+ "--svdquant-smooth-num-grids",
+ default=20,
+ type=int,
+ help="Number of candidates per SVDQuant smooth search grid family.",
+ )
+ group.add_argument(
+ "--svdquant-smooth-max-calibration-calls",
+ default=128,
+ type=int,
+ help="Maximum calibration calls retained per SVDQuant smooth group.",
+ )
+ group.add_argument(
+ "--svdquant-residual-iters",
+ default=1,
+ type=int,
+ help="Number of alternating low-rank and residual quantization iterations.",
+ )
+ group.add_argument(
+ "--enable-svdquant-residual-early-stop",
+ dest="svdquant_residual_early_stop",
+ default=False,
+ action="store_true",
+ help="Stop residual iteration when reconstruction error no longer improves.",
+ )
+ group.add_argument(
+ "--svdquant-low-rank-dtype",
+ default="bf16",
+ choices=["bf16", "bfloat16", "fp16", "float16", "fp32", "float32"],
+ help="Data type for the SVDQuant low-rank branch.",
+ )
+ group.add_argument(
+ "--svdquant-target-modules",
+ default=None,
+ type=str,
+ help="Comma-separated module-name substrings to transform.",
+ )
+ group.add_argument(
+ "--svdquant-exclude-modules",
+ default=None,
+ type=str,
+ help="Comma-separated module-name substrings to exclude.",
+ )
+ group.add_argument(
+ "--svdquant-model-adapter",
+ default="auto",
+ choices=["auto", "identity", "flux"],
+ help="Architecture adapter used by SVDQuant export.",
+ )
+
+ def build(self, args, common_kwargs: dict[str, Any]):
+ from auto_round.algorithms.transforms.svdquant.config import SVDQuantConfig
+
+ return SVDQuantConfig(
+ rank=getattr(args, "svdquant_rank", 32),
+ smooth_enabled=getattr(args, "svdquant_smooth_enabled", False),
+ smooth_num_grids=getattr(args, "svdquant_smooth_num_grids", 20),
+ smooth_max_calibration_calls=getattr(args, "svdquant_smooth_max_calibration_calls", 128),
+ residual_iters=getattr(args, "svdquant_residual_iters", 1),
+ residual_early_stop=getattr(args, "svdquant_residual_early_stop", False),
+ low_rank_dtype=getattr(args, "svdquant_low_rank_dtype", "bf16"),
+ target_modules=getattr(args, "svdquant_target_modules", None),
+ exclude_modules=getattr(args, "svdquant_exclude_modules", None),
+ model_adapter=getattr(args, "svdquant_model_adapter", "auto"),
+ **common_kwargs,
+ )
+
+
class RTN(AlgorithmHandler):
name = "rtn"
aliases = ("rtn",)
@@ -445,6 +528,7 @@ def _register_builtin_algorithm_factories() -> None:
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
from auto_round.algorithms.transforms.awq.config import AWQConfig
from auto_round.algorithms.transforms.hadamard.config import RotationConfig
+ from auto_round.algorithms.transforms.svdquant.config import SVDQuantConfig
register_algorithm("rtn", aliases=("rtn",), config_factory=RTNConfig, cli_handler=RTN, summary=RTN.summary)
register_algorithm(
@@ -455,6 +539,13 @@ def _register_builtin_algorithm_factories() -> None:
summary=AutoRound.summary,
)
register_algorithm("awq", aliases=("awq",), config_factory=AWQConfig, cli_handler=AWQ, summary=AWQ.summary)
+ register_algorithm(
+ "svdquant",
+ aliases=("svdquant",),
+ config_factory=SVDQuantConfig,
+ cli_handler=SVDQuant,
+ summary=SVDQuant.summary,
+ )
register_algorithm(
"hadamard",
aliases=("hadamard", "random_hadamard", "quarot_hadamard"),
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 1943c0e5c..9ef935928 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -1542,10 +1542,20 @@ class BaseOrchestrator(object):
return
formats = getattr(self, "formats", [])
+ if any(not format.is_supported_immediate_packing() for format in formats):
+ self.compress_context.is_immediate_packing = False
+ if any(not format.is_supported_immediate_saving() for format in formats):
+ self.compress_context.is_immediate_saving = False
+
has_single_gguf_format = len(formats) == 1 and formats[0].is_gguf()
# GGUF supports per-block / per-layer immediate packing even when
# full-model in-place rewriting is disabled by outside-block layers.
- if len(formats) == 1 and not formats[0].is_fake() and (self.inplace or has_single_gguf_format):
+ if (
+ len(formats) == 1
+ and not formats[0].is_fake()
+ and formats[0].is_supported_immediate_packing()
+ and (self.inplace or has_single_gguf_format)
+ ):
self.compress_context.is_immediate_packing = True
if self.has_qlayer_outside_block and self.need_calib and not has_single_gguf_format:
diff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.py
index 7b22117e7..a42988c15 100644
--- a/auto_round/compressors/diffusion_mixin.py
+++ b/auto_round/compressors/diffusion_mixin.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import inspect
+import json
import os
from typing import Any, Optional, Union
@@ -373,6 +374,10 @@ class DiffusionMixin:
layer_names=[],
)
self.inputs = all_inputs
+ if getattr(self.calibration, "_cpu_offload_mode", None) == "model":
+ from accelerate.hooks import remove_hook_from_submodules
+
+ remove_hook_from_submodules(self.model_context.model)
clear_memory()
self._inputs_cached = True
return super().quantize()
@@ -486,7 +491,7 @@ class DiffusionMixin:
def save_quantized(
self,
output_dir: Optional[str] = None,
- format: Union[str, list] = "auto_round",
+ format: Optional[Union[str, list]] = None,
inplace: bool = True,
return_folders: bool = False,
**kwargs,
@@ -517,7 +522,7 @@ class DiffusionMixin:
has_multiple_quantized_transformers = bool(quantized_transformers)
# Handle multi-format (convert string to list if needed)
- _format = format
+ _format = format if format is not None else getattr(self, "formats", None) or "auto_round"
if isinstance(_format, str):
_format = self._resolve_format_string(_format)
@@ -587,8 +592,6 @@ class DiffusionMixin:
pipe.config.save_pretrained(output_dir)
else:
# FrozenDict / plain dict — write model_index.json manually
- import json
-
model_index_path = os.path.join(output_dir, "model_index.json")
with open(model_index_path, "w", encoding="utf-8") as f:
f.write(json.dumps(dict(pipe.config), indent=2, sort_keys=True) + "\n")
diff --git a/auto_round/export/formats/backends/__init__.py b/auto_round/export/formats/backends/__init__.py
index 263ecbf1d..3f77b50cb 100644
--- a/auto_round/export/formats/backends/__init__.py
+++ b/auto_round/export/formats/backends/__init__.py
@@ -20,6 +20,7 @@ from auto_round.export.formats.backends.auto_awq import AutoAWQFormat
from auto_round.export.formats.backends.auto_gptq import AutoGPTQFormat
from auto_round.export.formats.backends.llm_compressor import LLMCompressorFormat
from auto_round.export.formats.backends.mlx import MLXFormat
+from auto_round.export.formats.backends.svdquant_nunchaku import SVDQuantNunchakuFormat
__all__ = [
"AutoAWQFormat",
@@ -30,4 +31,5 @@ __all__ = [
"GGUFFormat",
"LLMCompressorFormat",
"MLXFormat",
+ "SVDQuantNunchakuFormat",
]
diff --git a/auto_round/export/formats/backends/svdquant_nunchaku.py b/auto_round/export/formats/backends/svdquant_nunchaku.py
new file mode 100644
index 000000000..2b58f9abf
--- /dev/null
+++ b/auto_round/export/formats/backends/svdquant_nunchaku.py
@@ -0,0 +1,185 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import os
+from typing import Any, Callable, Union
+
+import torch
+
+from auto_round.export.formats.base import OutputFormat
+from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme
+
+
+def _invoke_save_hook(hook: Callable[[str], Any], output_dir: str) -> None:
+ hook(output_dir)
+
+
[email protected]("svdquant_nunchaku")
+class SVDQuantNunchakuFormat(OutputFormat):
+ support_schemes = ["MXFP4"]
+ format_name = "svdquant_nunchaku"
+ _e2m1_aliases = frozenset({"mx_fp", "mx_fp4", "mx_fp4e2m1"})
+
+ def __init__(self, format: str, scheme: QuantizationScheme, ctx: Any):
+ self.output_format = format
+ self.backend = None
+ self.mllm = ctx.mllm
+ self.check_scheme_args(scheme)
+ self._resolved_scheme = scheme.copy()
+
+ def is_supported_immediate_packing(self) -> bool:
+ return False
+
+ def is_supported_immediate_saving(self) -> bool:
+ return False
+
+ @classmethod
+ def check_scheme_args(cls, scheme: QuantizationScheme) -> bool:
+ rules = {
+ "data_type": cls._e2m1_aliases,
+ "bits": 4,
+ "group_size": 32,
+ "sym": True,
+ "act_data_type": cls._e2m1_aliases,
+ "act_bits": 4,
+ "act_group_size": 32,
+ "act_sym": True,
+ "act_dynamic": True,
+ }
+ for name, expected in rules.items():
+ actual = getattr(scheme, name, None)
+ if isinstance(expected, frozenset):
+ valid = isinstance(actual, str) and actual in expected
+ expected_text = f"one of {sorted(expected)}"
+ elif isinstance(expected, int) and not isinstance(expected, bool):
+ valid = isinstance(actual, int) and not isinstance(actual, bool) and actual == expected
+ expected_text = repr(expected)
+ else:
+ valid = actual is expected
+ expected_text = repr(expected)
+ if not valid:
+ raise ValueError(
+ f"{cls.format_name} got {name}={actual!r}; expected {name}={expected_text} "
+ "for Nunchaku E2M1 group32 export."
+ )
+ return True
+
+ def _validate_svd_layer_overrides(self, model: torch.nn.Module, layer_config: dict | None) -> None:
+ if model is None or not layer_config:
+ return
+ from auto_round.algorithms.transforms.svdquant.wrapper import SVDQuantLinear
+
+ for name, module in model.named_modules():
+ if not isinstance(module, SVDQuantLinear):
+ continue
+ for layer_name in (name, f"{name}.residual_linear"):
+ if layer_name not in layer_config:
+ continue
+ override = layer_config[layer_name]
+ if isinstance(override, str):
+ scheme = PRESET_SCHEMES[override.upper()].copy()
+ elif isinstance(override, QuantizationScheme):
+ scheme = override.copy()
+ elif isinstance(override, dict):
+ values = dict(override)
+ preset_name = values.pop("scheme", None)
+ scheme = (
+ PRESET_SCHEMES[preset_name.upper()].copy()
+ if preset_name is not None
+ else self._resolved_scheme.copy()
+ )
+ scheme.update_from_dict(values)
+ else:
+ raise TypeError(
+ f"Unsupported layer_config value for SVDQuant residual {layer_name!r}: {type(override)}"
+ )
+ try:
+ self.check_scheme_args(scheme)
+ except ValueError as exc:
+ raise ValueError(
+ f"{self.format_name} layer {layer_name!r} has an incompatible residual scheme: {exc}"
+ ) from exc
+
+ def check_and_reset_format(self, scheme: QuantizationScheme, ctx: Any):
+ self._validate_svd_layer_overrides(ctx.model, ctx.layer_config)
+ return None, scheme, ctx.layer_config, ctx.quant_block_list
+
+ def pack_layer(self, *args, **kwargs):
+ return None
+
+ def save_quantized(
+ self,
+ output_dir: str,
+ model: torch.nn.Module = None,
+ tokenizer: Callable = None,
+ layer_config: dict = None,
+ inplace: bool = True,
+ device: Union[str, torch.device] = "cpu",
+ serialization_dict: dict = None,
+ *,
+ config=None,
+ residual_provider=None,
+ adapter=None,
+ model_adapter=None,
+ **kwargs,
+ ) -> torch.nn.Module:
+ if output_dir is None:
+ return model
+ from auto_round.export.svdquant_nunchaku import (
+ NUNCHAKU_WEIGHT_FILENAME,
+ SVDQuantExportConfig,
+ save_svdquant_nunchaku_safetensors,
+ )
+
+ self._validate_svd_layer_overrides(model, layer_config)
+ save_config = getattr(model, "save_config", None)
+ if callable(save_config):
+ _invoke_save_hook(save_config, output_dir)
+ else:
+ model_config = getattr(model, "config", None)
+ save_pretrained = getattr(model_config, "save_pretrained", None)
+ if callable(save_pretrained):
+ _invoke_save_hook(save_pretrained, output_dir)
+ model_adapter = model_adapter or getattr(model, "_autoround_svdquant_model_adapter", "auto")
+ if isinstance(model_adapter, str):
+ from auto_round.export.svdquant_adapters import resolve_svdquant_model_adapter
+
+ model_adapter = resolve_svdquant_model_adapter(model_adapter, model, decomposition_device=device)
+ if model_adapter is not None:
+ if adapter is not None:
+ raise TypeError("Pass only one of model_adapter and adapter.")
+ adapter = model_adapter
+ from auto_round.export.svdquant_nunchaku import IdentitySVDQuantModelAdapter
+
+ if isinstance(adapter, IdentitySVDQuantModelAdapter):
+ raise ValueError(
+ "svdquant_nunchaku requires a runtime model adapter; "
+ "use a supported architecture or select one with model_adapter"
+ )
+ if config is None:
+ config = SVDQuantExportConfig(runtime_loadable=True)
+ elif not config.runtime_loadable:
+ raise ValueError("svdquant_nunchaku requires config.runtime_loadable=True")
+
+ output_path = os.path.join(os.fspath(output_dir), NUNCHAKU_WEIGHT_FILENAME)
+ save_svdquant_nunchaku_safetensors(
+ model,
+ output_path,
+ config=config,
+ residual_provider=residual_provider,
+ adapter=adapter,
+ )
+ return model
diff --git a/auto_round/export/formats/base.py b/auto_round/export/formats/base.py
index 80636f99b..47c386d3b 100644
--- a/auto_round/export/formats/base.py
+++ b/auto_round/export/formats/base.py
@@ -152,6 +152,14 @@ class OutputFormat(ABC):
else:
return self.backend.get_backend_name()
+ def is_supported_immediate_packing(self) -> bool:
+ """Whether this format can pack each block before full-model export."""
+ return self.backend.is_supported_immediate_packing() if self.backend is not None else True
+
+ def is_supported_immediate_saving(self) -> bool:
+ """Whether this format can serialize each block during quantization."""
+ return self.backend.is_supported_immediate_saving() if self.backend is not None else True
+
@classmethod
def is_support_scheme(cls: OutputFormat, scheme: Union[str, QuantizationScheme]) -> bool:
if isinstance(scheme, str) and scheme.upper() in cls.support_schemes:
diff --git a/auto_round/export/svdquant_adapters/__init__.py b/auto_round/export/svdquant_adapters/__init__.py
new file mode 100644
index 000000000..e20874ead
--- /dev/null
+++ b/auto_round/export/svdquant_adapters/__init__.py
@@ -0,0 +1,75 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Architecture adapters for SVDQuant Nunchaku export."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+import torch
+
+from auto_round.export.svdquant_nunchaku import IdentitySVDQuantModelAdapter
+
+from auto_round.export.svdquant_adapters.flux import (
+ FLUX_TOP_LEVEL_TENSOR_KEYS,
+ FLUX_SVDQUANT_TARGET_MODULES,
+ FluxSVDQuantNunchakuAdapter,
+ flux_onefile_tensor_count,
+)
+
+
+def _model_config(model: torch.nn.Module) -> dict:
+ config = getattr(model, "config", None)
+ if isinstance(config, Mapping):
+ return dict(config)
+ to_dict = getattr(config, "to_dict", None)
+ if callable(to_dict):
+ value = to_dict()
+ if isinstance(value, Mapping):
+ return dict(value)
+ return {}
+
+
+def resolve_svdquant_model_adapter(
+ name: str,
+ model: torch.nn.Module,
+ *,
+ decomposition_device: str | torch.device = "cpu",
+):
+ """Resolve a registered architecture adapter without runtime dependencies."""
+
+ normalized = name.strip().lower()
+ if normalized not in {"auto", "identity", "flux"}:
+ raise ValueError(f"unknown SVDQuant model adapter {name!r}; expected auto, identity, or flux")
+ config = _model_config(model)
+ class_name = str(config.get("_class_name", type(model).__name__)).lower()
+ if normalized == "auto":
+ normalized = "flux" if "fluxtransformer" in class_name else "identity"
+ if normalized == "flux":
+ return FluxSVDQuantNunchakuAdapter(
+ config=config or None,
+ decomposition_device=decomposition_device,
+ require_complete_model=True,
+ )
+ return IdentitySVDQuantModelAdapter()
+
+
+__all__ = [
+ "FLUX_TOP_LEVEL_TENSOR_KEYS",
+ "FLUX_SVDQUANT_TARGET_MODULES",
+ "FluxSVDQuantNunchakuAdapter",
+ "flux_onefile_tensor_count",
+ "resolve_svdquant_model_adapter",
+]
diff --git a/auto_round/export/svdquant_adapters/flux.py b/auto_round/export/svdquant_adapters/flux.py
new file mode 100644
index 000000000..e95f625f8
--- /dev/null
+++ b/auto_round/export/svdquant_adapters/flux.py
@@ -0,0 +1,625 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""FLUX model mapping for runtime-loadable SVDQuant Nunchaku artifacts."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+import torch
+
+from auto_round.export.svdquant_nunchaku import SourceLinearRecord, SVDQuantExportRecord
+from auto_round.export.svdquant_w4a16 import quantize_adanorm_w4a16_rtn
+
+_BLOCK_RE = re.compile(r"^(transformer_blocks|single_transformer_blocks)\.(\d+)\.(.+)$")
+_DOUBLE_DIRECT = {
+ "attn.to_out.0": "out_proj",
+ "attn.to_add_out": "out_proj_context",
+ "ff.net.0.proj": "mlp_fc1",
+ "ff.net.2": "mlp_fc2",
+ "ff.net.2.linear": "mlp_fc2",
+ "ff_context.net.0.proj": "mlp_context_fc1",
+ "ff_context.net.2": "mlp_context_fc2",
+ "ff_context.net.2.linear": "mlp_context_fc2",
+}
+_DOUBLE_FUSED = {
+ "qkv_proj": ("attn.to_q", "attn.to_k", "attn.to_v"),
+ "qkv_proj_context": ("attn.add_q_proj", "attn.add_k_proj", "attn.add_v_proj"),
+}
+_SINGLE_FUSED = {"qkv_proj": ("attn.to_q", "attn.to_k", "attn.to_v")}
+_RMS_MAP = {
+ "attn.norm_q": "norm_q",
+ "attn.norm_k": "norm_k",
+ "attn.norm_added_q": "norm_added_q",
+ "attn.norm_added_k": "norm_added_k",
+}
+_TOP_LEVEL_PREFIXES = ("x_embedder.", "context_embedder.", "time_text_embed.", "norm_out.linear.", "proj_out.")
+FLUX_SVDQUANT_TARGET_MODULES = (
+ "attn.to_q",
+ "attn.to_k",
+ "attn.to_v",
+ "attn.add_q_proj",
+ "attn.add_k_proj",
+ "attn.add_v_proj",
+ "attn.to_out.0",
+ "attn.to_add_out",
+ "ff.net.0.proj",
+ "ff.net.2",
+ "ff_context.net.0.proj",
+ "ff_context.net.2",
+ "proj_mlp",
+ "proj_out",
+)
+FLUX_TOP_LEVEL_TENSOR_KEYS = frozenset(
+ {
+ "x_embedder.weight",
+ "x_embedder.bias",
+ "context_embedder.weight",
+ "context_embedder.bias",
+ "norm_out.linear.weight",
+ "norm_out.linear.bias",
+ "proj_out.weight",
+ "proj_out.bias",
+ *(
+ f"time_text_embed.{embedder}_embedder.linear_{linear}.{parameter}"
+ for embedder in ("timestep", "text", "guidance")
+ for linear in (1, 2)
+ for parameter in ("weight", "bias")
+ ),
+ }
+)
+
+
+def flux_onefile_tensor_count(
+ num_layers: int, num_single_layers: int, top_level_tensors: int = len(FLUX_TOP_LEVEL_TENSOR_KEYS)
+) -> int:
+ """Return the key count without constructing model-sized tensors."""
+
+ return num_layers * (8 * 7 + 2 * 4 + 4) + num_single_layers * (4 * 7 + 4 + 2) + top_level_tensors
+
+
+def _config_dict(value: Any) -> dict[str, Any]:
+ if isinstance(value, Mapping):
+ return dict(value)
+ to_dict = getattr(value, "to_dict", None)
+ if callable(to_dict):
+ result = to_dict()
+ if isinstance(result, Mapping):
+ return dict(result)
+ if hasattr(value, "__dict__"):
+ return {key: item for key, item in vars(value).items() if not key.startswith("_")}
+ raise ValueError("FLUX config must be a mapping or serialize to a JSON object")
+
+
+def _normalize_config_paths(value: Any) -> Any:
+ if isinstance(value, os.PathLike):
+ return os.fspath(value)
+ if isinstance(value, Mapping):
+ return {key: _normalize_config_paths(item) for key, item in value.items()}
+ if isinstance(value, list):
+ return [_normalize_config_paths(item) for item in value]
+ if isinstance(value, tuple):
+ return tuple(_normalize_config_paths(item) for item in value)
+ return value
+
+
+def _effective_weight(source: SourceLinearRecord, device: torch.device) -> torch.Tensor:
+ if source.residual_weight.ndim != 2 or source.lora_down.ndim != 2 or source.lora_up.ndim != 2:
+ raise ValueError(f"{source.name} residual and low-rank weights must be 2D")
+ out_features, in_features = source.residual_weight.shape
+ rank = source.lora_down.shape[0]
+ if (
+ source.lora_down.shape != (rank, in_features)
+ or source.lora_up.shape != (out_features, rank)
+ or source.smooth.shape != (in_features,)
+ ):
+ raise ValueError(f"{source.name} source dimensions are inconsistent")
+ residual = source.residual_weight.detach().to(device=device, dtype=torch.float32)
+ up = source.lora_up.detach().to(device=device, dtype=torch.float32)
+ down = source.lora_down.detach().to(device=device, dtype=torch.float32)
+ smooth = source.smooth.detach().to(device=device, dtype=torch.float32)
+ return (residual + up @ down) * smooth.reshape(1, -1)
+
+
+def _has_shared_input_decomposition(sources: tuple[SourceLinearRecord, ...]) -> bool:
+ first = sources[0]
+ return all(
+ source.scheme == first.scheme
+ and torch.equal(source.lora_down, first.lora_down)
+ and torch.equal(source.smooth, first.smooth)
+ and torch.equal(source.smooth_orig, first.smooth_orig)
+ for source in sources[1:]
+ )
+
+
+def _concatenate_shared_sources(
+ prefix: str,
+ sources: tuple[SourceLinearRecord, ...],
+) -> SVDQuantExportRecord:
+ first = sources[0]
+ biases = [source.bias for source in sources]
+ if any(bias is None for bias in biases) and not all(bias is None for bias in biases):
+ raise ValueError(f"{prefix} fused sources must either all have bias or all omit bias")
+ bias = None if biases[0] is None else torch.cat(biases).detach().cpu().contiguous()
+ return SVDQuantExportRecord(
+ prefix=prefix,
+ residual_weight=torch.cat([source.residual_weight for source in sources]).detach().cpu().contiguous(),
+ lora_down=first.lora_down.detach().cpu().contiguous(),
+ lora_up=torch.cat([source.lora_up for source in sources]).detach().cpu().contiguous(),
+ smooth=first.smooth.detach().cpu().contiguous(),
+ smooth_orig=first.smooth_orig.detach().cpu().contiguous(),
+ bias=bias,
+ scheme=first.scheme,
+ sources=sources,
+ )
+
+
+def _decompose(
+ weight: torch.Tensor,
+ *,
+ rank: int,
+ template: SourceLinearRecord,
+ prefix: str,
+ sources: tuple[SourceLinearRecord, ...],
+ bias: torch.Tensor | None,
+) -> SVDQuantExportRecord:
+ if rank > min(weight.shape):
+ raise ValueError(f"{prefix} configured rank={rank} exceeds fused dimensions {tuple(weight.shape)}")
+ if not bool(torch.isfinite(weight).all()):
+ raise ValueError(f"{prefix} effective weight contains non-finite values")
+ u, singular_values, vh = torch.linalg.svd(weight, full_matrices=False)
+ up = u[:, :rank] * singular_values[:rank]
+ down = vh[:rank]
+ residual = weight - up @ down
+ weight_dtype = template.residual_weight.dtype
+ low_rank_dtype = template.lora_down.dtype
+ in_features = weight.shape[1]
+ record = SVDQuantExportRecord(
+ prefix=prefix,
+ residual_weight=residual.to(dtype=weight_dtype).cpu().contiguous(),
+ lora_down=down.to(dtype=low_rank_dtype).cpu().contiguous(),
+ lora_up=up.to(dtype=template.lora_up.dtype).cpu().contiguous(),
+ smooth=torch.ones(in_features, dtype=template.smooth.dtype).cpu(),
+ smooth_orig=torch.ones(in_features, dtype=template.smooth_orig.dtype).cpu(),
+ bias=None if bias is None else bias.detach().to(dtype=weight_dtype).cpu().contiguous(),
+ scheme=template.scheme,
+ sources=sources,
+ )
+ del u, singular_values, vh, up, down, residual, weight
+ return record
+
+
+@dataclass
+class FluxSVDQuantNunchakuAdapter:
+ """Map Diffusers FLUX modules to the Nunchaku one-file tensor schema."""
+
+ config: Mapping[str, Any] | None = None
+ decomposition_device: str | torch.device = "cpu"
+ require_complete_model: bool = True
+
+ def __post_init__(self) -> None:
+ try:
+ self.decomposition_device = torch.device(self.decomposition_device)
+ except (RuntimeError, TypeError) as exc:
+ raise ValueError(f"invalid FLUX decomposition_device {self.decomposition_device!r}") from exc
+ if self.decomposition_device.type not in ("cpu", "cuda"):
+ raise ValueError("FLUX decomposition_device must be CPU or CUDA")
+ if self.decomposition_device.type == "cuda":
+ if not torch.cuda.is_available():
+ raise ValueError("FLUX decomposition_device requests CUDA, but CUDA is not available")
+ device_count = torch.cuda.device_count()
+ index = self.decomposition_device.index
+ if index is not None and (index < 0 or index >= device_count):
+ raise ValueError(
+ f"FLUX decomposition_device index {index} is invalid for CUDA device_count={device_count}"
+ )
+ if self.config is not None:
+ self.config = _config_dict(self.config)
+ if not isinstance(self.require_complete_model, bool):
+ raise ValueError("require_complete_model must be a bool")
+
+ def _resolved_config(self, model: torch.nn.Module) -> dict[str, Any]:
+ value = self.config if self.config is not None else getattr(model, "config", None)
+ if value is None:
+ raise ValueError("FLUX export requires explicit config or model.config")
+ config = _normalize_config_paths(_config_dict(value))
+ try:
+ json.dumps(config)
+ except (TypeError, ValueError) as exc:
+ raise ValueError("FLUX config must be JSON serializable") from exc
+ return config
+
+ def metadata(self, model: torch.nn.Module, rank: int) -> Mapping[str, str]:
+ return {
+ "model_class": "NunchakuFluxTransformer2dModel",
+ "config": json.dumps(self._resolved_config(model), sort_keys=True),
+ "format": "pt",
+ "comfy_config": "{}",
+ }
+
+ @staticmethod
+ def _direct(source: SourceLinearRecord, prefix: str) -> SVDQuantExportRecord:
+ return SVDQuantExportRecord(
+ prefix=prefix,
+ residual_weight=source.residual_weight,
+ lora_down=source.lora_down,
+ lora_up=source.lora_up,
+ smooth=source.smooth,
+ smooth_orig=source.smooth_orig,
+ bias=source.bias,
+ scheme=source.scheme,
+ sources=(source,),
+ )
+
+ def _fuse(self, prefix: str, sources: tuple[SourceLinearRecord, ...], rank: int) -> SVDQuantExportRecord:
+ def operation() -> SVDQuantExportRecord:
+ if _has_shared_input_decomposition(sources):
+ return _concatenate_shared_sources(prefix, sources)
+ effective = [_effective_weight(source, self.decomposition_device) for source in sources]
+ input_dims = {weight.shape[1] for weight in effective}
+ if len(input_dims) != 1:
+ raise ValueError(f"{prefix} fused sources have incompatible input dimensions {sorted(input_dims)}")
+ weight = torch.cat(effective, dim=0)
+ biases = [source.bias for source in sources]
+ if any(bias is None for bias in biases) and not all(bias is None for bias in biases):
+ raise ValueError(f"{prefix} fused sources must either all have bias or all omit bias")
+ bias = None if biases[0] is None else torch.cat([item.to(self.decomposition_device) for item in biases])
+ return _decompose(weight, rank=rank, template=sources[0], prefix=prefix, sources=sources, bias=bias)
+
+ try:
+ return operation()
+ finally:
+ self._clear_decomposition_cache()
+
+ def _split_single_proj_out(
+ self, model: torch.nn.Module, block_prefix: str, source: SourceLinearRecord, rank: int
+ ) -> tuple[SVDQuantExportRecord, SVDQuantExportRecord]:
+ def operation() -> tuple[SVDQuantExportRecord, SVDQuantExportRecord]:
+ config = self._resolved_config(model)
+ heads, head_dim = config.get("num_attention_heads"), config.get("attention_head_dim")
+ in_features = source.residual_weight.shape[1]
+ inner_dim = (
+ heads * head_dim
+ if isinstance(heads, int) and isinstance(head_dim, int)
+ else source.residual_weight.shape[0]
+ )
+ if inner_dim <= 0 or inner_dim >= in_features:
+ raise ValueError(
+ f"{block_prefix}.proj_out cannot split input columns at inner_dim={inner_dim} "
+ f"for shape {tuple(source.residual_weight.shape)}"
+ )
+ out_proj = SVDQuantExportRecord(
+ prefix=f"{block_prefix}.out_proj",
+ residual_weight=source.residual_weight[:, :inner_dim].contiguous(),
+ lora_down=source.lora_down[:, :inner_dim].contiguous(),
+ lora_up=source.lora_up,
+ smooth=source.smooth[:inner_dim].contiguous(),
+ smooth_orig=source.smooth_orig[:inner_dim].contiguous(),
+ bias=None,
+ scheme=source.scheme,
+ sources=(source,),
+ )
+ mlp_fc2 = SVDQuantExportRecord(
+ prefix=f"{block_prefix}.mlp_fc2",
+ residual_weight=source.residual_weight[:, inner_dim:].contiguous(),
+ lora_down=source.lora_down[:, inner_dim:].contiguous(),
+ lora_up=source.lora_up,
+ smooth=source.smooth[inner_dim:].contiguous(),
+ smooth_orig=source.smooth_orig[inner_dim:].contiguous(),
+ bias=source.bias,
+ scheme=source.scheme,
+ sources=(source,),
+ )
+ return out_proj, mlp_fc2
+
+ try:
+ return operation()
+ finally:
+ self._clear_decomposition_cache()
+
+ def _clear_decomposition_cache(self) -> None:
+ if self.decomposition_device.type == "cuda":
+ torch.cuda.empty_cache()
+
+ def map_modules(
+ self, model: torch.nn.Module, records: Iterable[SourceLinearRecord]
+ ) -> Iterable[SVDQuantExportRecord]:
+ records = tuple(records)
+ by_block: dict[tuple[str, int], dict[str, SourceLinearRecord]] = {}
+ for record in records:
+ match = _BLOCK_RE.match(record.name)
+ if match is None:
+ raise ValueError(f"unrecognized FLUX SVDQuant source {record.name!r}")
+ family, index, local_name = match.groups()
+ local = by_block.setdefault((family, int(index)), {})
+ if local_name in local:
+ raise ValueError(f"duplicate FLUX source {record.name!r}")
+ local[local_name] = record
+
+ if self.require_complete_model:
+ config = self._resolved_config(model)
+ try:
+ expected_indices = {
+ "transformer_blocks": set(range(int(config["num_layers"]))),
+ "single_transformer_blocks": set(range(int(config["num_single_layers"]))),
+ }
+ except (KeyError, TypeError, ValueError) as exc:
+ raise ValueError("complete FLUX export requires integer num_layers and num_single_layers") from exc
+ for family, expected in expected_indices.items():
+ actual = {index for candidate, index in by_block if candidate == family}
+ if actual != expected:
+ raise ValueError(
+ f"complete FLUX export {family} indices mismatch: expected {sorted(expected)}, "
+ f"got {sorted(actual)}"
+ )
+ double_fixed = {name for names in _DOUBLE_FUSED.values() for name in names} | {
+ "attn.to_out.0",
+ "attn.to_add_out",
+ "ff.net.0.proj",
+ "ff_context.net.0.proj",
+ }
+ for (family, index), local in by_block.items():
+ names = set(local)
+ if family == "single_transformer_blocks":
+ required = {"attn.to_q", "attn.to_k", "attn.to_v", "proj_mlp", "proj_out"}
+ missing, extra = required - names, names - required
+ else:
+ missing = double_fixed - names
+ extra = (
+ names
+ - set(_DOUBLE_DIRECT)
+ - {name for fused_names in _DOUBLE_FUSED.values() for name in fused_names}
+ )
+ for stem in ("ff.net.2", "ff_context.net.2"):
+ variants = {stem, f"{stem}.linear"}
+ present = names & variants
+ if len(present) != 1:
+ missing.add(f"exactly one of {sorted(variants)}")
+ if missing or extra:
+ raise ValueError(
+ f"complete FLUX export rejects {family}.{index} coverage: "
+ f"missing={sorted(missing)}, extras={sorted(extra)}"
+ )
+
+ output: list[SVDQuantExportRecord] = []
+ for (family, index), local in sorted(by_block.items()):
+ block_prefix = f"{family}.{index}"
+ rank = next(iter(local.values())).lora_down.shape[0]
+ if family == "transformer_blocks":
+ for target, names in _DOUBLE_FUSED.items():
+ present = tuple(local[name] for name in names if name in local)
+ if present and len(present) != len(names):
+ missing = [name for name in names if name not in local]
+ raise ValueError(f"{block_prefix}.{target} missing fused sources {missing}")
+ if present:
+ output.append(self._fuse(f"{block_prefix}.{target}", present, rank))
+ for source_name, target in _DOUBLE_DIRECT.items():
+ if source_name in local:
+ output.append(self._direct(local[source_name], f"{block_prefix}.{target}"))
+ else:
+ names = _SINGLE_FUSED["qkv_proj"]
+ present = tuple(local[name] for name in names if name in local)
+ if present and len(present) != len(names):
+ missing = [name for name in names if name not in local]
+ raise ValueError(f"{block_prefix}.qkv_proj missing fused sources {missing}")
+ if present:
+ output.append(self._fuse(f"{block_prefix}.qkv_proj", present, rank))
+ if "proj_mlp" in local:
+ output.append(self._direct(local["proj_mlp"], f"{block_prefix}.mlp_fc1"))
+ if "proj_out" in local:
+ source = local["proj_out"]
+ output.extend(self._split_single_proj_out(model, block_prefix, source, rank))
+ return output
+
+ def validate_records(
+ self, sources: tuple[SourceLinearRecord, ...], records: tuple[SVDQuantExportRecord, ...]
+ ) -> None:
+ ranks = {source.lora_down.shape[0] for source in sources}
+ if len(ranks) != 1:
+ raise ValueError(f"FLUX source ranks must agree, got {sorted(ranks)}")
+ if self.require_complete_model:
+ # Config was already resolved during mapping for single blocks; retain model-independent
+ # structural checks here and perform exact source coverage in map_modules via cached names.
+ families: dict[str, set[int]] = {"transformer_blocks": set(), "single_transformer_blocks": set()}
+ for source in sources:
+ match = _BLOCK_RE.match(source.name)
+ assert match is not None
+ families[match.group(1)].add(int(match.group(2)))
+ for family, indices in families.items():
+ if indices and indices != set(range(max(indices) + 1)):
+ raise ValueError(f"complete FLUX export rejects gaps in {family}: {sorted(indices)}")
+ prefixes = [record.prefix for record in records]
+ if len(prefixes) != len(set(prefixes)):
+ raise ValueError("FLUX adapter produced duplicate logical record prefixes")
+
+ @staticmethod
+ def _module_map(model: torch.nn.Module) -> dict[str, torch.nn.Module]:
+ return dict(model.named_modules())
+
+ def extra_tensors(self, model: torch.nn.Module) -> Mapping[str, torch.Tensor]:
+ modules = self._module_map(model)
+ tensors: dict[str, torch.Tensor] = {}
+ block_indices: dict[str, set[int]] = {"transformer_blocks": set(), "single_transformer_blocks": set()}
+ for name in modules:
+ match = _BLOCK_RE.match(name)
+ if match:
+ block_indices[match.group(1)].add(int(match.group(2)))
+
+ if self.require_complete_model:
+ config = self._resolved_config(model)
+ for family, config_name in (
+ ("transformer_blocks", "num_layers"),
+ ("single_transformer_blocks", "num_single_layers"),
+ ):
+ expected = set(range(int(config[config_name])))
+ if block_indices[family] != expected:
+ raise ValueError(
+ f"complete FLUX extras {family} indices mismatch: expected {sorted(expected)}, "
+ f"got {sorted(block_indices[family])}"
+ )
+
+ for family, indices in block_indices.items():
+ for index in sorted(indices):
+ block = f"{family}.{index}"
+ adanorms = (
+ (("norm1.linear", 6), ("norm1_context.linear", 6))
+ if family == "transformer_blocks"
+ else (("norm.linear", 3),)
+ )
+ for local_name, splits in adanorms:
+ module = modules.get(f"{block}.{local_name}")
+ if module is None:
+ if self.require_complete_model:
+ raise ValueError(f"complete FLUX export missing {block}.{local_name}")
+ continue
+ weight = getattr(module, "weight", None)
+ bias = getattr(module, "bias", None)
+ if not isinstance(weight, torch.Tensor):
+ raise ValueError(f"{block}.{local_name}.weight must be a tensor")
+ packed = quantize_adanorm_w4a16_rtn(
+ weight.detach().to(device="cpu", dtype=torch.bfloat16),
+ bias=None if bias is None else bias.detach().to(device="cpu", dtype=torch.bfloat16),
+ splits=splits,
+ group_size=64,
+ )
+ prefix = f"{block}.{local_name}"
+ tensors.update(
+ {
+ f"{prefix}.qweight": packed.qweight,
+ f"{prefix}.wscales": packed.wscales,
+ f"{prefix}.wzeros": packed.wzeros,
+ f"{prefix}.bias": packed.bias,
+ }
+ )
+ for source_name, target_name in _RMS_MAP.items():
+ module = modules.get(f"{block}.{source_name}")
+ weight = None if module is None else getattr(module, "weight", None)
+ if weight is None:
+ required_rms = family == "transformer_blocks" or source_name in ("attn.norm_q", "attn.norm_k")
+ if self.require_complete_model and required_rms:
+ raise ValueError(f"complete FLUX export missing {block}.{source_name}.weight")
+ continue
+ tensors[f"{block}.{target_name}.weight"] = weight.detach().to(torch.bfloat16).cpu()
+
+ passthrough = {
+ name: parameter for name, parameter in model.named_parameters() if name.startswith(_TOP_LEVEL_PREFIXES)
+ }
+ if self.require_complete_model:
+ missing_top_level = FLUX_TOP_LEVEL_TENSOR_KEYS - passthrough.keys()
+ if missing_top_level:
+ raise ValueError(f"complete FLUX model is missing top-level parameters: {sorted(missing_top_level)}")
+ extra_top_level = passthrough.keys() - FLUX_TOP_LEVEL_TENSOR_KEYS
+ if extra_top_level:
+ raise ValueError(f"complete FLUX model has unexpected top-level parameters: {sorted(extra_top_level)}")
+ for name in FLUX_TOP_LEVEL_TENSOR_KEYS & passthrough.keys():
+ tensors[name] = passthrough[name].detach().to(torch.bfloat16).cpu()
+ return {key: value.contiguous() for key, value in tensors.items()}
+
+ def validate(self, tensors: Mapping[str, torch.Tensor], metadata: Mapping[str, str]) -> None:
+ if metadata.get("model_class") != "NunchakuFluxTransformer2dModel":
+ raise ValueError("FLUX metadata has incorrect model_class")
+ if metadata.get("format") != "pt" or metadata.get("comfy_config") != "{}":
+ raise ValueError("FLUX metadata requires format='pt' and empty comfy_config")
+ try:
+ config = json.loads(metadata["config"])
+ except (KeyError, json.JSONDecodeError) as exc:
+ raise ValueError("FLUX metadata config must be a JSON object") from exc
+ if not isinstance(config, dict):
+ raise ValueError("FLUX metadata config must be a JSON object")
+ required: set[str] = set()
+ if self.require_complete_model:
+ try:
+ num_layers = int(config["num_layers"])
+ num_single_layers = int(config["num_single_layers"])
+ except (KeyError, TypeError, ValueError) as exc:
+ raise ValueError("complete FLUX metadata requires layer counts") from exc
+ top_level_keys = {
+ key for key in tensors if not key.startswith(("transformer_blocks.", "single_transformer_blocks."))
+ }
+ missing_top_level = FLUX_TOP_LEVEL_TENSOR_KEYS - top_level_keys
+ if missing_top_level:
+ raise ValueError(f"complete FLUX artifact is missing top-level tensors: {sorted(missing_top_level)}")
+ extra_top_level = top_level_keys - FLUX_TOP_LEVEL_TENSOR_KEYS
+ if extra_top_level:
+ raise ValueError(f"complete FLUX artifact has unexpected top-level tensors: {sorted(extra_top_level)}")
+ linear_suffixes = ("qweight", "wscales", "smooth", "smooth_orig", "lora_down", "lora_up", "bias")
+ for index in range(num_layers):
+ block = f"transformer_blocks.{index}"
+ for linear in (
+ "qkv_proj",
+ "qkv_proj_context",
+ "out_proj",
+ "out_proj_context",
+ "mlp_fc1",
+ "mlp_fc2",
+ "mlp_context_fc1",
+ "mlp_context_fc2",
+ ):
+ required.update(f"{block}.{linear}.{suffix}" for suffix in linear_suffixes)
+ for norm in ("norm1.linear", "norm1_context.linear"):
+ required.update(f"{block}.{norm}.{suffix}" for suffix in ("qweight", "wscales", "wzeros", "bias"))
+ required.update(
+ f"{block}.{norm}.weight" for norm in ("norm_q", "norm_k", "norm_added_q", "norm_added_k")
+ )
+ for index in range(num_single_layers):
+ block = f"single_transformer_blocks.{index}"
+ for linear in ("qkv_proj", "out_proj", "mlp_fc1", "mlp_fc2"):
+ required.update(f"{block}.{linear}.{suffix}" for suffix in linear_suffixes)
+ required.update(f"{block}.norm.linear.{suffix}" for suffix in ("qweight", "wscales", "wzeros", "bias"))
+ required.update(f"{block}.{norm}.weight" for norm in ("norm_q", "norm_k"))
+ missing = required - tensors.keys()
+ if missing:
+ raise ValueError(f"complete FLUX artifact is missing expected tensors: {sorted(missing)[:5]}")
+ if num_layers == 19 and num_single_layers == 38 and len(tensors) != flux_onefile_tensor_count(19, 38):
+ raise ValueError(f"standard FLUX one-file artifact must contain 2604 tensors, got {len(tensors)}")
+ for key, tensor in tensors.items():
+ if tensor.device.type != "cpu" or not tensor.is_contiguous():
+ raise ValueError(f"FLUX tensor {key!r} must be contiguous on CPU")
+ if tensor.is_floating_point() and not bool(torch.isfinite(tensor).all()):
+ raise ValueError(f"FLUX tensor {key!r} must be finite")
+ if not key.startswith(("transformer_blocks.", "single_transformer_blocks.")):
+ if not key.startswith(_TOP_LEVEL_PREFIXES) or tensor.dtype != torch.bfloat16:
+ raise ValueError(f"FLUX passthrough tensor {key!r} must be in an allowed top-level BF16 family")
+ is_adanorm = ".norm" in key and any(
+ marker in key for marker in (".norm.linear.", ".norm1.linear.", ".norm1_context.linear.")
+ )
+ if key.endswith(".qweight"):
+ expected_dtype = torch.int32 if is_adanorm else torch.int8
+ if tensor.dtype != expected_dtype or tensor.ndim != 2:
+ raise ValueError(f"FLUX qweight {key!r} must be 2D {expected_dtype}")
+ elif key.endswith(".wscales"):
+ expected_dtype = torch.bfloat16 if is_adanorm else torch.uint8
+ if tensor.dtype != expected_dtype or tensor.ndim != 2:
+ raise ValueError(f"FLUX wscales {key!r} must be 2D {expected_dtype}")
+ elif key.endswith(".wzeros"):
+ if not is_adanorm or tensor.dtype != torch.bfloat16 or tensor.ndim != 2:
+ raise ValueError(f"FLUX wzeros {key!r} must be a 2D AdaNorm BF16 tensor")
+ elif key.endswith((".lora_down", ".lora_up")):
+ if tensor.dtype != torch.bfloat16 or tensor.ndim != 2:
+ raise ValueError(f"FLUX low-rank tensor {key!r} must be 2D BF16")
+ elif key.endswith((".smooth", ".smooth_orig", ".bias")):
+ if tensor.dtype != torch.bfloat16 or tensor.ndim != 1:
+ raise ValueError(f"FLUX vector tensor {key!r} must be 1D BF16")
+ elif key.endswith(".weight") and key.startswith(("transformer_blocks.", "single_transformer_blocks.")):
+ if tensor.dtype != torch.bfloat16 or tensor.ndim != 1:
+ raise ValueError(f"FLUX RMSNorm tensor {key!r} must be 1D BF16")
+
+
+__all__ = ["FLUX_TOP_LEVEL_TENSOR_KEYS", "FluxSVDQuantNunchakuAdapter", "flux_onefile_tensor_count"]
diff --git a/auto_round/export/svdquant_mxfp4.py b/auto_round/export/svdquant_mxfp4.py
new file mode 100644
index 000000000..534fdc110
--- /dev/null
+++ b/auto_round/export/svdquant_mxfp4.py
@@ -0,0 +1,404 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Logical MXFP4 codecs and Nunchaku-compatible residual packing.
+
+E2M1 scales use an explicit rank contract: scalars are global, scales with one
+fewer dimension than values are group-aligned along a new trailing singleton,
+and scales with the same rank use ordinary PyTorch broadcasting.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import torch
+
+from auto_round.data_type.mxfp import quant_element, quant_mx_rceil
+
+_E2M1_MAGNITUDES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
+_SUPPORTED_DECODE_DTYPES = (torch.float16, torch.bfloat16, torch.float32, torch.float64)
+_SUPPORTED_PACKED_DTYPES = (torch.float16, torch.bfloat16)
+
+
+def _validate_lowrank_weight(weight: torch.Tensor, down: bool) -> None:
+ if not isinstance(weight, torch.Tensor) or weight.ndim != 2:
+ raise ValueError("weight must be a 2D torch.Tensor")
+ if weight.dtype not in _SUPPORTED_PACKED_DTYPES:
+ raise ValueError("weight dtype must be torch.float16 or torch.bfloat16")
+ if not isinstance(down, bool):
+ raise ValueError("down must be a bool")
+ if 0 in weight.shape:
+ raise ValueError("weight dimensions must be non-empty")
+ if not bool(torch.isfinite(weight).all()):
+ raise ValueError("weight must contain only finite values")
+
+
+def pack_lowrank_weight(weight: torch.Tensor, down: bool) -> torch.Tensor:
+ """Pack a logical low-rank matrix with 128-feature and 16-rank alignment."""
+
+ _validate_lowrank_weight(weight, down)
+ rows = NunchakuMXFP4Packer._ceil_to(weight.shape[0], 16 if down else 128)
+ columns = NunchakuMXFP4Packer._ceil_to(weight.shape[1], 128 if down else 16)
+ padded = torch.zeros((rows, columns), dtype=weight.dtype, device=weight.device)
+ padded[: weight.shape[0], : weight.shape[1]] = weight
+ pack_n = pack_k = 16
+ if down:
+ rank, channels = padded.shape
+ rank_packs, channel_packs = rank // pack_n, channels // pack_k
+ packed = padded.view(rank_packs, pack_n, channel_packs, pack_k).permute(2, 0, 1, 3)
+ else:
+ channels, rank = padded.shape
+ channel_packs, rank_packs = channels // pack_n, rank // pack_k
+ packed = padded.view(channel_packs, pack_n, rank_packs, pack_k).permute(0, 2, 1, 3)
+ packed = packed.reshape(channel_packs, rank_packs, 2, 8, 1, 2, 4, 2)
+ return packed.permute(0, 1, 3, 6, 2, 5, 4, 7).contiguous().view(channels, rank)
+
+
+def unpack_lowrank_weight(weight: torch.Tensor, down: bool) -> torch.Tensor:
+ """Invert :func:`pack_lowrank_weight`, retaining its padded logical shape."""
+
+ _validate_lowrank_weight(weight, down)
+ channels, rank = weight.shape
+ if channels % 128 or rank % 16:
+ raise ValueError("packed weight feature and rank dimensions must be divisible by 128 and 16 respectively")
+ if down:
+ rank_packs, channel_packs = rank // 16, channels // 16
+ else:
+ channel_packs, rank_packs = channels // 16, rank // 16
+ unpacked = weight.view(channel_packs, rank_packs, 8, 4, 2, 2, 1, 2)
+ unpacked = unpacked.permute(0, 1, 4, 2, 6, 5, 3, 7).contiguous().view(channel_packs, rank_packs, 16, 16)
+ if down:
+ return unpacked.permute(1, 2, 0, 3).contiguous().view(rank, channels)
+ return unpacked.permute(0, 2, 1, 3).contiguous().view(channels, rank)
+
+
+@dataclass(frozen=True)
+class PackedMXFP4:
+ """Physical MXFP4 residual tensors and their logical/padded dimensions."""
+
+ qweight: torch.Tensor
+ wscales: torch.Tensor
+ logical_shape: tuple[int, int]
+ padded_shape: tuple[int, int]
+
+
+class NunchakuMXFP4Packer:
+ """Pack MXFP4 residuals into the Nunchaku 4-bit MMA memory layout.
+
+ The reshape/permutation constants follow the Nunchaku weight and MXFP4
+ micro-scale packers at reference source commit 0abaaf0.
+ """
+
+ comp_n = 16
+ comp_k = mem_k = 64
+ num_n_lanes = 8
+ num_k_lanes = 4
+ n_pack_size = 2
+ k_pack_size = 2
+ reg_n = 1
+ reg_k = 8
+ num_k_unrolls = 2
+
+ def __init__(self, warp_n: int = 128) -> None:
+ if warp_n != 128:
+ raise ValueError("warp_n must be 128")
+ self.warp_n = warp_n
+ self.mem_n = warp_n
+
+ @staticmethod
+ def _ceil_to(value: int, divisor: int) -> int:
+ return (value + divisor - 1) // divisor * divisor
+
+ def _pack_weight_codes(self, codes: torch.Tensor) -> torch.Tensor:
+ n, k = codes.shape
+ weight = codes.to(torch.int32).reshape(
+ n // self.mem_n,
+ self.mem_n // (self.n_pack_size * self.num_n_lanes * self.reg_n),
+ self.n_pack_size,
+ self.num_n_lanes,
+ self.reg_n,
+ k // self.mem_k,
+ 1,
+ self.k_pack_size,
+ self.num_k_lanes,
+ self.reg_k,
+ )
+ weight = weight.permute(0, 5, 6, 1, 3, 8, 2, 7, 4, 9).contiguous()
+ shifts = torch.arange(0, 32, 4, dtype=torch.int32, device=codes.device)
+ packed = ((weight & 0xF) << shifts).sum(dim=-1, dtype=torch.int32)
+ return packed.view(torch.int8).view(n, k // 2)
+
+ def _pack_scale_codes(self, scales: torch.Tensor) -> torch.Tensor:
+ n, num_groups = scales.shape
+ scale = scales.view(n // self.warp_n, 1, 4, 4, 8, num_groups // 2, 2)
+ return scale.permute(0, 5, 1, 4, 3, 2, 6).contiguous().view(num_groups, n)
+
+ def _unpack_weight_codes(self, qweight: torch.Tensor) -> torch.Tensor:
+ n, packed_k = qweight.shape
+ k = packed_k * 2
+ packed = (
+ qweight.contiguous()
+ .view(torch.int32)
+ .reshape(
+ n // self.mem_n,
+ k // self.mem_k,
+ 1,
+ self.mem_n // (self.n_pack_size * self.num_n_lanes * self.reg_n),
+ self.num_n_lanes,
+ self.num_k_lanes,
+ self.n_pack_size,
+ self.k_pack_size,
+ self.reg_n,
+ )
+ )
+ shifts = torch.arange(0, 32, 4, dtype=torch.int32, device=qweight.device)
+ weight = (packed.unsqueeze(-1) >> shifts) & 0xF
+ return weight.permute(0, 3, 6, 4, 8, 1, 2, 7, 5, 9).contiguous().view(n, k).to(torch.uint8)
+
+ def _unpack_scale_codes(self, wscales: torch.Tensor) -> torch.Tensor:
+ num_groups, n = wscales.shape
+ scale = wscales.reshape(n // self.warp_n, num_groups // 2, 1, 8, 4, 4, 2)
+ return scale.permute(0, 2, 5, 4, 3, 1, 6).contiguous().view(n, num_groups)
+
+ def pack_residual(self, weight: torch.Tensor, group_size: int = 32) -> PackedMXFP4:
+ """Quantize and physically pack a logical ``[N, K]`` residual."""
+
+ if not isinstance(weight, torch.Tensor) or weight.ndim != 2 or not weight.is_floating_point():
+ raise ValueError("weight must be a 2D floating-point torch.Tensor")
+ if not bool(torch.isfinite(weight).all()):
+ raise ValueError("weight must contain only finite values")
+ if group_size != 32:
+ raise ValueError("group_size must be 32")
+
+ n, k = weight.shape
+ if n == 0 or k == 0:
+ raise ValueError("weight dimensions must be non-empty")
+ n_padded = self._ceil_to(n, self.mem_n)
+ k_group_padded = self._ceil_to(k, group_size)
+ k_padded = self._ceil_to(k_group_padded, self.mem_k * self.num_k_unrolls)
+
+ qdq, shared_exponent, _ = quant_mx_rceil(
+ weight,
+ bits=4,
+ group_size=group_size,
+ data_type="mx_fp4e2m1",
+ )
+ num_logical_groups = k_group_padded // group_size
+ scales = torch.exp2(shared_exponent.reshape(n, num_logical_groups).to(torch.float32))
+
+ grouped_qdq = torch.zeros((n, k_group_padded), dtype=qdq.dtype, device=qdq.device)
+ grouped_qdq[:, :k] = qdq
+ logical_codes = encode_e2m1(grouped_qdq.reshape(n, num_logical_groups, group_size), scales)
+
+ padded_codes = torch.zeros((n_padded, k_padded), dtype=torch.uint8, device=weight.device)
+ padded_codes[:n, :k_group_padded] = logical_codes.reshape(n, k_group_padded)
+ padded_scales = torch.full((n_padded, k_padded // group_size), 127, dtype=torch.uint8, device=weight.device)
+ padded_scales[:n, :num_logical_groups] = encode_ue8m0(scales)
+
+ return PackedMXFP4(
+ qweight=self._pack_weight_codes(padded_codes),
+ wscales=self._pack_scale_codes(padded_scales),
+ logical_shape=(n, k),
+ padded_shape=(n_padded, k_padded),
+ )
+
+ def unpack_residual(
+ self,
+ qweight: torch.Tensor,
+ wscales: torch.Tensor,
+ logical_shape: tuple[int, int],
+ dtype: torch.dtype = torch.float32,
+ ) -> torch.Tensor:
+ """Inverse the physical layout and dequantize the logical residual."""
+
+ if not isinstance(qweight, torch.Tensor) or qweight.dtype != torch.int8:
+ raise ValueError("qweight must be a torch.int8 tensor")
+ if qweight.ndim != 2 or qweight.shape[0] == 0 or qweight.shape[0] % self.mem_n:
+ raise ValueError("qweight shape must be [Npad, Kpad/2] with Npad divisible by 128")
+ if qweight.shape[1] == 0 or (qweight.shape[1] * 2) % (self.mem_k * self.num_k_unrolls):
+ raise ValueError("qweight shape must have Kpad divisible by 128")
+ n_padded, packed_k = qweight.shape
+ k_padded = packed_k * 2
+ expected_scale_shape = (k_padded // 32, n_padded)
+ if not isinstance(wscales, torch.Tensor) or wscales.dtype != torch.uint8:
+ raise ValueError("wscales must be a torch.uint8 tensor")
+ if tuple(wscales.shape) != expected_scale_shape:
+ raise ValueError(f"wscales shape must be {expected_scale_shape}")
+ if wscales.device != qweight.device:
+ raise ValueError("qweight and wscales must be on the same device")
+ if (
+ not isinstance(logical_shape, tuple)
+ or len(logical_shape) != 2
+ or any(isinstance(value, bool) or not isinstance(value, int) for value in logical_shape)
+ or any(value <= 0 for value in logical_shape)
+ or logical_shape[0] > n_padded
+ or logical_shape[1] > k_padded
+ ):
+ raise ValueError("logical_shape must be a positive (N, K) tuple within the padded shape")
+ if dtype not in _SUPPORTED_DECODE_DTYPES:
+ raise ValueError("dtype must be one of torch.float16, torch.bfloat16, torch.float32, or torch.float64")
+ weight_codes = self._unpack_weight_codes(qweight)
+ scale_codes = self._unpack_scale_codes(wscales)
+ scales = decode_ue8m0(scale_codes)
+ dequantized = decode_e2m1(weight_codes.reshape(n_padded, k_padded // 32, 32), scales, dtype=dtype).reshape(
+ n_padded, k_padded
+ )
+ n, k = logical_shape
+ return dequantized[:n, :k].contiguous()
+
+
+def _broadcast_scales(scales: torch.Tensor, shape: torch.Size, *, device: torch.device) -> torch.Tensor:
+ """Apply the codec scale layout contract.
+
+ A scalar scale is global. A scale tensor with one fewer dimension than the
+ values is group-aligned by appending a trailing singleton. A scale tensor
+ with equal rank uses ordinary PyTorch broadcasting. Other ranks are invalid.
+ """
+
+ if not isinstance(scales, torch.Tensor):
+ raise ValueError("scales must be a torch.Tensor")
+ if not scales.is_floating_point():
+ raise ValueError("scales must have a floating-point dtype")
+ if scales.device != device:
+ raise ValueError(f"scales must be on device {device}, got {scales.device}")
+ if not bool(torch.isfinite(scales).all()) or not bool((scales > 0).all()):
+ raise ValueError("scales must contain only positive finite values")
+ tensor_ndim = len(shape)
+ if scales.ndim == 0:
+ aligned_scales = scales
+ layout = "global"
+ elif scales.ndim == tensor_ndim - 1:
+ aligned_scales = scales.unsqueeze(-1)
+ layout = "group-aligned"
+ elif scales.ndim == tensor_ndim:
+ aligned_scales = scales
+ layout = "ordinary-broadcast"
+ else:
+ raise ValueError(
+ f"scales rank {scales.ndim} is invalid for tensor rank {tensor_ndim}; expected a scalar, "
+ f"rank {tensor_ndim - 1} for group alignment, or rank {tensor_ndim} for ordinary broadcasting"
+ )
+ try:
+ return torch.broadcast_to(aligned_scales, shape)
+ except RuntimeError as exc:
+ raise ValueError(
+ f"scales shape {tuple(scales.shape)} with {layout} layout cannot broadcast to tensor shape {tuple(shape)}"
+ ) from exc
+
+
+def _validate_codes(codes: torch.Tensor, *, maximum: int) -> None:
+ if not isinstance(codes, torch.Tensor):
+ raise ValueError("codes must be a torch.Tensor")
+ if codes.dtype == torch.bool or codes.is_floating_point() or codes.is_complex():
+ raise ValueError("codes must have an integer dtype")
+ if codes.numel() and (int(codes.min()) < 0 or int(codes.max()) > maximum):
+ raise ValueError(f"codes must be in the range 0..{maximum}")
+
+
+def encode_e2m1(values: torch.Tensor, scales: torch.Tensor) -> torch.Tensor:
+ """Encode raw values as logical E2M1 codes after dividing by ``scales``."""
+
+ if not isinstance(values, torch.Tensor) or not values.is_floating_point():
+ raise ValueError("values must be a floating-point torch.Tensor")
+ if not bool(torch.isfinite(values).all()):
+ raise ValueError("values must contain only finite values")
+ expanded_scales = _broadcast_scales(scales, values.shape, device=values.device)
+ normalized = values.to(torch.float32) / expanded_scales.to(torch.float32)
+ invalid = torch.isnan(normalized)
+ if bool(invalid.any()):
+ fallback = (values.to(torch.float64) / expanded_scales.to(torch.float64)).clamp(min=-6.0, max=6.0)
+ normalized = torch.where(invalid, fallback.to(torch.float32), normalized)
+ normalized = normalized.clamp(min=-6.0, max=6.0)
+ quantized = quant_element(normalized, ebits=2, mbits=3, max_norm=6.0)
+
+ magnitudes = torch.tensor(_E2M1_MAGNITUDES, dtype=quantized.dtype, device=quantized.device)
+ magnitude_codes = torch.searchsorted(magnitudes, torch.abs(quantized)).to(torch.uint8)
+ sign_codes = torch.signbit(normalized).to(torch.uint8) << 3
+ return magnitude_codes | sign_codes
+
+
+def decode_e2m1(codes: torch.Tensor, scales: torch.Tensor, dtype: torch.dtype = torch.float32) -> torch.Tensor:
+ """Decode logical E2M1 codes and multiply by ``scales``."""
+
+ _validate_codes(codes, maximum=15)
+ if dtype not in _SUPPORTED_DECODE_DTYPES:
+ raise ValueError("dtype must be one of torch.float16, torch.bfloat16, torch.float32, or torch.float64")
+ expanded_scales = _broadcast_scales(scales, codes.shape, device=codes.device)
+ codebook = torch.tensor(
+ (*_E2M1_MAGNITUDES, *(-value for value in _E2M1_MAGNITUDES)),
+ dtype=dtype,
+ device=codes.device,
+ )
+ return codebook[codes.to(torch.long)] * expanded_scales.to(dtype)
+
+
+def encode_ue8m0(scales: torch.Tensor) -> torch.Tensor:
+ """Encode scales using an unsigned exponent with bias 127.
+
+ Non-finite and nonpositive scales use code 127, which decodes to 1.0.
+ """
+
+ if not isinstance(scales, torch.Tensor) or not scales.is_floating_point():
+ raise ValueError("scales must be a floating-point torch.Tensor")
+ valid = torch.isfinite(scales) & (scales > 0)
+ safe_scales = torch.where(valid, scales, torch.ones_like(scales))
+ exponents = torch.ceil(torch.log2(safe_scales)).clamp(min=-127, max=127)
+ codes = (exponents + 127).to(torch.uint8)
+ return torch.where(valid, codes, torch.full_like(codes, 127))
+
+
+def decode_ue8m0(codes: torch.Tensor) -> torch.Tensor:
+ """Decode UE8M0 codes to float32 powers of two.
+
+ Code 255 remains decodable for Nunchaku runtime parity, but
+ :func:`encode_ue8m0` clamps valid exponents to 127 and never emits it.
+ """
+
+ _validate_codes(codes, maximum=255)
+ exponents = codes.to(torch.int16) - 127
+ return torch.exp2(exponents.to(torch.float32))
+
+
+def pack_nibbles(codes: torch.Tensor) -> torch.Tensor:
+ """Pack logical codes along the last dimension, low nibble first."""
+
+ _validate_codes(codes, maximum=15)
+ if codes.ndim == 0:
+ raise ValueError("codes must have at least one dimension")
+ codes = codes.to(torch.uint8)
+ if codes.shape[-1] % 2:
+ padding = torch.zeros((*codes.shape[:-1], 1), dtype=torch.uint8, device=codes.device)
+ codes = torch.cat((codes, padding), dim=-1)
+ pairs = codes.reshape(*codes.shape[:-1], codes.shape[-1] // 2, 2)
+ return (pairs[..., 0] | (pairs[..., 1] << 4)).contiguous()
+
+
+def unpack_nibbles(packed: torch.Tensor, logical_count: int | None = None) -> torch.Tensor:
+ """Unpack low-first nibbles along the last dimension."""
+
+ _validate_codes(packed, maximum=255)
+ if packed.ndim == 0:
+ raise ValueError("packed must have at least one dimension")
+ capacity = packed.shape[-1] * 2
+ if logical_count is not None:
+ if isinstance(logical_count, bool) or not isinstance(logical_count, int):
+ raise ValueError("logical_count must be an integer or None")
+ if logical_count < 0 or logical_count > capacity:
+ raise ValueError(f"logical_count must be between 0 and {capacity}, got {logical_count}")
+ packed = packed.to(torch.uint8)
+ unpacked = torch.stack((packed & 0x0F, packed >> 4), dim=-1).reshape(*packed.shape[:-1], capacity)
+ if logical_count is not None:
+ unpacked = unpacked[..., :logical_count]
+ return unpacked.contiguous()
diff --git a/auto_round/export/svdquant_nunchaku.py b/auto_round/export/svdquant_nunchaku.py
new file mode 100644
index 000000000..e493e9ad8
--- /dev/null
+++ b/auto_round/export/svdquant_nunchaku.py
@@ -0,0 +1,591 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from typing import Iterable, Mapping, Protocol
+
+import torch
+
+from auto_round.algorithms.transforms.svdquant.wrapper import SVDQuantLinear
+from auto_round.export.svdquant_mxfp4 import NunchakuMXFP4Packer, pack_lowrank_weight
+from auto_round.wrapper import WrapperWALayer
+
+_DEPLOYABLE_E2M1_ALIASES = frozenset({"mx_fp", "mx_fp4", "mx_fp4e2m1"})
+NUNCHAKU_WEIGHT_FILENAME = "diffusion_pytorch_model.safetensors"
+
+
+class ResidualTensorProvider(Protocol):
+ """Provides packed MXFP4 residual tensors for one logical export record."""
+
+ def tensors_for(self, record: SVDQuantExportRecord) -> Mapping[str, torch.Tensor]:
+ """Return tensors keyed by suffix, for example ``qweight`` and ``wscales``."""
+
+
+@dataclass(frozen=True)
+class SVDQuantLinearScheme:
+ """AutoRound-selected weight and activation scheme values."""
+
+ data_type: str | None
+ bits: int | None
+ group_size: int | tuple[int, int] | None
+ sym: bool | None
+ act_data_type: str | None
+ act_bits: int | None
+ act_group_size: int | tuple[int, int] | None
+ act_sym: bool | None
+ act_dynamic: bool | None
+
+
+@dataclass(frozen=True)
+class SourceLinearRecord:
+ """Logical tensors from one source ``SVDQuantLinear`` before adapter mapping."""
+
+ name: str
+ residual_weight: torch.Tensor
+ lora_down: torch.Tensor
+ lora_up: torch.Tensor
+ smooth: torch.Tensor
+ smooth_orig: torch.Tensor
+ bias: torch.Tensor | None
+ scheme: SVDQuantLinearScheme
+
+
+@dataclass(frozen=True)
+class SVDQuantExportRecord:
+ """Adapter-selected logical tensors and their exported key mapping.
+
+ Fusion adapters must reconstruct effective source weights, fuse or split those
+ weights, and recompute the low-rank decomposition at the configured output
+ rank. Nunchaku metadata stores one rank, so exact rank-sum LoRA fusion is not
+ representable by this export schema.
+ """
+
+ prefix: str
+ residual_weight: torch.Tensor
+ lora_down: torch.Tensor
+ lora_up: torch.Tensor
+ smooth: torch.Tensor
+ smooth_orig: torch.Tensor
+ bias: torch.Tensor | None
+ scheme: SVDQuantLinearScheme
+ sources: tuple[SourceLinearRecord, ...]
+ key_mapping: Mapping[str, str] = field(default_factory=dict)
+
+
+class SVDQuantModelAdapter(Protocol):
+ """Boundary for model-level mapping and runtime metadata.
+
+ ``map_modules`` receives every logical source together so architecture adapters
+ can recompose effective weights before sibling fusion or splitting. Its output
+ must retain source provenance and use the configured source rank.
+ """
+
+ def map_modules(
+ self, model: torch.nn.Module, records: Iterable[SourceLinearRecord]
+ ) -> Iterable[SVDQuantExportRecord]: ...
+
+ def metadata(self, model: torch.nn.Module, rank: int) -> Mapping[str, str]: ...
+
+ def extra_tensors(self, model: torch.nn.Module) -> Mapping[str, torch.Tensor]: ...
+
+ def validate_records(
+ self, sources: tuple[SourceLinearRecord, ...], records: tuple[SVDQuantExportRecord, ...]
+ ) -> None: ...
+
+ def validate(self, tensors: Mapping[str, torch.Tensor], metadata: Mapping[str, str]) -> None: ...
+
+
+class IdentitySVDQuantModelAdapter:
+ """Map modules unchanged and explicitly identify a generic intermediate."""
+
+ def map_modules(
+ self, model: torch.nn.Module, records: Iterable[SourceLinearRecord]
+ ) -> Iterable[SVDQuantExportRecord]:
+ return (
+ SVDQuantExportRecord(
+ prefix=record.name or "model",
+ residual_weight=record.residual_weight,
+ lora_down=record.lora_down,
+ lora_up=record.lora_up,
+ smooth=record.smooth,
+ smooth_orig=record.smooth_orig,
+ bias=record.bias,
+ scheme=record.scheme,
+ sources=(record,),
+ )
+ for record in records
+ )
+
+ def metadata(self, model: torch.nn.Module, rank: int) -> Mapping[str, str]:
+ return {"artifact_type": "generic_intermediate"}
+
+ def extra_tensors(self, model: torch.nn.Module) -> Mapping[str, torch.Tensor]:
+ return {}
+
+ def validate_records(
+ self, sources: tuple[SourceLinearRecord, ...], records: tuple[SVDQuantExportRecord, ...]
+ ) -> None:
+ return
+
+ def validate(self, tensors: Mapping[str, torch.Tensor], metadata: Mapping[str, str]) -> None:
+ if metadata.get("artifact_type") != "generic_intermediate":
+ raise ValueError("identity adapter output must be marked as a generic intermediate")
+
+
+@dataclass
+class SVDQuantExportConfig:
+ """Strict configuration for SVDQuant Nunchaku serialization."""
+
+ weight_dtype: str = "fp4_e2m1_all"
+ activation_dtype: str = "fp4_e2m1_all"
+ scale_dtype: str = "ue8m0"
+ group_size: int = 32
+ low_rank_dtype: torch.dtype = torch.bfloat16
+ debug_unpacked: bool = False
+ runtime_loadable: bool = False
+
+ def __post_init__(self) -> None:
+ if self.weight_dtype != "fp4_e2m1_all":
+ raise ValueError("weight_dtype must be 'fp4_e2m1_all'")
+ if self.activation_dtype != "fp4_e2m1_all":
+ raise ValueError("activation_dtype must be 'fp4_e2m1_all'")
+ if self.scale_dtype != "ue8m0":
+ raise ValueError("scale_dtype must be 'ue8m0'")
+ if isinstance(self.group_size, bool) or not isinstance(self.group_size, int) or self.group_size != 32:
+ raise ValueError("group_size must be 32")
+ if self.low_rank_dtype not in (torch.bfloat16, torch.float16):
+ raise ValueError("low_rank_dtype must be torch.bfloat16 or torch.float16")
+ if not isinstance(self.debug_unpacked, bool):
+ raise ValueError("debug_unpacked must be a bool")
+ if not isinstance(self.runtime_loadable, bool):
+ raise ValueError("runtime_loadable must be a bool")
+
+ def to_quantization_config(self) -> dict:
+ config = {
+ "method": "svdquant",
+ "weight": {
+ "dtype": self.weight_dtype,
+ "scale_dtype": self.scale_dtype,
+ "group_size": self.group_size,
+ },
+ "activation": {
+ "dtype": self.activation_dtype,
+ "scale_dtype": self.scale_dtype,
+ "group_size": self.group_size,
+ },
+ }
+ return config
+
+
+def pack_nunchaku_16bit_vector(vector: torch.Tensor) -> torch.Tensor:
+ """Pack and pad one BF16/FP16 vector like Nunchaku's scalar scale path."""
+
+ if not isinstance(vector, torch.Tensor) or vector.ndim != 1:
+ raise ValueError("vector must be a 1D torch.Tensor")
+ if vector.dtype not in (torch.bfloat16, torch.float16):
+ raise ValueError("vector dtype must be torch.bfloat16 or torch.float16")
+ if vector.numel() == 0:
+ raise ValueError("vector must be non-empty")
+ if not bool(torch.isfinite(vector).all()):
+ raise ValueError("vector must contain only finite values")
+ padded_size = NunchakuMXFP4Packer._ceil_to(vector.numel(), 128)
+ padded = torch.ones(padded_size, dtype=vector.dtype, device=vector.device)
+ padded[: vector.numel()] = vector
+ packed = padded.reshape(padded_size // 128, 1, 8, 2, 4, 2, 1)
+ return packed.permute(0, 6, 1, 2, 4, 3, 5).contiguous().view(-1)
+
+
+def unpack_nunchaku_16bit_vector(vector: torch.Tensor) -> torch.Tensor:
+ """Invert :func:`pack_nunchaku_16bit_vector`, retaining padding."""
+
+ if not isinstance(vector, torch.Tensor) or vector.ndim != 1:
+ raise ValueError("vector must be a 1D torch.Tensor")
+ if vector.dtype not in (torch.bfloat16, torch.float16):
+ raise ValueError("vector dtype must be torch.bfloat16 or torch.float16")
+ if vector.numel() == 0 or vector.numel() % 128:
+ raise ValueError("packed vector length must be a non-zero multiple of 128")
+ unpacked = vector.reshape(vector.numel() // 128, 1, 1, 8, 4, 2, 2)
+ return unpacked.permute(0, 2, 3, 5, 4, 6, 1).contiguous().view(-1)
+
+
+class MXFP4ResidualTensorProvider:
+ """Quantize and physically pack residual weights as E2M1 with UE8M0 scales."""
+
+ def __init__(self, group_size: int = 32) -> None:
+ if isinstance(group_size, bool) or not isinstance(group_size, int) or group_size != 32:
+ raise ValueError("group_size must be 32")
+ self.group_size = group_size
+ self.packer = NunchakuMXFP4Packer()
+
+ def tensors_for(self, record: SVDQuantExportRecord) -> Mapping[str, torch.Tensor]:
+ weight = record.residual_weight
+ if not bool(torch.isfinite(weight).all()):
+ raise ValueError(f"{record.prefix or '<root>'} residual weight must contain only finite values")
+ packed = self.packer.pack_residual(weight, group_size=self.group_size)
+ return {"qweight": packed.qweight, "wscales": packed.wscales}
+
+
+def _source_records(model: torch.nn.Module) -> tuple[SourceLinearRecord, ...]:
+ records = []
+ for name, module in model.named_modules():
+ if not isinstance(module, SVDQuantLinear):
+ continue
+ residual_linear = module.residual_linear
+ while isinstance(residual_linear, WrapperWALayer):
+ residual_linear = residual_linear.orig_layer
+ records.append(
+ SourceLinearRecord(
+ name=name,
+ residual_weight=residual_linear.weight.detach(),
+ lora_down=module.lora_down.weight.detach(),
+ lora_up=module.lora_up.weight.detach(),
+ smooth=module.smooth.detach(),
+ smooth_orig=getattr(module, "smooth_orig", module.smooth).detach(),
+ bias=None if residual_linear.bias is None else residual_linear.bias.detach(),
+ scheme=SVDQuantLinearScheme(
+ data_type=getattr(residual_linear, "data_type", None),
+ bits=getattr(residual_linear, "bits", None),
+ group_size=getattr(residual_linear, "group_size", None),
+ sym=getattr(residual_linear, "sym", None),
+ act_data_type=getattr(residual_linear, "act_data_type", None),
+ act_bits=getattr(residual_linear, "act_bits", None),
+ act_group_size=getattr(residual_linear, "act_group_size", None),
+ act_sym=getattr(residual_linear, "act_sym", None),
+ act_dynamic=getattr(residual_linear, "act_dynamic", None),
+ ),
+ )
+ )
+ if not records:
+ raise ValueError("No SVDQuantLinear modules found to export.")
+ return tuple(records)
+
+
+def _validate_selected_scheme(scheme: SVDQuantLinearScheme, prefix: str) -> tuple:
+ required_weight = {
+ "data_type": scheme.data_type,
+ "bits": scheme.bits,
+ "group_size": scheme.group_size,
+ "sym": scheme.sym,
+ }
+ missing_weight = [name for name, value in required_weight.items() if value is None]
+ if missing_weight:
+ raise ValueError(f"{prefix} selected residual scheme is missing required {missing_weight[0]}")
+ weight_data_type = scheme.data_type.removesuffix("_rceil") if isinstance(scheme.data_type, str) else None
+ if weight_data_type not in _DEPLOYABLE_E2M1_ALIASES:
+ raise ValueError(
+ f"{prefix} residual data_type must be one of {sorted(_DEPLOYABLE_E2M1_ALIASES)}, got {scheme.data_type!r}"
+ )
+ if isinstance(scheme.bits, bool) or not isinstance(scheme.bits, int) or scheme.bits != 4:
+ raise ValueError(f"{prefix} residual scheme requires bits=4, got {scheme.bits!r}")
+ if isinstance(scheme.group_size, bool) or not isinstance(scheme.group_size, int) or scheme.group_size != 32:
+ raise ValueError(f"{prefix} residual scheme requires scalar group_size=32, got {scheme.group_size!r}")
+ if scheme.sym is not True:
+ raise ValueError(f"{prefix} residual scheme requires sym=True, got {scheme.sym!r}")
+
+ required_activation = {
+ "act_data_type": scheme.act_data_type,
+ "act_bits": scheme.act_bits,
+ "act_group_size": scheme.act_group_size,
+ "act_sym": scheme.act_sym,
+ "act_dynamic": scheme.act_dynamic,
+ }
+ missing_activation = [name for name, value in required_activation.items() if value is None]
+ if missing_activation:
+ raise ValueError(f"{prefix} selected scheme is missing required activation value {missing_activation[0]}")
+ activation_data_type = (
+ scheme.act_data_type.removesuffix("_rceil") if isinstance(scheme.act_data_type, str) else None
+ )
+ if activation_data_type not in _DEPLOYABLE_E2M1_ALIASES:
+ raise ValueError(
+ f"{prefix} activation data_type must be one of {sorted(_DEPLOYABLE_E2M1_ALIASES)}, "
+ f"got {scheme.act_data_type!r}"
+ )
+ if isinstance(scheme.act_bits, bool) or not isinstance(scheme.act_bits, int) or scheme.act_bits != 4:
+ raise ValueError(f"{prefix} activation scheme requires activation bits=4, got {scheme.act_bits!r}")
+ if (
+ isinstance(scheme.act_group_size, bool)
+ or not isinstance(scheme.act_group_size, int)
+ or scheme.act_group_size != 32
+ ):
+ raise ValueError(
+ f"{prefix} activation scheme requires activation scalar group_size=32, got {scheme.act_group_size!r}"
+ )
+ if scheme.act_sym is not True:
+ raise ValueError(f"{prefix} activation scheme requires activation sym=True, got {scheme.act_sym!r}")
+ if scheme.act_dynamic is not True:
+ raise ValueError(f"{prefix} activation scheme requires act_dynamic=True, got {scheme.act_dynamic!r}")
+ return ("mx_fp4e2m1", 4, 32, True, "mx_fp4e2m1", 4, 32, True, True)
+
+
+def _validate_export_record(record: SVDQuantExportRecord, config: SVDQuantExportConfig) -> int:
+ tensors = (record.residual_weight, record.lora_down, record.lora_up, record.smooth, record.smooth_orig)
+ if not record.prefix or not isinstance(record.prefix, str):
+ raise ValueError("export record prefix must be a non-empty string")
+ if any(not isinstance(tensor, torch.Tensor) or not tensor.is_floating_point() for tensor in tensors):
+ raise ValueError(f"{record.prefix} logical tensors must be floating-point torch.Tensor values")
+ if record.residual_weight.ndim != 2 or record.lora_down.ndim != 2 or record.lora_up.ndim != 2:
+ raise ValueError(f"{record.prefix} residual and low-rank tensors must be 2D")
+ out_features, in_features = record.residual_weight.shape
+ rank = record.lora_down.shape[0]
+ if (
+ record.lora_down.shape[1] != in_features
+ or record.lora_up.shape != (out_features, rank)
+ or record.smooth.shape != (in_features,)
+ or record.smooth_orig.shape != (in_features,)
+ or (record.bias is not None and record.bias.shape != (out_features,))
+ ):
+ raise ValueError(f"{record.prefix} logical tensor shapes are inconsistent")
+ finite_tensors = tensors + (() if record.bias is None else (record.bias,))
+ if any(not bool(torch.isfinite(tensor).all()) for tensor in finite_tensors):
+ raise ValueError(f"{record.prefix} logical tensors must contain only finite values")
+ if rank <= 0:
+ raise ValueError(f"{record.prefix} rank must be positive")
+ selected_scheme = _validate_selected_scheme(record.scheme, record.prefix)
+ if not record.sources or any(not isinstance(source, SourceLinearRecord) for source in record.sources):
+ raise ValueError(f"{record.prefix} export record must retain at least one logical source record")
+ source_schemes = {_validate_selected_scheme(source.scheme, source.name) for source in record.sources}
+ if len(source_schemes) != 1 or selected_scheme not in source_schemes:
+ raise ValueError(f"{record.prefix} adapter sources have incompatible selected quantization schemes")
+ if config.group_size != selected_scheme[2]:
+ raise ValueError(
+ f"{record.prefix} export group_size={config.group_size} disagrees with "
+ f"selected group_size={selected_scheme[2]}"
+ )
+ return rank
+
+
+def _export_key(record: SVDQuantExportRecord, suffix: str) -> str:
+ mapped_suffix = record.key_mapping.get(suffix, suffix)
+ if not isinstance(mapped_suffix, str) or not mapped_suffix:
+ raise ValueError(f"{record.prefix} key mapping for {suffix!r} must be a non-empty string")
+ return f"{record.prefix}.{mapped_suffix}"
+
+
+def _validate_packed_residual(payload: Mapping[str, torch.Tensor], record: SVDQuantExportRecord) -> None:
+ prefix = record.prefix
+ if set(payload) != {"qweight", "wscales"}:
+ raise ValueError(f"{prefix} packed residual must contain exactly qweight and wscales")
+ qweight, wscales = payload["qweight"], payload["wscales"]
+ if not isinstance(qweight, torch.Tensor) or qweight.dtype != torch.int8 or qweight.ndim != 2:
+ raise ValueError(f"{prefix} qweight must be a 2D torch.int8 tensor")
+ out_features, in_features = record.residual_weight.shape
+ padded_out = NunchakuMXFP4Packer._ceil_to(out_features, 128)
+ padded_in = NunchakuMXFP4Packer._ceil_to(in_features, 128)
+ expected_weight_shape = (padded_out, padded_in // 2)
+ if tuple(qweight.shape) != expected_weight_shape:
+ raise ValueError(f"{prefix} qweight shape must be {expected_weight_shape}")
+ expected_scale_shape = (padded_in // 32, padded_out)
+ if not isinstance(wscales, torch.Tensor) or wscales.dtype != torch.uint8:
+ raise ValueError(f"{prefix} wscales must be a torch.uint8 tensor")
+ if tuple(wscales.shape) != expected_scale_shape:
+ raise ValueError(f"{prefix} wscales shape must be {expected_scale_shape}")
+
+
+def _validate_adapter_provenance(
+ sources: tuple[SourceLinearRecord, ...], records: tuple[SVDQuantExportRecord, ...]
+) -> None:
+ """Require complete source identity coverage and configured-rank outputs."""
+
+ source_ids = {id(source) for source in sources}
+ referenced_ids: set[int] = set()
+ for record in records:
+ output_rank = record.lora_down.shape[0]
+ for source in record.sources:
+ if id(source) not in source_ids:
+ raise ValueError(f"{record.prefix} adapter record references a foreign logical source")
+ if source.lora_down.shape[0] != output_rank:
+ raise ValueError(
+ f"{record.prefix} source rank={source.lora_down.shape[0]} must equal output rank={output_rank}; "
+ "the Nunchaku schema stores one configured rank, so adapters must recompose effective source "
+ "weights and decompose them at that configured rank; exact rank-sum fusion is unsupported"
+ )
+ referenced_ids.add(id(source))
+ missing = [source.name or "<root>" for source in sources if id(source) not in referenced_ids]
+ if missing:
+ raise ValueError(f"model adapter dropped logical sources: {missing}")
+
+
+def _prepare_export_records(
+ model: torch.nn.Module,
+ config: SVDQuantExportConfig,
+ adapter: SVDQuantModelAdapter,
+) -> tuple[tuple[SourceLinearRecord, ...], tuple[SVDQuantExportRecord, ...], int]:
+ source_records = _source_records(model)
+ for source in source_records:
+ _validate_selected_scheme(source.scheme, source.name or "<root>")
+ records = tuple(adapter.map_modules(model, source_records))
+ if not records:
+ raise ValueError("model adapter produced no SVDQuant export records")
+ ranks = {_validate_export_record(record, config) for record in records}
+ if len(ranks) != 1:
+ raise ValueError(f"mixed SVDQuant ranks are not supported: {sorted(ranks)}")
+ _validate_adapter_provenance(source_records, records)
+ validate_records = getattr(adapter, "validate_records", None)
+ if validate_records is not None:
+ validate_records(source_records, records)
+ return source_records, records, ranks.pop()
+
+
+def _serialize_export_records(
+ records: tuple[SVDQuantExportRecord, ...],
+ config: SVDQuantExportConfig,
+ residual_provider: ResidualTensorProvider,
+) -> dict[str, torch.Tensor]:
+ tensors: dict[str, torch.Tensor] = {}
+ for record in records:
+ if not bool((record.smooth > 0).all()) or not bool((record.smooth_orig > 0).all()):
+ raise ValueError(f"{record.prefix} smooth tensors must contain only positive values")
+
+ # SVDQuantLinear multiplies both branches' input by ``smooth``. Nunchaku
+ # divides only the residual input by its smooth factor and applies the
+ # low-rank branch to the original input, so convert between coordinates.
+ runtime_smooth = record.smooth.to(torch.float64).reciprocal()
+ runtime_smooth_orig = record.smooth_orig.to(torch.float64).reciprocal()
+ runtime_lora_down = record.lora_down.to(torch.float64) * record.smooth.to(torch.float64).unsqueeze(0)
+ high_precision = {
+ "smooth": pack_nunchaku_16bit_vector(runtime_smooth.to(config.low_rank_dtype)),
+ "smooth_orig": pack_nunchaku_16bit_vector(runtime_smooth_orig.to(config.low_rank_dtype)),
+ "lora_down": pack_lowrank_weight(runtime_lora_down.to(config.low_rank_dtype), down=True),
+ "lora_up": pack_lowrank_weight(record.lora_up.to(config.low_rank_dtype), down=False),
+ "bias": pack_nunchaku_16bit_vector(
+ torch.zeros(
+ record.residual_weight.shape[0],
+ dtype=config.low_rank_dtype,
+ device=record.residual_weight.device,
+ )
+ if record.bias is None
+ else record.bias.to(config.low_rank_dtype)
+ ),
+ }
+ residual_payload = residual_provider.tensors_for(record)
+ _validate_packed_residual(residual_payload, record)
+ serialized = {**high_precision, **residual_payload}
+ if config.debug_unpacked:
+ serialized["residual.weight"] = record.residual_weight
+ for suffix, tensor in serialized.items():
+ key = _export_key(record, suffix)
+ if key in tensors:
+ raise ValueError(f"model adapter produced duplicate tensor key {key!r}")
+ tensors[key] = tensor.detach().cpu().contiguous()
+ return tensors
+
+
+def _merge_adapter_tensors(
+ tensors: dict[str, torch.Tensor], adapter: SVDQuantModelAdapter, model: torch.nn.Module
+) -> dict[str, torch.Tensor]:
+ extra_tensors = getattr(adapter, "extra_tensors", None)
+ extras = {} if extra_tensors is None else extra_tensors(model)
+ if not isinstance(extras, Mapping):
+ raise ValueError("model adapter extra_tensors must return a mapping")
+ for key, tensor in extras.items():
+ if not isinstance(key, str) or not key:
+ raise ValueError("model adapter extra tensor keys must be non-empty strings")
+ if key in tensors:
+ raise ValueError(f"model adapter produced duplicate tensor key {key!r}")
+ if not isinstance(tensor, torch.Tensor):
+ raise ValueError(f"model adapter extra tensor {key!r} must be a torch.Tensor")
+ if tensor.layout != torch.strided or tensor.is_quantized:
+ raise ValueError(f"model adapter extra tensor {key!r} must be a dense strided tensor")
+ if tensor.is_complex() or tensor.dtype == torch.bool:
+ raise ValueError(f"model adapter extra tensor {key!r} has unsupported dtype {tensor.dtype}")
+ if (tensor.is_floating_point() or tensor.is_complex()) and not bool(torch.isfinite(tensor).all()):
+ raise ValueError(f"model adapter extra tensor {key!r} must contain only finite values")
+ tensors[key] = tensor.detach().cpu().contiguous()
+ return tensors
+
+
+def collect_svdquant_tensors(
+ model: torch.nn.Module,
+ *,
+ config: SVDQuantExportConfig | None = None,
+ residual_provider: ResidualTensorProvider | None = None,
+ adapter: SVDQuantModelAdapter | None = None,
+) -> dict[str, torch.Tensor]:
+ """Collect generic SVDQuant tensors from all ``SVDQuantLinear`` modules."""
+
+ config = config or SVDQuantExportConfig()
+ residual_provider = residual_provider or MXFP4ResidualTensorProvider(config.group_size)
+ adapter = adapter or IdentitySVDQuantModelAdapter()
+ _, records, rank = _prepare_export_records(model, config, adapter)
+ if config.runtime_loadable:
+ build_svdquant_metadata(model, config=config, adapter=adapter, rank=rank)
+ tensors = _serialize_export_records(records, config, residual_provider)
+ return _merge_adapter_tensors(tensors, adapter, model)
+
+
+def build_svdquant_metadata(
+ model: torch.nn.Module,
+ *,
+ config: SVDQuantExportConfig | None = None,
+ adapter: SVDQuantModelAdapter | None = None,
+ rank: int | None = None,
+) -> dict[str, str]:
+ """Build validated string metadata for one SVDQuant artifact."""
+
+ config = config or SVDQuantExportConfig()
+ adapter = adapter or IdentitySVDQuantModelAdapter()
+ if rank is None:
+ _, _, rank = _prepare_export_records(model, config, adapter)
+ quantization_config = config.to_quantization_config()
+ quantization_config["rank"] = rank
+ metadata = dict(adapter.metadata(model, rank))
+ metadata["quantization_config"] = json.dumps(quantization_config, sort_keys=True)
+ if not metadata or any(not isinstance(key, str) or not isinstance(value, str) for key, value in metadata.items()):
+ raise ValueError("safetensors metadata keys and values must be strings")
+ if config.runtime_loadable:
+ if config.debug_unpacked:
+ raise ValueError("debug_unpacked output is not runtime-loadable")
+ if not metadata.get("model_class") or not metadata.get("config"):
+ raise ValueError("runtime-loadable export requires adapter metadata 'model_class' and serialized 'config'")
+ try:
+ serialized_config = json.loads(metadata["config"])
+ except (TypeError, json.JSONDecodeError) as exc:
+ raise ValueError("adapter metadata 'config' must be serialized JSON") from exc
+ if not isinstance(serialized_config, dict):
+ raise ValueError("adapter metadata 'config' must serialize a JSON object")
+ return metadata
+
+
+def save_svdquant_nunchaku_safetensors(
+ model: torch.nn.Module,
+ output_path: str,
+ *,
+ config: SVDQuantExportConfig | None = None,
+ residual_provider: ResidualTensorProvider | None = None,
+ adapter: SVDQuantModelAdapter | None = None,
+) -> str:
+ """Save decomposed SVDQuant tensors to one safetensors file.
+
+ This function is model-family agnostic. It exports every ``SVDQuantLinear``
+ it finds and does not import inference runtimes.
+ """
+
+ from safetensors.torch import save_file
+
+ config = config or SVDQuantExportConfig()
+ adapter = adapter or IdentitySVDQuantModelAdapter()
+ residual_provider = residual_provider or MXFP4ResidualTensorProvider(config.group_size)
+ _, records, rank = _prepare_export_records(model, config, adapter)
+ metadata = build_svdquant_metadata(model, config=config, adapter=adapter, rank=rank)
+ tensors = _serialize_export_records(records, config, residual_provider)
+ tensors = _merge_adapter_tensors(tensors, adapter, model)
+ adapter.validate(tensors, metadata)
+
+ os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True)
+ save_file(tensors, output_path, metadata=metadata)
+ return output_path
diff --git a/auto_round/export/svdquant_w4a16.py b/auto_round/export/svdquant_w4a16.py
new file mode 100644
index 000000000..b82a2dfb4
--- /dev/null
+++ b/auto_round/export/svdquant_w4a16.py
@@ -0,0 +1,314 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""AutoRound-owned Nunchaku W4A16 AdaNorm tensor codec."""
+
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass
+
+import torch
+
+
+@dataclass(frozen=True)
+class PackedW4A16:
+ """Physical W4A16 tensors and the metadata needed to invert their layout."""
+
+ qweight: torch.Tensor
+ wscales: torch.Tensor
+ wzeros: torch.Tensor
+ bias: torch.Tensor
+ dtype: torch.dtype
+ logical_shape: tuple[int, int]
+ splits: int
+ group_size: int
+
+
+def _require_little_endian() -> None:
+ if sys.byteorder != "little":
+ raise ValueError("Nunchaku W4A16 packing requires a little-endian host")
+
+
+def _pack_code_rows(codes: torch.Tensor) -> torch.Tensor:
+ rows, in_features = codes.shape
+ packed16 = codes.view(-1, 4, 8)
+ packed16 = packed16[:, 0] | (packed16[:, 1] << 4) | (packed16[:, 2] << 8) | (packed16[:, 3] << 12)
+ packed16 = (
+ packed16.view(rows // 4, 4, in_features // 64, 16)
+ .permute(0, 2, 1, 3)
+ .reshape(rows // 4, in_features)
+ .to(torch.int16)
+ )
+ return packed16.view(torch.int32)
+
+
+def _effective_chunk_rows(chunk_rows: int | None, out_features: int) -> int:
+ if chunk_rows is None:
+ return out_features
+ if isinstance(chunk_rows, bool) or not isinstance(chunk_rows, int) or chunk_rows <= 0 or chunk_rows % 4:
+ raise ValueError("chunk_rows must be None or a positive multiple of 4")
+ return min(chunk_rows, out_features)
+
+
+def _validate_inputs(
+ weight: torch.Tensor,
+ scale: torch.Tensor,
+ bias: torch.Tensor | None,
+ splits: int,
+ group_size: int,
+) -> tuple[int, int, int, torch.Tensor]:
+ if (
+ not isinstance(weight, torch.Tensor)
+ or weight.ndim != 2
+ or weight.dtype
+ not in (
+ torch.bfloat16,
+ torch.float16,
+ )
+ ):
+ raise ValueError("weight must be a BF16 or FP16 tensor with shape [O, K]")
+ if not bool(torch.isfinite(weight).all()):
+ raise ValueError("weight must contain only finite values")
+ if isinstance(group_size, bool) or not isinstance(group_size, int) or group_size != 64:
+ raise ValueError("group_size must be 64")
+ if isinstance(splits, bool) or not isinstance(splits, int) or splits not in (3, 6):
+ raise ValueError("splits must be 3 or 6")
+ out_features, in_features = weight.shape
+ if out_features == 0 or in_features == 0:
+ raise ValueError("weight dimensions must be non-empty")
+ if in_features % group_size:
+ raise ValueError("K must be divisible by group_size 64")
+ if out_features % splits:
+ raise ValueError("O must be divisible by splits")
+ if out_features % 4:
+ raise ValueError("O must be divisible by 4")
+ num_groups = in_features // group_size
+ if num_groups % 16:
+ raise ValueError("runtime G=K/64 must be divisible by 16")
+ if not isinstance(scale, torch.Tensor) or not scale.is_floating_point():
+ raise ValueError("scale must be a floating-point tensor")
+ if scale.dtype != weight.dtype or scale.dtype not in (torch.bfloat16, torch.float16):
+ raise ValueError("scale dtype must exactly match weight dtype and be BF16 or FP16")
+ if scale.device != weight.device:
+ raise ValueError("scale must be on the same device as weight")
+ if tuple(scale.shape) not in ((out_features, num_groups), (out_features, 1, num_groups, 1)):
+ raise ValueError(f"scale shape must be [O, G] or [O, 1, G, 1], got {tuple(scale.shape)}")
+ if not bool(torch.isfinite(scale).all()) or not bool((scale > 0).all()):
+ raise ValueError("scale must contain only positive finite values")
+ if bias is not None:
+ if not isinstance(bias, torch.Tensor) or not bias.is_floating_point():
+ raise ValueError("bias must be a floating-point tensor")
+ if bias.dtype != weight.dtype or bias.dtype not in (torch.bfloat16, torch.float16):
+ raise ValueError("bias dtype must exactly match weight dtype and be BF16 or FP16")
+ if tuple(bias.shape) != (out_features,) or bias.device != weight.device or not bool(torch.isfinite(bias).all()):
+ raise ValueError("bias must be a finite floating-point [O] tensor on the weight device")
+ return out_features, in_features, num_groups, scale.reshape(out_features, num_groups)
+
+
+def pack_adanorm_w4a16(
+ weight: torch.Tensor,
+ scale: torch.Tensor,
+ bias: torch.Tensor | None = None,
+ splits: int = 3,
+ group_size: int = 64,
+ chunk_rows: int | None = 256,
+) -> PackedW4A16:
+ """Pack pre-scaled signed INT4 AdaNorm weights for Nunchaku W4A16."""
+
+ _require_little_endian()
+ out_features, in_features, num_groups, logical_scale = _validate_inputs(weight, scale, bias, splits, group_size)
+ rows_per_chunk = _effective_chunk_rows(chunk_rows, out_features)
+ channels_per_field = out_features // splits
+ qweight = torch.empty((out_features // 4, in_features // 2), dtype=torch.int32, device=weight.device)
+ channel_scale = torch.empty((out_features, num_groups), dtype=weight.dtype, device=weight.device)
+ channel_bias = torch.zeros(out_features, dtype=weight.dtype, device=weight.device)
+ for start in range(0, out_features, rows_per_chunk):
+ end = min(start + rows_per_chunk, out_features)
+ output_rows = torch.arange(start, end, device=weight.device)
+ source_rows = output_rows.remainder(splits) * channels_per_field + torch.div(
+ output_rows, splits, rounding_mode="floor"
+ )
+ weight_rows = weight.index_select(0, source_rows)
+ scale_rows = logical_scale.index_select(0, source_rows)
+ normalized = weight_rows.float().reshape(end - start, num_groups, group_size)
+ normalized.div_(scale_rows.float().unsqueeze(-1)).round_()
+ if not bool(((normalized >= -7) & (normalized <= 7)).all()):
+ raise ValueError("quantized weight must be in [-7, 7]")
+ codes = normalized.reshape(end - start, in_features).to(torch.int32).add_(7)
+ qweight[start // 4 : end // 4] = _pack_code_rows(codes)
+ channel_scale[start:end] = scale_rows
+ if bias is not None:
+ channel_bias[start:end] = bias.index_select(0, source_rows)
+ del codes, normalized, scale_rows, weight_rows
+ channel_bias = channel_bias.reshape(out_features // splits, splits)
+ identity_fields = sorted({1, splits - 2})
+ identity_before = channel_bias[:, identity_fields].clone()
+ channel_bias[:, identity_fields] += 1
+ if bool((channel_bias[:, identity_fields] == identity_before).any()):
+ raise ValueError(f"AdaNorm bias identity offset +1 must change the stored {weight.dtype} value")
+ channel_bias = channel_bias.reshape(out_features)
+ wscales = channel_scale.t().contiguous()
+ wzeros = (-7 * channel_scale).t().contiguous()
+ for name, tensor in (("wscales", wscales), ("wzeros", wzeros), ("bias", channel_bias)):
+ if not bool(torch.isfinite(tensor).all()):
+ raise ValueError(f"{name} must remain finite in {weight.dtype} after packing arithmetic")
+ return PackedW4A16(
+ qweight=qweight,
+ wscales=wscales,
+ wzeros=wzeros,
+ bias=channel_bias,
+ dtype=weight.dtype,
+ logical_shape=(out_features, in_features),
+ splits=splits,
+ group_size=group_size,
+ )
+
+
+def _validate_packed_w4a16(packed: PackedW4A16) -> tuple[int, int, int]:
+ _require_little_endian()
+ if not isinstance(packed, PackedW4A16):
+ raise ValueError("packed must be a PackedW4A16 payload")
+ if packed.dtype not in (torch.bfloat16, torch.float16):
+ raise ValueError("payload dtype must be torch.bfloat16 or torch.float16")
+ if (
+ not isinstance(packed.logical_shape, tuple)
+ or len(packed.logical_shape) != 2
+ or any(isinstance(value, bool) or not isinstance(value, int) or value <= 0 for value in packed.logical_shape)
+ ):
+ raise ValueError("logical_shape must be a positive (O, K) integer tuple")
+ out_features, in_features = packed.logical_shape
+ if isinstance(packed.splits, bool) or not isinstance(packed.splits, int) or packed.splits not in (3, 6):
+ raise ValueError("splits must be 3 or 6")
+ if isinstance(packed.group_size, bool) or not isinstance(packed.group_size, int) or packed.group_size != 64:
+ raise ValueError("group_size must be 64")
+ if out_features % packed.splits or out_features % 4 or in_features % packed.group_size:
+ raise ValueError("logical_shape is incompatible with splits, output packing, or group_size")
+ num_groups = in_features // packed.group_size
+ if num_groups % 16:
+ raise ValueError("logical_shape must produce a runtime group count divisible by 16")
+ if not isinstance(packed.qweight, torch.Tensor) or packed.qweight.dtype != torch.int32:
+ raise ValueError("qweight dtype must be torch.int32")
+ if tuple(packed.qweight.shape) != (out_features // 4, in_features // 2):
+ raise ValueError("qweight shape is inconsistent with logical_shape")
+ expected_shapes = {
+ "wscales": (num_groups, out_features),
+ "wzeros": (num_groups, out_features),
+ "bias": (out_features,),
+ }
+ for name, expected_shape in expected_shapes.items():
+ tensor = getattr(packed, name)
+ if not isinstance(tensor, torch.Tensor) or tensor.dtype != packed.dtype:
+ raise ValueError(f"{name} dtype must exactly match payload dtype")
+ if tuple(tensor.shape) != expected_shape:
+ raise ValueError(f"{name} shape must be {expected_shape}")
+ if tensor.device != packed.qweight.device:
+ raise ValueError("all packed tensors must be on the same device")
+ for name in ("wscales", "wzeros", "bias"):
+ if not bool(torch.isfinite(getattr(packed, name)).all()):
+ raise ValueError(f"{name} must contain only finite values")
+ if not bool((packed.wscales > 0).all()):
+ raise ValueError("wscales must contain only positive values")
+ if not torch.equal(packed.wzeros, packed.wscales * -7):
+ raise ValueError("wzeros must equal -7 * wscales in payload dtype")
+ return out_features, in_features, num_groups
+
+
+def unpack_adanorm_w4a16(packed: PackedW4A16) -> torch.Tensor:
+ """Recover channel-major signed INT4 values from a packed payload."""
+
+ out_features, in_features, _ = _validate_packed_w4a16(packed)
+ packed16 = packed.qweight.contiguous().view(torch.int16).reshape(out_features // 4, in_features)
+ packed16 = packed16.view(out_features // 4, in_features // 64, 4, 16).permute(0, 2, 1, 3).contiguous().view(-1, 8)
+ shifts = torch.arange(0, 16, 4, dtype=torch.int16, device=packed.qweight.device)
+ codes = ((packed16.unsqueeze(1) >> shifts.view(1, 4, 1)) & 0xF).reshape(out_features, in_features)
+ if not bool((codes <= 14).all()):
+ raise ValueError("qweight contains codes outside [0, 14]")
+ return (codes - 7).to(torch.int8)
+
+
+def dequantize_adanorm_w4a16(packed: PackedW4A16) -> torch.Tensor:
+ """Decode the packed weights into their QDQ values in runtime channel order."""
+
+ signed = unpack_adanorm_w4a16(packed).to(torch.float32)
+ out_features, in_features = signed.shape
+ scales = packed.wscales.t().reshape(out_features, in_features // packed.group_size, 1).float()
+ return (signed.reshape(out_features, -1, packed.group_size) * scales).reshape_as(signed).to(packed.dtype)
+
+
+def _rtn_scale_bounds(dtype: torch.dtype, device: torch.device) -> tuple[torch.Tensor, torch.Tensor]:
+ zero = torch.tensor(0, dtype=dtype, device=device)
+ smallest = torch.nextafter(zero, torch.tensor(1, dtype=dtype, device=device))
+ largest = torch.tensor(torch.finfo(dtype).max / 7, dtype=dtype, device=device)
+ while not bool(torch.isfinite(largest * -7)):
+ largest = torch.nextafter(largest, zero)
+ return smallest, largest
+
+
+def quantize_adanorm_w4a16_rtn(
+ weight: torch.Tensor,
+ bias: torch.Tensor | None = None,
+ splits: int = 3,
+ group_size: int = 64,
+ chunk_rows: int | None = 256,
+) -> PackedW4A16:
+ """Compute symmetric per-row-group RTN scales and pack the result.
+
+ An all-zero group uses scale one. Its signed values and QDQ values remain
+ exactly zero, while the emitted scale continues to satisfy the runtime's
+ positive-scale contract.
+ """
+
+ _require_little_endian()
+ if (
+ not isinstance(weight, torch.Tensor)
+ or weight.ndim != 2
+ or weight.dtype
+ not in (
+ torch.bfloat16,
+ torch.float16,
+ )
+ ):
+ raise ValueError("weight must be a BF16 or FP16 tensor with shape [O, K]")
+ if isinstance(group_size, bool) or not isinstance(group_size, int) or group_size != 64:
+ raise ValueError("group_size must be 64")
+ out_features, in_features = weight.shape
+ if out_features == 0 or in_features == 0:
+ raise ValueError("weight dimensions must be non-empty")
+ if in_features % group_size:
+ raise ValueError("K must be divisible by group_size 64")
+ rows_per_chunk = _effective_chunk_rows(chunk_rows, out_features)
+ num_groups = in_features // group_size
+ scales = torch.empty((out_features, num_groups), dtype=weight.dtype, device=weight.device)
+ smallest_scale, largest_scale = _rtn_scale_bounds(weight.dtype, weight.device)
+ for start in range(0, out_features, rows_per_chunk):
+ end = min(start + rows_per_chunk, out_features)
+ grouped = weight[start:end].float().reshape(end - start, num_groups, group_size)
+ absmax = grouped.abs().amax(dim=-1)
+ chunk_scales = torch.where(absmax == 0, torch.ones_like(absmax), absmax / 7).to(weight.dtype)
+ chunk_scales = torch.where((absmax > 0) & (chunk_scales <= 0), smallest_scale, chunk_scales)
+ scales[start:end] = torch.minimum(chunk_scales, largest_scale)
+ try:
+ return pack_adanorm_w4a16(
+ weight,
+ scales,
+ bias=bias,
+ splits=splits,
+ group_size=group_size,
+ chunk_rows=chunk_rows,
+ )
+ except ValueError as exc:
+ if "quantized weight must be in [-7, 7]" in str(exc):
+ raise ValueError("RTN scale cannot represent the weight in signed range [-7, 7]") from exc
+ raise
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index 4af0291a2..c23aad8eb 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -619,6 +619,7 @@ class SupportedFormats:
"auto_round:fp8",
"mlx",
"auto_round:mlx",
+ "svdquant_nunchaku",
)
self._gguf_format = tuple(sorted(GGUF_CONFIG.keys()))
self._support_list = self._support_format + self._gguf_format
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index f8cc7a276..617f59b54 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -18,7 +18,7 @@ import os
import re
from collections import UserDict
from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union
import psutil
import torch
@@ -1761,6 +1761,19 @@ def unsupported_meta_device(model):
return False
+def map_nested_tensors(value: Any, transform: Callable[[torch.Tensor], torch.Tensor]) -> Any:
+ """Apply ``transform`` to tensors in nested tuples, lists, and mappings."""
+ if torch.is_tensor(value):
+ return transform(value)
+ if isinstance(value, tuple):
+ return tuple(map_nested_tensors(item, transform) for item in value)
+ if isinstance(value, list):
+ return [map_nested_tensors(item, transform) for item in value]
+ if isinstance(value, (dict, UserDict)):
+ return type(value)({key: map_nested_tensors(item, transform) for key, item in value.items()})
+ return value
+
+
def to_device(input, device=torch.device("cpu")):
"""Moves input data to the specified device.
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index b63882653..5dc2f3719 100755
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -28,6 +28,7 @@ This document presents step-by-step instructions for auto-round llm quantization
- [Hyperparameters in AutoScheme](#hyperparameters-in-autoscheme)
+ [OPT RTN mode](#opt-rtn-mode)
+ [AWQ Algorithm-Experimental](#awq-algorithm)
+ + [SVDQuant Algorithm-Experimental](#svdquant-algorithm)
+ [Model-Free Mode](#model-free-mode)
+ [GGUF format](#gguf-format)
+ [Quantization Costs](#quantization-costs)
@@ -419,6 +420,45 @@ format selection such as `format="auto_awq"`. For example:
- `alg_configs="awq"` + `format="auto_round"`: AWQ smoothing with AutoRound packing.
- `alg_configs="signround"` + `format="auto_awq"`: AutoAWQ packing without AWQ smoothing.
+### SVDQuant Algorithm
+
+**Experimental feature: the end-to-end workflow has currently been validated with FLUX.1-dev only.**
+
+SVDQuant decomposes each Linear weight into a quantized residual branch and a small floating-point low-rank branch. It can be combined with RTN or SignRound to produce an MXFP4 model for Nunchaku inference.
+
+RTN (recommended starting point):
+
+```bash
+auto-round-rtn --model /path/to/FLUX.1-dev --model_dtype bf16 \
+ --scheme MXFP4 --algorithm svdquant --device 0 \
+ --format svdquant_nunchaku \
+ --output_dir ./flux-dev-mxfp4-svdquant-rtn
+```
+
+SignRound:
+
+```bash
+auto-round --model /path/to/FLUX.1-dev --model_dtype bf16 \
+ --scheme MXFP4 --algorithm svdquant,auto_round \
+ --format svdquant_nunchaku \
+ --dataset /path/to/captions.tsv --batch_size 1 --device 0 \
+ --output_dir ./flux-dev-mxfp4-svdquant-signround
+```
+
+FLUX.1-dev quality results using Nunchaku commit [`4de4986`](https://github.com/changwangss/nunchaku/commit/4de49869eaa8565d8c29da344323e82298bdf198):
+
+| Configuration | CLIP | CLIP-IQA | ImageReward |
+|---|---:|---:|---:|
+| BF16 | 26.0189 | 0.954360 | 1.018340 |
+| MXFP4, smooth + SVDQuant + SignRound | **26.1039** | **0.962655** | **1.021020** |
+| MXFP4, no smooth + SVDQuant + SignRound | 26.0727 | 0.959363 | 1.002380 |
+| MXFP4, smooth + SVDQuant + RTN | 25.9719 | 0.947763 | 0.939392 |
+| MXFP4, no smooth + SVDQuant + RTN | 25.9624 | 0.946939 | 0.934579 |
+
+SignRound used 128 calibration samples, 50 inference steps, 200 tuning iterations, rank 32, and 20 residual iterations.
+
+See [SVDQuant Details](./svdquant_details.md) for smooth search, residual iterations, export, and inference.
+
### AutoScheme
AutoScheme automatically generates adaptive mixed-bit and mixed-data-type quantization recipes. For accuracy results, see [AutoScheme Accuracy Report](./auto_scheme_acc.md).
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index b6341bfd0..3c4421ae0 100755
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -28,6 +28,7 @@
- [AutoScheme 中的超参数](#autoscheme-超参数说明)
+ [OPT RTN 模式](#opt-rtn-模式)
+ [AWQ 算法-实验性功能](#awq-算法)
+ + [SVDQuant 算法-实验性功能](#svdquant-算法)
+ [免模型架构量化模式](#免模型架构量化模式)
+ [GGUF 格式](#gguf-格式量化)
+ [量化成本](#量化成本)
@@ -412,6 +413,45 @@ ar.quantize_and_save(output_dir, format="auto_round:llm_compressor")
- `alg_configs="awq"` + `format="auto_round"`:使用 AWQ 平滑,并采用 AutoRound 打包。
- `alg_configs="signround"` + `format="auto_awq"`:不使用 AWQ 平滑,但采用 AutoAWQ 打包。
+### SVDQuant 算法
+
+**实验性功能:目前仅在 FLUX.1-dev 上完成端到端流程验证。**
+
+SVDQuant 将每个 Linear 权重拆分为量化残差分支和较小的浮点低秩分支,并可与 RTN 或 SignRound 组合,生成供 Nunchaku 推理使用的 MXFP4 模型。
+
+RTN(建议先使用此配置):
+
+```bash
+auto-round-rtn --model /path/to/FLUX.1-dev --model_dtype bf16 \
+ --scheme MXFP4 --algorithm svdquant --device 0 \
+ --format svdquant_nunchaku \
+ --output_dir ./flux-dev-mxfp4-svdquant-rtn
+```
+
+SignRound:
+
+```bash
+auto-round --model /path/to/FLUX.1-dev --model_dtype bf16 \
+ --scheme MXFP4 --algorithm svdquant,auto_round \
+ --format svdquant_nunchaku \
+ --dataset /path/to/captions.tsv --batch_size 1 --device 0 \
+ --output_dir ./flux-dev-mxfp4-svdquant-signround
+```
+
+使用 Nunchaku commit [`4de4986`](https://github.com/changwangss/nunchaku/commit/4de49869eaa8565d8c29da344323e82298bdf198) 测得的 FLUX.1-dev 质量结果:
+
+| 配置 | CLIP | CLIP-IQA | ImageReward |
+|---|---:|---:|---:|
+| BF16 | 26.0189 | 0.954360 | 1.018340 |
+| MXFP4,smooth + SVDQuant + SignRound | **26.1039** | **0.962655** | **1.021020** |
+| MXFP4,no smooth + SVDQuant + SignRound | 26.0727 | 0.959363 | 1.002380 |
+| MXFP4,smooth + SVDQuant + RTN | 25.9719 | 0.947763 | 0.939392 |
+| MXFP4,no smooth + SVDQuant + RTN | 25.9624 | 0.946939 | 0.934579 |
+
+SignRound 配置使用 128 个 calibration samples、50 个 inference steps、200 次 tuning iterations、rank 32 和 20 次 residual iterations。
+
+Smooth 搜索、residual iterations、导出和推理说明请参阅 [SVDQuant 详细说明](./svdquant_details_CN.md)。
+
### AutoScheme 自动混合精度量化方案
diff --git a/docs/svdquant_details.md b/docs/svdquant_details.md
new file mode 100644
index 000000000..6f709daaf
--- /dev/null
+++ b/docs/svdquant_details.md
@@ -0,0 +1,243 @@
+# SVDQuant (Experimental)
+
+SVDQuant splits a linear weight into a quantized residual branch and a small
+floating-point low-rank branch:
+
+```text
+W ~= Q(R) + U @ V
+Linear(x) ~= QuantizedLinear(x, Q(R)) + Linear(Linear(x, V), U)
+```
+
+This feature follows the decomposition proposed in [SVDQuant: Absorbing
+Outliers by Low-Rank Components for 4-Bit Diffusion
+Models](https://arxiv.org/abs/2411.05007). AutoRound can combine the transform
+with RTN or SignRound and export a Nunchaku-loadable MXFP4 FLUX pipeline.
+
+> The end-to-end export path has been validated with FLUX.1-dev only. Other
+> models using compatible Diffusers FLUX block classes are experimental and
+> produce a warning.
+
+## Quick start
+
+Install AutoRound and its diffusion dependencies in an isolated environment:
+
+```bash
+pip install --no-build-isolation -e .
+pip install -r test/test_cuda/requirements_diffusion.txt
+```
+
+### Default RTN workflow
+
+The recommended starting point uses rank 32, one residual iteration, no
+smooth search, and plain RTN:
+
+```bash
+CUDA_VISIBLE_DEVICES=0 auto-round-rtn \
+ --model /path/to/FLUX.1-dev \
+ --model_dtype bf16 \
+ --scheme MXFP4 \
+ --algorithm svdquant \
+ --format svdquant_nunchaku \
+ --device 0 \
+ --output_dir ./flux-dev-mxfp4-svdquant-rtn
+```
+
+When `svdquant` is selected and `--format` is omitted, the CLI also defaults to
+`svdquant_nunchaku`.
+
+### Default SignRound workflow
+
+Use SignRound as the terminal quantizer by selecting `auto_round`:
+
+```bash
+CUDA_VISIBLE_DEVICES=0 auto-round \
+ --model /path/to/FLUX.1-dev \
+ --model_dtype bf16 \
+ --scheme MXFP4 \
+ --algorithm svdquant,auto_round \
+ --format svdquant_nunchaku \
+ --dataset /path/to/coco2017-captions.tsv \
+ --batch_size 1 \
+ --device 0 \
+ --low_gpu_mem_usage \
+ --output_dir ./flux-dev-mxfp4-svdquant-signround
+```
+
+The caption dataset is a tab-separated file with `id` and `caption` columns:
+
+```text
+id caption
+0 A photo of a cat
+1 A city street at night
+```
+
+The COCO2017 calibration captions are available from
+[`changwangss/coco2017`](https://huggingface.co/datasets/changwangss/coco2017).
+
+### Optional quality settings
+
+Smooth search and additional residual iterations are opt-in:
+
+```bash
+--enable-svdquant-smooth \
+--svdquant-smooth-num-grids 20 \
+--svdquant-smooth-max-calibration-calls 128 \
+--svdquant-residual-iters 20 \
+--enable-svdquant-residual-early-stop
+```
+
+For diffusion calibration, `--nsamples` controls prompts and
+`--num_inference_steps` controls denoising steps. Start small before running a
+full quality configuration.
+
+## Processing flow
+
+SVDQuant is a structural preprocessor and must run before one terminal
+quantizer:
+
+```text
+Original linear weights
+ -> optional activation-aware smooth search
+ -> group related projections
+ -> low-rank decomposition and residual iteration
+ -> replace nn.Linear with SVDQuantLinear
+ -> quantized residual_linear
+ -> floating-point lora_down
+ -> floating-point lora_up
+ -> optional input smooth factor
+ -> RTN or SignRound
+ -> Nunchaku export
+```
+
+FLUX Q/K/V projections are grouped so they share one low-rank down factor and
+retain separate up factors, matching the runtime layout.
+
+## Residual iteration
+
+With the default `residual_iters=1`:
+
+1. Compute a rank-limited decomposition of the grouped weight `W`.
+2. Materialize `U` and `V` in `low_rank_dtype`.
+3. Form `R = W - U @ V`.
+4. Let the terminal RTN or SignRound stage quantize the residual linear.
+
+With `residual_iters > 1`, each outer iteration alternates between fitting the
+low-rank branch to `W - Q(R_previous)` and applying deployment-compatible RTN
+QDQ to the new residual. Early stop retains the best candidate and stops when
+the objective becomes worse.
+
+The objective is weight reconstruction error without smooth search and
+calibration output error with smooth search. Residual iterations are separate
+from SignRound `--iters`.
+
+## Smooth search
+
+For `G = --svdquant-smooth-num-grids`, the candidates are:
+
+```text
+(alpha, beta) = (0, 0)
+(alpha, 0) for alpha in {1/G, ..., (G-1)/G}
+(alpha, 1-alpha) for the same alpha values
+```
+
+The default `G=20` produces 39 candidates. Each candidate constructs
+`x_span**alpha / w_span**beta` and is scored against floating-point group
+outputs. The lowest finite output error wins.
+
+`--svdquant-smooth-max-calibration-calls` uses reservoir sampling to bound the
+retained calls per group. Calls that are not selected are not copied into the
+smooth CPU cache. This limit does not replace SignRound calibration samples.
+
+## Configuration
+
+| Python field | CLI option | Default | Meaning |
+|---|---|---:|---|
+| `rank` | `--svdquant-rank` | `32` | Low-rank branch rank. |
+| `smooth_enabled` | `--enable-svdquant-smooth` | `False` | Enable Alpha/Beta search. |
+| `smooth_num_grids` | `--svdquant-smooth-num-grids` | `20` | Candidate grid resolution. |
+| `smooth_max_calibration_calls` | `--svdquant-smooth-max-calibration-calls` | `128` | Retained calls per group. |
+| `residual_iters` | `--svdquant-residual-iters` | `1` | Residual outer iterations. |
+| `residual_early_stop` | `--enable-svdquant-residual-early-stop` | `False` | Stop after the objective worsens. |
+| `low_rank_dtype` | `--svdquant-low-rank-dtype` | `bf16` | BF16, FP16, or FP32 factors. |
+| `target_modules` | `--svdquant-target-modules` | `None` | Included module-name substrings. |
+| `exclude_modules` | `--svdquant-exclude-modules` | `None` | Excluded module-name substrings. |
+| `model_adapter` | `--svdquant-model-adapter` | `auto` | Runtime grouping/export adapter. |
+
+## Python API
+
+```python
+from auto_round import AutoRound
+from auto_round.algorithms.quantization.rtn.config import RTNConfig
+from auto_round.algorithms.transforms.svdquant import SVDQuantConfig
+
+autoround = AutoRound(
+ "/path/to/FLUX.1-dev",
+ scheme="MXFP4",
+ model_dtype="bf16",
+ alg_configs=[
+ SVDQuantConfig(rank=32, model_adapter="flux"),
+ RTNConfig(disable_opt_rtn=True),
+ ],
+ device_map=0,
+ low_gpu_mem_usage=True,
+)
+autoround.quantize_and_save(
+ "./flux-dev-mxfp4-svdquant-rtn",
+ format="svdquant_nunchaku",
+)
+```
+
+## Export and inference
+
+The `svdquant_nunchaku` format currently requires MXFP4 E2M1 W4A4, group size
+32, symmetric weights/activations, dynamic activation quantization, and a
+supported runtime adapter. Nunchaku is not imported during quantization; it is
+required only for inference.
+
+Pre-quantized FLUX.1-dev checkpoints are available for
+[smooth SVDQuant + SignRound](https://huggingface.co/changwangss/smooth_svdquant_signround)
+and
+[no-smooth SVDQuant + SignRound](https://huggingface.co/changwangss/nosmooth_svdquant_signround).
+
+The output is a self-contained Diffusers pipeline. Load its Nunchaku transformer
+explicitly, then pass it to Diffusers:
+
+```python
+import torch
+from diffusers import FluxPipeline
+from nunchaku import NunchakuFluxTransformer2dModel
+
+model_dir = "./flux-dev-mxfp4-svdquant-signround"
+transformer = NunchakuFluxTransformer2dModel.from_pretrained(
+ f"{model_dir}/transformer/diffusion_pytorch_model.safetensors",
+ torch_dtype=torch.bfloat16,
+ precision="mxfp4",
+ device="cuda:0",
+)
+pipe = FluxPipeline.from_pretrained(
+ model_dir,
+ transformer=transformer,
+ torch_dtype=torch.bfloat16,
+ local_files_only=True,
+)
+pipe.enable_model_cpu_offload()
+
+image = pipe(
+ "A cat holding a sign that says Hello world",
+ num_inference_steps=20,
+ guidance_scale=3.5,
+ generator=torch.Generator(device="cuda").manual_seed(12345),
+ height=512,
+ width=512,
+).images[0]
+image.save("flux-svdquant-mxfp4.png")
+```
+
+## Limitations
+
+- Runtime-loadable export is currently validated only for FLUX.1-dev.
+- SVDQuant requires `nblocks=1`.
+- Smooth search replays every retained call for every Alpha/Beta candidate.
+- More residual iterations repeat decomposition and QDQ work.
+- A smoke image validates loading and numerical stability, not dataset-level
+ generation quality.
diff --git a/docs/svdquant_details_CN.md b/docs/svdquant_details_CN.md
new file mode 100644
index 000000000..bce111140
--- /dev/null
+++ b/docs/svdquant_details_CN.md
@@ -0,0 +1,235 @@
+# SVDQuant(实验性功能)
+
+SVDQuant 将 Linear 权重拆成一个量化残差分支和一个较小的浮点低秩分支:
+
+```text
+W ~= Q(R) + U @ V
+Linear(x) ~= QuantizedLinear(x, Q(R)) + Linear(Linear(x, V), U)
+```
+
+该功能采用 [SVDQuant: Absorbing Outliers by Low-Rank Components for 4-Bit
+Diffusion Models](https://arxiv.org/abs/2411.05007) 提出的分解方法。
+AutoRound 可以将该变换与 RTN 或 SignRound 组合,并导出 Nunchaku 可直接加载的
+MXFP4 FLUX pipeline。
+
+> 目前端到端导出流程仅在 FLUX.1-dev 上完成验证。其他使用兼容 Diffusers
+> FLUX block 类的模型仍属于实验性支持,运行时会输出警告。
+
+## 快速开始
+
+请在隔离环境中安装 AutoRound 和 diffusion 依赖:
+
+```bash
+pip install --no-build-isolation -e .
+pip install -r test/test_cuda/requirements_diffusion.txt
+```
+
+### 默认 RTN 流程
+
+建议先从 rank 32、一次 residual iteration、关闭 smooth、使用普通 RTN 的默认配置开始:
+
+```bash
+CUDA_VISIBLE_DEVICES=0 auto-round-rtn \
+ --model /path/to/FLUX.1-dev \
+ --model_dtype bf16 \
+ --scheme MXFP4 \
+ --algorithm svdquant \
+ --format svdquant_nunchaku \
+ --device 0 \
+ --output_dir ./flux-dev-mxfp4-svdquant-rtn
+```
+
+选择 `svdquant` 且未指定 `--format` 时,CLI 也会默认使用
+`svdquant_nunchaku`。
+
+### 默认 SignRound 流程
+
+选择 `auto_round` 即可使用 SignRound 作为最终量化算法:
+
+```bash
+CUDA_VISIBLE_DEVICES=0 auto-round \
+ --model /path/to/FLUX.1-dev \
+ --model_dtype bf16 \
+ --scheme MXFP4 \
+ --algorithm svdquant,auto_round \
+ --format svdquant_nunchaku \
+ --dataset /path/to/coco2017-captions.tsv \
+ --batch_size 1 \
+ --device 0 \
+ --low_gpu_mem_usage \
+ --output_dir ./flux-dev-mxfp4-svdquant-signround
+```
+
+Caption dataset 是包含 `id` 和 `caption` 两列的 TSV 文件:
+
+```text
+id caption
+0 A photo of a cat
+1 A city street at night
+```
+
+COCO2017 calibration captions 可从
+[`changwangss/coco2017`](https://huggingface.co/datasets/changwangss/coco2017)
+获取。
+
+### 可选的质量参数
+
+Smooth 搜索和更多 residual iterations 默认不开启,可按需添加:
+
+```bash
+--enable-svdquant-smooth \
+--svdquant-smooth-num-grids 20 \
+--svdquant-smooth-max-calibration-calls 128 \
+--svdquant-residual-iters 20 \
+--enable-svdquant-residual-early-stop
+```
+
+Diffusion calibration 中,`--nsamples` 控制 prompt 数量,
+`--num_inference_steps` 控制去噪步数。建议先用小配置跑通端到端流程,再运行完整质量配置。
+
+## 处理流程
+
+SVDQuant 是结构变换预处理器,后面必须接一个最终量化算法:
+
+```text
+原始 Linear 权重
+ -> 可选的 activation-aware smooth 搜索
+ -> 对相关 projection 分组
+ -> 低秩分解和 residual iteration
+ -> 用 SVDQuantLinear 替换 nn.Linear
+ -> 量化 residual_linear
+ -> 浮点 lora_down
+ -> 浮点 lora_up
+ -> 可选 input smooth factor
+ -> RTN 或 SignRound
+ -> Nunchaku 导出
+```
+
+FLUX 的 Q/K/V projection 会作为一组处理:共享一个低秩 down factor,并保留各自的
+up factor,以匹配运行时布局。
+
+## Residual iteration
+
+默认 `residual_iters=1` 时:
+
+1. 对分组权重 `W` 计算指定 rank 的低秩分解。
+2. 使用 `low_rank_dtype` 物化 `U` 和 `V`。
+3. 计算残差 `R = W - U @ V`。
+4. 由最终的 RTN 或 SignRound 量化 residual linear。
+
+当 `residual_iters > 1` 时,每次 outer iteration 会交替执行两步:对
+`W - Q(R_previous)` 拟合新的低秩分支,再对新残差执行与部署一致的 RTN QDQ。
+Early stop 会保留最佳候选,并在目标误差变差时停止。
+
+关闭 smooth 时使用权重重建误差;开启 smooth 时使用 calibration output error。
+Residual iterations 与 SignRound 的 `--iters` 是两个独立参数。
+
+## Smooth 搜索
+
+令 `G = --svdquant-smooth-num-grids`,候选集合为:
+
+```text
+(alpha, beta) = (0, 0)
+(alpha, 0),其中 alpha 属于 {1/G, ..., (G-1)/G}
+(alpha, 1-alpha),alpha 取值同上
+```
+
+默认 `G=20`,共生成 39 组候选。每组候选计算
+`x_span**alpha / w_span**beta`,并与浮点 group output 比较,选择有限且最小的
+output error。
+
+`--svdquant-smooth-max-calibration-calls` 使用 reservoir sampling,限制每个
+group 保留的 calls 数量。未被选中的 call 不会复制到 smooth CPU cache。该参数不会
+代替 SignRound 自身的 calibration samples。
+
+## 配置项
+
+| Python 字段 | CLI 参数 | 默认值 | 含义 |
+|---|---|---:|---|
+| `rank` | `--svdquant-rank` | `32` | 低秩分支 rank。 |
+| `smooth_enabled` | `--enable-svdquant-smooth` | `False` | 开启 Alpha/Beta 搜索。 |
+| `smooth_num_grids` | `--svdquant-smooth-num-grids` | `20` | 候选网格精度。 |
+| `smooth_max_calibration_calls` | `--svdquant-smooth-max-calibration-calls` | `128` | 每组保留的 calls 上限。 |
+| `residual_iters` | `--svdquant-residual-iters` | `1` | Residual outer iterations。 |
+| `residual_early_stop` | `--enable-svdquant-residual-early-stop` | `False` | 目标误差变差后停止。 |
+| `low_rank_dtype` | `--svdquant-low-rank-dtype` | `bf16` | BF16、FP16 或 FP32 factor。 |
+| `target_modules` | `--svdquant-target-modules` | `None` | 需要处理的模块名子串。 |
+| `exclude_modules` | `--svdquant-exclude-modules` | `None` | 排除的模块名子串。 |
+| `model_adapter` | `--svdquant-model-adapter` | `auto` | 运行时分组和导出 adapter。 |
+
+## Python API
+
+```python
+from auto_round import AutoRound
+from auto_round.algorithms.quantization.rtn.config import RTNConfig
+from auto_round.algorithms.transforms.svdquant import SVDQuantConfig
+
+autoround = AutoRound(
+ "/path/to/FLUX.1-dev",
+ scheme="MXFP4",
+ model_dtype="bf16",
+ alg_configs=[
+ SVDQuantConfig(rank=32, model_adapter="flux"),
+ RTNConfig(disable_opt_rtn=True),
+ ],
+ device_map=0,
+ low_gpu_mem_usage=True,
+)
+autoround.quantize_and_save(
+ "./flux-dev-mxfp4-svdquant-rtn",
+ format="svdquant_nunchaku",
+)
+```
+
+## 导出与推理
+
+`svdquant_nunchaku` 当前要求 MXFP4 E2M1 W4A4、group size 32、weight 和
+activation 均为对称量化、dynamic activation quantization,并使用受支持的运行时
+adapter。量化过程不会导入 Nunchaku;只有推理时需要安装 Nunchaku。
+
+可直接使用以下已量化的 FLUX.1-dev checkpoint:
+[smooth SVDQuant + SignRound](https://huggingface.co/changwangss/smooth_svdquant_signround)
+和
+[no-smooth SVDQuant + SignRound](https://huggingface.co/changwangss/nosmooth_svdquant_signround)。
+
+输出目录是一个完整的 Diffusers pipeline。需要显式加载其中的 Nunchaku
+transformer,再传给 Diffusers:
+
+```python
+import torch
+from diffusers import FluxPipeline
+from nunchaku import NunchakuFluxTransformer2dModel
+
+model_dir = "./flux-dev-mxfp4-svdquant-signround"
+transformer = NunchakuFluxTransformer2dModel.from_pretrained(
+ f"{model_dir}/transformer/diffusion_pytorch_model.safetensors",
+ torch_dtype=torch.bfloat16,
+ precision="mxfp4",
+ device="cuda:0",
+)
+pipe = FluxPipeline.from_pretrained(
+ model_dir,
+ transformer=transformer,
+ torch_dtype=torch.bfloat16,
+ local_files_only=True,
+)
+pipe.enable_model_cpu_offload()
+
+image = pipe(
+ "A cat holding a sign that says Hello world",
+ num_inference_steps=20,
+ guidance_scale=3.5,
+ generator=torch.Generator(device="cuda").manual_seed(12345),
+ height=512,
+ width=512,
+).images[0]
+image.save("flux-svdquant-mxfp4.png")
+```
+
+## 当前限制
+
+- Runtime-loadable export 目前只在 FLUX.1-dev 上完成验证。
+- SVDQuant 要求 `nblocks=1`。
+- Smooth 搜索会针对每组 Alpha/Beta 候选重放所有保留 calls。
+- 增加 residual iterations 会重复执行分解和 QDQ。
+- Smoke 图片只能验证加载和数值稳定性,不能代替数据集级的生成质量评估。
diff --git a/test/unit/test_cpu/algorithms/test_block_runner.py b/test/unit/test_cpu/algorithms/test_block_runner.py
index 394799e99..34bd6fb9e 100644
--- a/test/unit/test_cpu/algorithms/test_block_runner.py
+++ b/test/unit/test_cpu/algorithms/test_block_runner.py
@@ -64,3 +64,82 @@ def test_unregistered_tuple_output_keeps_first_tensor_behavior():
assert runner.last_output_dict is None
for output, input_tensor in zip(outputs, inputs):
torch.testing.assert_close(output, input_tensor + 1)
+
+
+def test_indexed_single_sample_forward_preserves_one_batch_dimension():
+ runner = BlockForwardRunner(
+ batch_dim=0,
+ batch_size=1,
+ device="cpu",
+ cache_device="cpu",
+ amp=True,
+ amp_dtype=torch.bfloat16,
+ )
+ sample = torch.randn(1, 3, 4)
+
+ output = runner(torch.nn.Identity(), [sample], {}, indices=torch.tensor([0]))
+
+ assert output.shape == sample.shape
+ torch.testing.assert_close(output, sample)
+
+
+def test_indexed_diffusion_outputs_preserve_batch_dimension():
+ class FluxTransformerBlock(torch.nn.Module):
+ def forward(self, hidden_states, **_kwargs):
+ return hidden_states + 1, hidden_states + 2
+
+ runner = BlockForwardRunner(
+ batch_dim=0,
+ batch_size=1,
+ device="cpu",
+ cache_device="cpu",
+ amp=True,
+ amp_dtype=torch.bfloat16,
+ is_diffusion=True,
+ )
+ sample = torch.randn(1, 3, 4)
+
+ output = runner(
+ FluxTransformerBlock(),
+ {"hidden_states": [sample]},
+ {},
+ indices=torch.tensor([0]),
+ )
+
+ assert output.shape == sample.shape
+ assert runner.last_output_dict["encoder_hidden_states"].shape == sample.shape
+ assert runner.last_output_dict["hidden_states"].shape == sample.shape
+
+
+def test_outputs_are_moved_to_cache_device_before_next_batch(monkeypatch):
+ events = []
+
+ class TrackingTensor(torch.Tensor):
+ @staticmethod
+ def __new__(cls, value, batch_index):
+ result = torch.Tensor._make_subclass(cls, value, require_grad=False)
+ result.batch_index = batch_index
+ return result
+
+ def to(self, *args, **kwargs):
+ events.append(("move", self.batch_index))
+ return super().to(*args, **kwargs)
+
+ runner = BlockForwardRunner(batch_size=1, device="cpu", cache_device="cpu", amp=False)
+ call_count = 0
+
+ def fake_forward_one_batch(_block, batch_inputs, _batch_others):
+ nonlocal call_count
+ if call_count:
+ assert ("move", call_count - 1) in events
+ output = TrackingTensor(batch_inputs, call_count)
+ call_count += 1
+ return output
+
+ monkeypatch.setattr(runner, "_forward_one_batch", fake_forward_one_batch)
+ inputs = [torch.full((1, 2), value, dtype=torch.float32) for value in range(3)]
+
+ outputs = runner(torch.nn.Identity(), inputs, {})
+
+ assert len(outputs) == len(inputs)
+ assert call_count == len(inputs)
diff --git a/test/unit/test_cpu/algorithms/test_svdquant.py b/test/unit/test_cpu/algorithms/test_svdquant.py
new file mode 100644
index 000000000..e3a42e9c2
--- /dev/null
+++ b/test/unit/test_cpu/algorithms/test_svdquant.py
@@ -0,0 +1,187 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from auto_round.algorithms.composer import AlgorithmComposer, BlockContext
+from auto_round.algorithms.quantization.rtn.config import RTNConfig
+from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
+from auto_round.algorithms.transforms.svdquant import SVDQuantConfig, SVDQuantLinear
+from auto_round.schemes import PRESET_SCHEMES
+from auto_round.utils.device_manager import device_manager
+
+
+class DummyBlock(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(32, 16)
+
+ def forward(self, hidden_states):
+ return self.proj(hidden_states)
+
+
+class DummyModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.blocks = torch.nn.ModuleList([DummyBlock()])
+
+
+class FluxTransformerBlock(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.attn = torch.nn.Module()
+ self.attn.to_q = torch.nn.Linear(32, 16)
+ self.norm1 = torch.nn.Module()
+ self.norm1.linear = torch.nn.Linear(32, 16)
+
+
+def _prepare_model():
+ model = DummyModel()
+ for name, module in model.named_modules():
+ module.global_name = name
+ if isinstance(module, torch.nn.Linear):
+ module.bits = 4
+ module.group_size = 32
+ module.sym = True
+ module.data_type = "mx_fp4e2m1"
+ module.act_bits = 4
+ module.act_group_size = 32
+ module.act_sym = True
+ module.act_data_type = "mx_fp4e2m1"
+ module.act_dynamic = True
+ module.scale_dtype = torch.float32
+ return model
+
+
[email protected]("smooth", [False, True], ids=["nosmooth", "smooth"])
[email protected]("terminal", ["rtn", "signround"])
+def test_svdquant_pipeline_smoke(monkeypatch, smooth, terminal):
+ monkeypatch.setattr(device_manager, "_device_map", "cpu")
+ monkeypatch.setattr(device_manager, "_device_list", ["cpu"])
+ monkeypatch.setattr(device_manager, "_major_device", "cpu")
+ model = _prepare_model()
+ block_name = "blocks.0"
+ svd_config = SVDQuantConfig(
+ rank=2,
+ smooth_enabled=smooth,
+ smooth_num_grids=2,
+ smooth_max_calibration_calls=1,
+ residual_iters=1,
+ low_rank_dtype="fp32",
+ target_modules=["proj"],
+ )
+ terminal_config = (
+ RTNConfig(disable_opt_rtn=True)
+ if terminal == "rtn"
+ else SignRoundConfig(
+ iters=1,
+ lr=1.0,
+ minmax_lr=1.0,
+ enable_minmax_tuning=False,
+ enable_quanted_input=False,
+ )
+ )
+ orchestrator = SimpleNamespace(
+ model_context=SimpleNamespace(
+ model=model,
+ amp=True,
+ amp_dtype=torch.bfloat16,
+ is_diffusion=False,
+ is_moe_model=False,
+ output_config=None,
+ ),
+ compress_context=SimpleNamespace(
+ enable_torch_compile=False,
+ low_gpu_mem_usage=False,
+ cache_device=torch.device("cpu"),
+ clear_memory=lambda: None,
+ ),
+ calibration_context=SimpleNamespace(batch_size=1, batch_dim=0),
+ scheme_context=PRESET_SCHEMES["MXFP4"],
+ scale_dtype=None,
+ nblocks=1,
+ quant_block_list=[[block_name]],
+ data_type="mx_fp4e2m1",
+ batch_dim=0,
+ batch_size=1,
+ cache_device="cpu",
+ amp=True,
+ amp_dtype=torch.bfloat16,
+ shared_cache_keys=(),
+ )
+ composer = AlgorithmComposer([svd_config, terminal_config], orchestrator=orchestrator)
+ composer.prepare_run()
+ block = model.blocks[0]
+ inputs = torch.randn(1, 2, 32)
+ ctx = BlockContext(
+ model=model,
+ block_names=[block_name],
+ block_name=block_name,
+ block_index=0,
+ block_cnt=1,
+ )
+
+ composer.compress_block(block, [inputs], {}, ctx)
+
+ assert isinstance(block.proj, SVDQuantLinear)
+ assert torch.isfinite(block(inputs)).all()
+ transform = composer.preprocessors[0]
+ assert not transform._smooth_calibration
+ assert block_name not in transform._block_groups
+
+
+def test_flux_targets_resolve_when_model_is_attached_after_prepare_run():
+ model = torch.nn.Module()
+ model.config = {"_class_name": "FluxTransformer2DModel"}
+ model.blocks = torch.nn.ModuleList([FluxTransformerBlock()])
+ for name, module in model.named_modules():
+ module.global_name = name
+ if isinstance(module, torch.nn.Linear):
+ for key, value in {
+ "bits": 4,
+ "group_size": 32,
+ "sym": True,
+ "data_type": "mx_fp4e2m1",
+ "act_bits": 4,
+ "act_group_size": 32,
+ "act_sym": True,
+ "act_data_type": "mx_fp4e2m1",
+ "act_dynamic": True,
+ }.items():
+ setattr(module, key, value)
+
+ model_context = SimpleNamespace(model=None)
+ transform = AlgorithmComposer([SVDQuantConfig(rank=2), RTNConfig(disable_opt_rtn=True)]).preprocessors[0]
+ transform.bind(
+ SimpleNamespace(
+ model_context=model_context,
+ compress_context=None,
+ calibration_context=None,
+ scheme_context=PRESET_SCHEMES["MXFP4"],
+ scale_dtype=None,
+ nblocks=1,
+ quant_block_list=[["blocks.0"]],
+ )
+ )
+ transform.prepare_run()
+ model_context.model = model
+ ctx = BlockContext(model=model, block_names=["blocks.0"], block_name="blocks.0", block_index=0)
+
+ transform.pre_quantize_block(ctx)
+
+ assert isinstance(model.blocks[0].attn.to_q, SVDQuantLinear)
+ assert isinstance(model.blocks[0].norm1.linear, torch.nn.Linear)
diff --git a/test/unit/test_cpu/algorithms/test_svdquant_residual.py b/test/unit/test_cpu/algorithms/test_svdquant_residual.py
new file mode 100644
index 000000000..c35dbb576
--- /dev/null
+++ b/test/unit/test_cpu/algorithms/test_svdquant_residual.py
@@ -0,0 +1,135 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+import torch
+
+import auto_round.algorithms.transforms.svdquant.residual as residual_module
+from auto_round.algorithms.transforms.svdquant.config import SVDQuantConfig
+from auto_round.algorithms.transforms.svdquant.residual import (
+ ResidualQuantScheme,
+ iterate_residual_decomposition,
+ rtn_qdq_residual,
+ truncated_svd,
+)
+from auto_round.data_type.mxfp import quant_mx_rceil
+
+
+def test_rtn_qdq_residual_matches_deployable_mxfp4_quantizer():
+ weight = torch.linspace(-3.0, 3.0, steps=3 * 64, dtype=torch.float32).reshape(3, 64)
+ scheme = ResidualQuantScheme(data_type="mx_fp4e2m1", bits=4, group_size=32, sym=True)
+
+ actual = rtn_qdq_residual(weight, scheme)
+ expected, _, _ = quant_mx_rceil(
+ tensor=weight,
+ bits=4,
+ group_size=32,
+ data_type="mx_fp4e2m1",
+ )
+
+ torch.testing.assert_close(actual, expected)
+ assert actual.shape == weight.shape
+ assert actual.dtype == weight.dtype
+ assert actual.device == weight.device
+
+
+def test_svdquant_config_defaults_to_data_free_single_iteration():
+ config = SVDQuantConfig()
+
+ assert config.rank == 32
+ assert config.smooth_enabled is False
+ assert config.smooth_max_calibration_calls == 128
+ assert config.residual_iters == 1
+ assert config.residual_early_stop is False
+ assert config.need_calib is False
+
+
+def test_svdquant_config_rejects_invalid_structural_options():
+ cases = (
+ ({"rank": -1}, "rank"),
+ ({"smooth_enabled": 1}, "smooth_enabled"),
+ ({"smooth_num_grids": 1}, "smooth_num_grids"),
+ ({"smooth_max_calibration_calls": 0}, "smooth_max_calibration_calls"),
+ ({"low_rank_dtype": "bf116"}, "low_rank_dtype"),
+ ({"residual_iters": 0}, "residual_iters"),
+ )
+ for kwargs, field in cases:
+ with pytest.raises(ValueError, match=field):
+ SVDQuantConfig(**kwargs)
+
+
+def test_truncated_svd_returns_shared_down_factor_for_stacked_projection_group():
+ torch.manual_seed(0)
+ qkv = torch.randn(12, 8, dtype=torch.float32)
+
+ low_rank, down, up = truncated_svd(qkv, rank=3)
+
+ assert low_rank.shape == qkv.shape
+ assert down.shape == (3, 8)
+ assert up.shape == (12, 3)
+ torch.testing.assert_close(low_rank, up @ down)
+ q_up, k_up, v_up = up.split((4, 4, 4), dim=0)
+ torch.testing.assert_close(torch.cat((q_up @ down, k_up @ down, v_up @ down)), low_rank)
+
+
+def test_residual_iteration_keeps_the_best_materialized_candidate():
+ torch.manual_seed(1)
+ weight = torch.randn(8, 32, dtype=torch.float32)
+ scheme = ResidualQuantScheme(data_type="mx_fp4e2m1", bits=4, group_size=32, sym=True)
+
+ first_low_rank, first_down, first_up = truncated_svd(weight, rank=2)
+ first_qdq = rtn_qdq_residual(weight - first_low_rank, scheme)
+ first_error = torch.sum((weight - (first_qdq + first_up @ first_down)).square()).item()
+
+ result = iterate_residual_decomposition(
+ weight,
+ rank=2,
+ scheme=scheme,
+ iterations=4,
+ early_stop=False,
+ residual_dtype=torch.float32,
+ low_rank_dtype=torch.float32,
+ )
+
+ assert 1 <= result.selected_iteration <= 4
+ assert result.error <= first_error
+ assert result.residual.shape == weight.shape
+ torch.testing.assert_close(result.residual + result.up @ result.down, weight)
+
+
+def test_residual_iteration_early_stops_after_candidate_worsens(monkeypatch):
+ torch.manual_seed(2)
+ weight = torch.randn(4, 32, dtype=torch.float32)
+ scheme = ResidualQuantScheme(data_type="mx_fp4e2m1", bits=4, group_size=32, sym=True)
+ calls = 0
+
+ def qdq_then_degrade(residual, _scheme):
+ nonlocal calls
+ calls += 1
+ return residual if calls == 1 else residual + 1
+
+ monkeypatch.setattr(residual_module, "rtn_qdq_residual", qdq_then_degrade)
+
+ result = iterate_residual_decomposition(
+ weight,
+ rank=2,
+ scheme=scheme,
+ iterations=10,
+ early_stop=True,
+ residual_dtype=torch.float32,
+ low_rank_dtype=torch.float32,
+ )
+
+ assert calls == 2
+ assert result.selected_iteration == 1
diff --git a/test/unit/test_cpu/algorithms/test_svdquant_smooth.py b/test/unit/test_cpu/algorithms/test_svdquant_smooth.py
new file mode 100644
index 000000000..0d44ff8c6
--- /dev/null
+++ b/test/unit/test_cpu/algorithms/test_svdquant_smooth.py
@@ -0,0 +1,75 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+import torch
+
+from auto_round.algorithms.transforms.svdquant.smooth import (
+ absmax_channel_span,
+ build_alpha_beta_candidates,
+ build_smooth_scale,
+ select_best_layer_candidate,
+ validate_smooth_scale_for_deployment,
+)
+
+
+def test_alpha_beta_grid_matches_proven_candidate_order():
+ assert build_alpha_beta_candidates(4) == [
+ (0.0, 0.0),
+ (0.25, 0.0),
+ (0.5, 0.0),
+ (0.75, 0.0),
+ (0.25, 0.75),
+ (0.5, 0.5),
+ (0.75, 0.25),
+ ]
+ assert len(build_alpha_beta_candidates(20)) == 39
+
+
+def test_smooth_scale_uses_activation_and_weight_channel_spans():
+ activations = torch.tensor([[[1.0, -9.0, 4.0], [16.0, 2.0, -1.0]]])
+ weights = torch.tensor([[1.0, 4.0, 16.0], [-0.5, 2.0, 8.0]])
+ x_span = absmax_channel_span(activations, -1)
+ w_span = absmax_channel_span(weights, 1)
+
+ scale = build_smooth_scale(x_span, w_span, alpha=0.5, beta=0.5)
+
+ torch.testing.assert_close(scale, x_span.sqrt() / w_span.sqrt())
+
+
+def test_smooth_scale_zero_channels_follow_identity_fallback():
+ scale = build_smooth_scale(
+ torch.tensor([0.0, 4.0]),
+ torch.tensor([0.0, 1.0]),
+ alpha=0.5,
+ beta=0.5,
+ eps=1e-6,
+ )
+
+ torch.testing.assert_close(scale, torch.ones(2))
+
+
+def test_deployment_validation_rejects_bfloat16_reciprocal_overflow():
+ with pytest.raises(ValueError, match="deployable.*proj"):
+ validate_smooth_scale_for_deployment(
+ torch.tensor([1e-40, 1.0]),
+ dtype=torch.bfloat16,
+ module_name="transformer_blocks.0.proj",
+ )
+
+
+def test_candidate_selection_keeps_later_exact_tie_and_skips_nonfinite():
+ candidates = [("nan", float("nan")), ("first", 1.0), ("later", 1.0), ("worse", 2.0)]
+
+ assert select_best_layer_candidate(candidates, module_name="blocks.0.qkv") == "later"
diff --git a/test/unit/test_cpu/algorithms/test_svdquant_smooth_adapters.py b/test/unit/test_cpu/algorithms/test_svdquant_smooth_adapters.py
new file mode 100644
index 000000000..6e798a274
--- /dev/null
+++ b/test/unit/test_cpu/algorithms/test_svdquant_smooth_adapters.py
@@ -0,0 +1,74 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from auto_round.algorithms.transforms.svdquant.smooth_adapters import discover_svdquant_groups
+
+
+class Attention(torch.nn.Module):
+ def __init__(self, width):
+ super().__init__()
+ self.to_q = torch.nn.Linear(width, width)
+ self.to_k = torch.nn.Linear(width, width)
+ self.to_v = torch.nn.Linear(width, width)
+ self.add_q_proj = torch.nn.Linear(width, width)
+ self.add_k_proj = torch.nn.Linear(width, width)
+ self.add_v_proj = torch.nn.Linear(width, width)
+ self.to_out = torch.nn.ModuleList([torch.nn.Linear(width, width)])
+ self.to_add_out = torch.nn.Linear(width, width)
+
+
+class Projection(torch.nn.Module):
+ def __init__(self, in_features, out_features):
+ super().__init__()
+ self.proj = torch.nn.Linear(in_features, out_features)
+
+
+class FeedForward(torch.nn.Module):
+ def __init__(self, width):
+ super().__init__()
+ self.net = torch.nn.ModuleList(
+ [Projection(width, width * 2), torch.nn.GELU(), torch.nn.Linear(width * 2, width)]
+ )
+
+
+class FluxTransformerBlock(torch.nn.Module):
+ def __init__(self, width=4):
+ super().__init__()
+ self.attn = Attention(width)
+ self.ff = FeedForward(width)
+ self.ff_context = FeedForward(width)
+
+
+def test_flux_adapter_discovers_fused_projection_groups_without_duplicates():
+ block = FluxTransformerBlock()
+ block.global_name = "transformer_blocks.7"
+
+ groups = discover_svdquant_groups(block, lambda _name, module: isinstance(module, torch.nn.Linear))
+ by_key = {group.key: group for group in groups}
+
+ assert set(by_key) == {
+ "transformer_blocks.7.attn.qkv",
+ "transformer_blocks.7.attn.add_qkv",
+ "transformer_blocks.7.attn.to_out.0",
+ "transformer_blocks.7.attn.to_add_out",
+ "transformer_blocks.7.ff.net.0.proj",
+ "transformer_blocks.7.ff.net.2",
+ "transformer_blocks.7.ff_context.net.0.proj",
+ "transformer_blocks.7.ff_context.net.2",
+ }
+ assert len(by_key["transformer_blocks.7.attn.qkv"].projections) == 3
+ assert len(by_key["transformer_blocks.7.attn.add_qkv"].projections) == 3
+ assert len({id(projection) for group in groups for projection in group.projections}) == 12
diff --git a/test/unit/test_cpu/export/test_svdquant_flux_adapter.py b/test/unit/test_cpu/export/test_svdquant_flux_adapter.py
new file mode 100644
index 000000000..08439b4f5
--- /dev/null
+++ b/test/unit/test_cpu/export/test_svdquant_flux_adapter.py
@@ -0,0 +1,249 @@
+import pytest
+import torch
+
+from auto_round.export.svdquant_adapters.flux import FluxSVDQuantNunchakuAdapter
+from auto_round.export.svdquant_nunchaku import (
+ SourceLinearRecord,
+ SVDQuantExportConfig,
+ SVDQuantLinearScheme,
+ collect_svdquant_tensors,
+ save_svdquant_nunchaku_safetensors,
+)
+
+SCHEME = SVDQuantLinearScheme("mx_fp4", 4, 32, True, "mx_fp4", 4, 32, True, True)
+
+
+def _source(name, out_features=8, in_features=8, rank=2, seed=0, bias=True):
+ generator = torch.Generator().manual_seed(seed)
+ return SourceLinearRecord(
+ name=name,
+ residual_weight=torch.randn(out_features, in_features, generator=generator),
+ lora_down=torch.randn(rank, in_features, generator=generator),
+ lora_up=torch.randn(out_features, rank, generator=generator),
+ smooth=torch.linspace(0.5, 1.5, in_features),
+ smooth_orig=torch.linspace(0.75, 1.75, in_features),
+ bias=torch.randn(out_features, generator=generator) if bias else None,
+ scheme=SCHEME,
+ )
+
+
+def _effective(source):
+ return (source.residual_weight + source.lora_up @ source.lora_down) * source.smooth
+
+
+def _model(config=None):
+ model = torch.nn.Module()
+ model.config = config or {
+ "num_layers": 1,
+ "num_single_layers": 1,
+ "num_attention_heads": 2,
+ "attention_head_dim": 4,
+ }
+ return model
+
+
+def test_double_qkv_reconstructs_effective_sources_at_fixed_rank_in_order():
+ sources = tuple(
+ _source(f"transformer_blocks.0.attn.to_{name}", seed=index + 1) for index, name in enumerate(("q", "k", "v"))
+ )
+ adapter = FluxSVDQuantNunchakuAdapter(require_complete_model=False)
+
+ (record,) = tuple(adapter.map_modules(_model(), sources))
+
+ assert record.prefix == "transformer_blocks.0.qkv_proj"
+ assert record.sources == sources
+ assert record.lora_down.shape[0] == 2
+ assert torch.equal(record.smooth, torch.ones(8))
+ expected = torch.cat([_effective(source) for source in sources])
+ actual = record.residual_weight + record.lora_up @ record.lora_down
+ torch.testing.assert_close(actual, expected, atol=2e-5, rtol=2e-5)
+ torch.testing.assert_close(record.bias, torch.cat([source.bias for source in sources]))
+ assert all(
+ tensor.device.type == "cpu" and tensor.is_contiguous()
+ for tensor in (record.residual_weight, record.lora_down, record.lora_up)
+ )
+
+
+def test_double_qkv_preserves_shared_low_rank_and_smooth_without_redecomposition():
+ shared_down = torch.randn(2, 8)
+ shared_smooth = torch.linspace(0.5, 1.5, 8)
+ shared_smooth_orig = torch.linspace(0.75, 1.75, 8)
+ sources = tuple(
+ _source(f"transformer_blocks.0.attn.to_{name}", seed=index + 21) for index, name in enumerate(("q", "k", "v"))
+ )
+ sources = tuple(
+ SourceLinearRecord(
+ name=source.name,
+ residual_weight=source.residual_weight,
+ lora_down=shared_down.clone(),
+ lora_up=source.lora_up,
+ smooth=shared_smooth.clone(),
+ smooth_orig=shared_smooth_orig.clone(),
+ bias=source.bias,
+ scheme=source.scheme,
+ )
+ for source in sources
+ )
+
+ (record,) = tuple(FluxSVDQuantNunchakuAdapter(require_complete_model=False).map_modules(_model(), sources))
+
+ torch.testing.assert_close(record.residual_weight, torch.cat([source.residual_weight for source in sources]))
+ torch.testing.assert_close(record.lora_down, shared_down)
+ torch.testing.assert_close(record.lora_up, torch.cat([source.lora_up for source in sources]))
+ torch.testing.assert_close(record.smooth, shared_smooth)
+ torch.testing.assert_close(record.smooth_orig, shared_smooth_orig)
+
+
+def test_single_proj_out_splits_input_columns_and_keeps_bias_only_on_mlp():
+ source = _source("single_transformer_blocks.0.proj_out", out_features=8, in_features=16, seed=10)
+ model = _model({"num_layers": 0, "num_single_layers": 1, "num_attention_heads": 2, "attention_head_dim": 4})
+ adapter = FluxSVDQuantNunchakuAdapter(require_complete_model=False)
+
+ out_proj, mlp_fc2 = tuple(adapter.map_modules(model, (source,)))
+
+ assert (out_proj.prefix, mlp_fc2.prefix) == (
+ "single_transformer_blocks.0.out_proj",
+ "single_transformer_blocks.0.mlp_fc2",
+ )
+ expected = _effective(source)
+ out_effective = (out_proj.residual_weight + out_proj.lora_up @ out_proj.lora_down) * out_proj.smooth
+ mlp_effective = (mlp_fc2.residual_weight + mlp_fc2.lora_up @ mlp_fc2.lora_down) * mlp_fc2.smooth
+ torch.testing.assert_close(out_effective, expected[:, :8], atol=2e-5, rtol=2e-5)
+ torch.testing.assert_close(mlp_effective, expected[:, 8:], atol=2e-5, rtol=2e-5)
+ torch.testing.assert_close(out_proj.lora_down, source.lora_down[:, :8])
+ torch.testing.assert_close(mlp_fc2.lora_down, source.lora_down[:, 8:])
+ torch.testing.assert_close(out_proj.smooth, source.smooth[:8])
+ torch.testing.assert_close(mlp_fc2.smooth, source.smooth[8:])
+ assert out_proj.bias is None
+ torch.testing.assert_close(mlp_fc2.bias, source.bias)
+
+
+def _install(root, path, module):
+ current = root
+ parts = path.split(".")
+ for part in parts[:-1]:
+ if not hasattr(current, part):
+ current.add_module(part, torch.nn.Module())
+ current = getattr(current, part)
+ current.add_module(parts[-1], module)
+
+
+def _install_parameter(root, name, tensor):
+ module_name, parameter_name = name.rsplit(".", 1)
+ current = root
+ for part in module_name.split("."):
+ if not hasattr(current, part):
+ current.add_module(part, torch.nn.Module())
+ current = getattr(current, part)
+ current.register_parameter(parameter_name, torch.nn.Parameter(tensor))
+
+
+def _small_complete_top_level_tensors():
+ from auto_round.export.svdquant_adapters.flux import FLUX_TOP_LEVEL_TENSOR_KEYS
+
+ return {
+ key: torch.ones(2, dtype=torch.bfloat16) if key.endswith(".bias") else torch.ones(2, 2, dtype=torch.bfloat16)
+ for key in FLUX_TOP_LEVEL_TENSOR_KEYS
+ }
+
+
+def _small_linear_tensors(prefix):
+ return {
+ f"{prefix}.qweight": torch.ones(1, 1, dtype=torch.int8),
+ f"{prefix}.wscales": torch.ones(1, 1, dtype=torch.uint8),
+ f"{prefix}.smooth": torch.ones(1, dtype=torch.bfloat16),
+ f"{prefix}.smooth_orig": torch.ones(1, dtype=torch.bfloat16),
+ f"{prefix}.lora_down": torch.ones(1, 1, dtype=torch.bfloat16),
+ f"{prefix}.lora_up": torch.ones(1, 1, dtype=torch.bfloat16),
+ f"{prefix}.bias": torch.ones(1, dtype=torch.bfloat16),
+ }
+
+
+def _small_adanorm_tensors(prefix):
+ return {
+ f"{prefix}.qweight": torch.ones(1, 1, dtype=torch.int32),
+ f"{prefix}.wscales": torch.ones(1, 1, dtype=torch.bfloat16),
+ f"{prefix}.wzeros": torch.ones(1, 1, dtype=torch.bfloat16),
+ f"{prefix}.bias": torch.ones(1, dtype=torch.bfloat16),
+ }
+
+
+def _small_standard_complete_tensors():
+ tensors = _small_complete_top_level_tensors()
+ for index in range(19):
+ block = f"transformer_blocks.{index}"
+ for linear in (
+ "qkv_proj",
+ "qkv_proj_context",
+ "out_proj",
+ "out_proj_context",
+ "mlp_fc1",
+ "mlp_fc2",
+ "mlp_context_fc1",
+ "mlp_context_fc2",
+ ):
+ tensors.update(_small_linear_tensors(f"{block}.{linear}"))
+ tensors.update(_small_adanorm_tensors(f"{block}.norm1.linear"))
+ tensors.update(_small_adanorm_tensors(f"{block}.norm1_context.linear"))
+ for norm in ("norm_q", "norm_k", "norm_added_q", "norm_added_k"):
+ tensors[f"{block}.{norm}.weight"] = torch.ones(1, dtype=torch.bfloat16)
+ for index in range(38):
+ block = f"single_transformer_blocks.{index}"
+ for linear in ("qkv_proj", "out_proj", "mlp_fc1", "mlp_fc2"):
+ tensors.update(_small_linear_tensors(f"{block}.{linear}"))
+ tensors.update(_small_adanorm_tensors(f"{block}.norm.linear"))
+ for norm in ("norm_q", "norm_k"):
+ tensors[f"{block}.{norm}.weight"] = torch.ones(1, dtype=torch.bfloat16)
+ return tensors
+
+
+def test_standard_complete_schema_validates_2604_tiny_tensors_without_model_allocation():
+ model = _model({"num_layers": 19, "num_single_layers": 38})
+ adapter = FluxSVDQuantNunchakuAdapter(require_complete_model=True)
+ tensors = _small_standard_complete_tensors()
+
+ assert len(tensors) == 2604
+ adapter.validate(tensors, adapter.metadata(model, 32))
+
+
+def test_complete_extra_collection_requires_and_copies_exact_top_level_parameters():
+ model = _model({"num_layers": 0, "num_single_layers": 0})
+ expected = _small_complete_top_level_tensors()
+ for key, tensor in expected.items():
+ _install_parameter(model, key, tensor)
+
+ actual = FluxSVDQuantNunchakuAdapter(require_complete_model=True).extra_tensors(model)
+
+ assert set(actual) == set(expected)
+ assert all(tensor.dtype == torch.bfloat16 for tensor in actual.values())
+
+
+def test_partial_flux_collect_and_save_roundtrip(tmp_path):
+ from safetensors import safe_open
+
+ from auto_round.algorithms.transforms.svdquant.wrapper import SVDQuantLinear
+
+ residual = torch.nn.Linear(32, 8)
+ residual.data_type, residual.bits, residual.group_size, residual.sym = "mx_fp4", 4, 32, True
+ residual.act_data_type, residual.act_bits, residual.act_group_size = "mx_fp4", 4, 32
+ residual.act_sym, residual.act_dynamic = True, True
+ wrapped = SVDQuantLinear(
+ residual,
+ torch.nn.Linear(32, 2, bias=False),
+ torch.nn.Linear(2, 8, bias=False),
+ torch.linspace(0.5, 1.5, 32),
+ )
+ model = _model({"num_layers": 1, "num_single_layers": 0})
+ _install(model, "transformer_blocks.0.attn.to_out.0", wrapped)
+ _install(model, "x_embedder", torch.nn.Linear(8, 8))
+ adapter = FluxSVDQuantNunchakuAdapter(require_complete_model=False)
+ config = SVDQuantExportConfig(runtime_loadable=True)
+
+ collected = collect_svdquant_tensors(model, adapter=adapter, config=config)
+ path = tmp_path / "flux.safetensors"
+ save_svdquant_nunchaku_safetensors(model, str(path), adapter=adapter, config=config)
+
+ with safe_open(path, framework="pt") as handle:
+ assert set(handle.keys()) == set(collected)
+ assert handle.metadata()["model_class"] == "NunchakuFluxTransformer2dModel"
+ assert handle.get_tensor("x_embedder.weight").dtype == torch.bfloat16
diff --git a/test/unit/test_cpu/export/test_svdquant_mxfp4.py b/test/unit/test_cpu/export/test_svdquant_mxfp4.py
new file mode 100644
index 000000000..299a8c490
--- /dev/null
+++ b/test/unit/test_cpu/export/test_svdquant_mxfp4.py
@@ -0,0 +1,55 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+
+import pytest
+import torch
+
+from auto_round.data_type.mxfp import quant_mx_rceil
+from auto_round.export.svdquant_mxfp4 import (
+ NunchakuMXFP4Packer,
+ pack_lowrank_weight,
+ unpack_lowrank_weight,
+)
+
+
[email protected]("down,shape", [(True, (3, 17)), (False, (17, 3))])
+def test_lowrank_pack_roundtrips_and_matches_fixed_layout(down, shape):
+ logical = torch.arange(shape[0] * shape[1], dtype=torch.float16).reshape(shape)
+ expected_hash = {
+ True: "f62c895e44a7139fc942941b1244857d65143dfe52ad852e8847339aa6119029",
+ False: "0690f5c24d1ca25ad4f7714d9ce6b626b792ac89e46986d7a73dd0f327b163b4",
+ }[down]
+
+ packed = pack_lowrank_weight(logical, down=down)
+ unpacked = unpack_lowrank_weight(packed, down=down)
+
+ assert packed.shape == (128, 16)
+ torch.testing.assert_close(unpacked[: shape[0], : shape[1]], logical)
+ assert hashlib.sha256(bytes(packed.view(torch.uint8).flatten().tolist())).hexdigest() == expected_hash
+
+
[email protected]("shape", [(128, 128), (7, 65)])
+def test_nunchaku_mxfp4_pack_roundtrip_matches_autoround_rceil_qdq(shape):
+ weight = torch.randn(shape, generator=torch.Generator().manual_seed(20260713)) * 3
+ expected, _, _ = quant_mx_rceil(weight, bits=4, group_size=32, data_type="mx_fp4e2m1")
+ packer = NunchakuMXFP4Packer()
+
+ packed = packer.pack_residual(weight)
+ actual = packer.unpack_residual(packed.qweight, packed.wscales, packed.logical_shape)
+
+ assert packed.padded_shape[0] % 128 == 0
+ assert packed.padded_shape[1] % 128 == 0
+ torch.testing.assert_close(actual, expected)
diff --git a/test/unit/test_cpu/export/test_svdquant_nunchaku_export.py b/test/unit/test_cpu/export/test_svdquant_nunchaku_export.py
new file mode 100644
index 000000000..2e433fad6
--- /dev/null
+++ b/test/unit/test_cpu/export/test_svdquant_nunchaku_export.py
@@ -0,0 +1,247 @@
+import json
+
+import pytest
+import torch
+from safetensors import safe_open
+
+from auto_round.algorithms.transforms.svdquant.wrapper import SVDQuantLinear
+from auto_round.export.svdquant_mxfp4 import unpack_lowrank_weight
+from auto_round.export.svdquant_nunchaku import (
+ MXFP4ResidualTensorProvider,
+ SVDQuantExportConfig,
+ SVDQuantExportRecord,
+ collect_svdquant_tensors,
+ pack_nunchaku_16bit_vector,
+ save_svdquant_nunchaku_safetensors,
+ unpack_nunchaku_16bit_vector,
+)
+
+
+def _toy_model(*, rank=3, bias=True, in_features=65, out_features=7):
+ residual = torch.nn.Linear(in_features, out_features, bias=bias)
+ residual.data_type = "mx_fp4e2m1"
+ residual.bits = 4
+ residual.group_size = 32
+ residual.sym = True
+ residual.act_data_type = "mx_fp4e2m1"
+ residual.act_bits = 4
+ residual.act_group_size = 32
+ residual.act_sym = True
+ residual.act_dynamic = True
+ lora_down = torch.nn.Linear(in_features, rank, bias=False)
+ lora_up = torch.nn.Linear(rank, out_features, bias=False)
+ smooth = torch.arange(1, in_features + 1, dtype=torch.float32)
+ layer = SVDQuantLinear(residual, lora_down, lora_up, smooth)
+ return torch.nn.Sequential(layer)
+
+
+def test_default_collection_emits_runtime_layout_tensors_without_debug_residual():
+ model = _toy_model()
+
+ tensors = collect_svdquant_tensors(model)
+
+ assert set(tensors) == {
+ "0.qweight",
+ "0.wscales",
+ "0.smooth",
+ "0.smooth_orig",
+ "0.lora_down",
+ "0.lora_up",
+ "0.bias",
+ }
+ assert tensors["0.qweight"].shape == (128, 64)
+ assert tensors["0.qweight"].dtype == torch.int8
+ assert tensors["0.wscales"].shape == (4, 128)
+ assert tensors["0.wscales"].dtype == torch.uint8
+ for key in ("0.smooth", "0.smooth_orig", "0.lora_down", "0.lora_up", "0.bias"):
+ assert tensors[key].dtype == torch.bfloat16
+ assert tensors["0.lora_down"].shape == (128, 16)
+ assert tensors["0.lora_up"].shape == (128, 16)
+ logical_down = model[0].lora_down.weight.detach().to(torch.bfloat16)
+ logical_up = model[0].lora_up.weight.detach().to(torch.bfloat16)
+ unpacked_down = unpack_lowrank_weight(tensors["0.lora_down"], down=True)
+ unpacked_up = unpack_lowrank_weight(tensors["0.lora_up"], down=False)
+ expected_runtime_down = logical_down * model[0].smooth.to(torch.bfloat16).unsqueeze(0)
+ torch.testing.assert_close(unpacked_down[:3, :65], expected_runtime_down)
+ torch.testing.assert_close(unpacked_up[:7, :3], logical_up)
+ torch.testing.assert_close(
+ unpack_nunchaku_16bit_vector(tensors["0.smooth"])[:65], model[0].smooth.reciprocal().to(torch.bfloat16)
+ )
+ torch.testing.assert_close(
+ unpack_nunchaku_16bit_vector(tensors["0.bias"])[:7],
+ model[0].residual_linear.bias.detach().to(torch.bfloat16),
+ )
+
+
+def test_bias_and_smooth_vectors_match_layout_fixture_and_identity_padding():
+ model = _toy_model(bias=False)
+ model[0].register_buffer("smooth_orig", torch.arange(1, 66, dtype=torch.float32))
+
+ tensors = collect_svdquant_tensors(model)
+
+ bias = unpack_nunchaku_16bit_vector(tensors["0.bias"])
+ assert torch.count_nonzero(bias[:7]) == 0
+ assert torch.equal(bias[7:], torch.ones_like(bias[7:]))
+ smooth_orig = unpack_nunchaku_16bit_vector(tensors["0.smooth_orig"])
+ torch.testing.assert_close(
+ smooth_orig[:65], torch.arange(1, 66, dtype=torch.float32).reciprocal().to(torch.bfloat16)
+ )
+ assert torch.equal(smooth_orig[65:], torch.ones_like(smooth_orig[65:]))
+ fixture = pack_nunchaku_16bit_vector(torch.arange(128, dtype=torch.float16))
+ assert fixture[:32].tolist() == [
+ 0.0,
+ 1.0,
+ 8.0,
+ 9.0,
+ 2.0,
+ 3.0,
+ 10.0,
+ 11.0,
+ 4.0,
+ 5.0,
+ 12.0,
+ 13.0,
+ 6.0,
+ 7.0,
+ 14.0,
+ 15.0,
+ 16.0,
+ 17.0,
+ 24.0,
+ 25.0,
+ 18.0,
+ 19.0,
+ 26.0,
+ 27.0,
+ 20.0,
+ 21.0,
+ 28.0,
+ 29.0,
+ 22.0,
+ 23.0,
+ 30.0,
+ 31.0,
+ ]
+
+
+def test_config_rejects_non_nunchaku_formats():
+ cases = (
+ ("weight_dtype", "mx_fp4e2m1", "weight_dtype"),
+ ("activation_dtype", "fp16", "activation_dtype"),
+ ("scale_dtype", "fp16", "scale_dtype"),
+ ("group_size", 16, "group_size"),
+ ("low_rank_dtype", torch.float32, "low_rank_dtype"),
+ ("debug_unpacked", 1, "debug_unpacked"),
+ )
+ for field, value, message in cases:
+ with pytest.raises(ValueError, match=message):
+ SVDQuantExportConfig(**{field: value})
+
+
+def test_collection_rejects_incompatible_selected_scheme():
+ cases = (
+ ("data_type", "int", "data_type"),
+ ("bits", 8, "bits=4"),
+ ("group_size", 64, "scalar group_size=32"),
+ ("sym", False, "sym=True"),
+ ("act_data_type", "int", "activation data_type"),
+ ("act_bits", 16, "activation bits=4"),
+ ("act_group_size", 64, "activation scalar group_size=32"),
+ ("act_sym", False, "activation sym=True"),
+ ("act_dynamic", False, "act_dynamic=True"),
+ )
+ for field, value, message in cases:
+ model = _toy_model()
+ setattr(model[0].residual_linear, field, value)
+ with pytest.raises(ValueError, match=message):
+ collect_svdquant_tensors(model)
+
+ with pytest.raises(ValueError, match="group_size must be 32"):
+ MXFP4ResidualTensorProvider(group_size=True)
+
+
+def test_collection_rejects_nonfinite_values_and_mixed_ranks():
+ nonfinite = _toy_model()
+ nonfinite[0].smooth[0] = torch.nan
+ with pytest.raises(ValueError, match="finite"):
+ collect_svdquant_tensors(nonfinite)
+
+ mixed = torch.nn.Sequential(_toy_model(rank=2)[0], _toy_model(rank=3)[0])
+ with pytest.raises(ValueError, match="mixed SVDQuant ranks"):
+ collect_svdquant_tensors(mixed)
+
+
+def test_save_rejects_empty_model_and_missing_runtime_metadata_before_writing(tmp_path):
+ empty_path = tmp_path / "empty.safetensors"
+ with pytest.raises(ValueError, match="No SVDQuantLinear"):
+ save_svdquant_nunchaku_safetensors(torch.nn.Linear(2, 2), empty_path)
+ assert not empty_path.exists()
+
+ runtime_path = tmp_path / "runtime.safetensors"
+ with pytest.raises(ValueError, match="model_class.*serialized 'config'"):
+ save_svdquant_nunchaku_safetensors(
+ _toy_model(), runtime_path, config=SVDQuantExportConfig(runtime_loadable=True)
+ )
+ assert not runtime_path.exists()
+
+
+def test_collection_rejects_malformed_packed_residual_payloads():
+ cases = (
+ ({"qweight": torch.zeros(128, 64), "wscales": torch.zeros(4, 128, dtype=torch.uint8)}, "qweight"),
+ ({"qweight": torch.zeros(128, 64, dtype=torch.int8)}, "qweight.*wscales"),
+ (
+ {
+ "qweight": torch.zeros(64, 64, dtype=torch.int8),
+ "wscales": torch.zeros(4, 128, dtype=torch.uint8),
+ },
+ "qweight shape",
+ ),
+ (
+ {
+ "qweight": torch.zeros(128, 64, dtype=torch.int8),
+ "wscales": torch.zeros(3, 128, dtype=torch.uint8),
+ },
+ "wscales shape",
+ ),
+ )
+ for payload, message in cases:
+
+ class Provider:
+ def tensors_for(self, record):
+ return payload
+
+ with pytest.raises(ValueError, match=message):
+ collect_svdquant_tensors(_toy_model(), residual_provider=Provider())
+
+
+def test_save_svdquant_nunchaku_safetensors_writes_metadata(tmp_path):
+ output_path = tmp_path / "svdquant.safetensors"
+
+ save_svdquant_nunchaku_safetensors(_toy_model(), str(output_path))
+
+ with safe_open(output_path, framework="pt") as handle:
+ keys = set(handle.keys())
+ metadata = handle.metadata()
+ quantization_config = json.loads(metadata["quantization_config"])
+
+ assert "0.qweight" in keys
+ assert "0.wscales" in keys
+ assert "0.residual.weight" not in keys
+ assert metadata == {
+ "artifact_type": "generic_intermediate",
+ "quantization_config": json.dumps(
+ {
+ "method": "svdquant",
+ "weight": {"dtype": "fp4_e2m1_all", "scale_dtype": "ue8m0", "group_size": 32},
+ "activation": {"dtype": "fp4_e2m1_all", "scale_dtype": "ue8m0", "group_size": 32},
+ "rank": 3,
+ },
+ sort_keys=True,
+ ),
+ }
+ assert quantization_config == {
+ "method": "svdquant",
+ "weight": {"dtype": "fp4_e2m1_all", "scale_dtype": "ue8m0", "group_size": 32},
+ "activation": {"dtype": "fp4_e2m1_all", "scale_dtype": "ue8m0", "group_size": 32},
+ "rank": 3,
+ }
diff --git a/test/unit/test_cpu/export/test_svdquant_nunchaku_format.py b/test/unit/test_cpu/export/test_svdquant_nunchaku_format.py
new file mode 100644
index 000000000..f84663744
--- /dev/null
+++ b/test/unit/test_cpu/export/test_svdquant_nunchaku_format.py
@@ -0,0 +1,184 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+import torch
+from safetensors.torch import save_file
+
+from auto_round.compressors.base import BaseCompressor
+from auto_round.export.formats.backends.svdquant_nunchaku import SVDQuantNunchakuFormat
+from auto_round.formats import get_formats
+from auto_round.schemes import PRESET_SCHEMES
+
+
+def _mxfp4_compressor(**updates):
+ values = PRESET_SCHEMES["MXFP4"].to_dict()
+ values.update(scheme="MXFP4", **updates)
+ return SimpleNamespace(**values)
+
+
+def test_svdquant_nunchaku_rejects_incompatible_scheme():
+ scheme = PRESET_SCHEMES["MXFP4"].copy()
+ scheme.group_size = 64
+
+ with pytest.raises(ValueError, match=r"group_size=64.*group_size=32"):
+ SVDQuantNunchakuFormat.check_scheme_args(scheme)
+
+
+def test_format_rejects_incompatible_residual_override():
+ from auto_round.algorithms.transforms.svdquant.wrapper import SVDQuantLinear
+
+ residual = torch.nn.Linear(32, 32)
+ model = torch.nn.Module()
+ model.svd = SVDQuantLinear(
+ residual,
+ torch.nn.Linear(32, 1, bias=False),
+ torch.nn.Linear(1, 32, bias=False),
+ torch.ones(32),
+ )
+ output_format = get_formats("svdquant_nunchaku", _mxfp4_compressor())[0]
+
+ with pytest.raises(ValueError, match=r"group_size=64.*group_size=32"):
+ output_format._validate_svd_layer_overrides(model, {"svd.residual_linear": {"group_size": 64}})
+
+
+def test_full_model_format_disables_immediate_packing_and_saving():
+ output_format = get_formats("svdquant_nunchaku", _mxfp4_compressor())[0]
+ assert output_format.format_name == "svdquant_nunchaku"
+ assert output_format.is_supported_immediate_packing() is False
+ assert output_format.is_supported_immediate_saving() is False
+ compressor = SimpleNamespace(
+ formats=[output_format],
+ inplace=True,
+ has_qlayer_outside_block=False,
+ need_calib=True,
+ model_context=SimpleNamespace(model=torch.nn.Module(), is_mllm=False),
+ compress_context=SimpleNamespace(
+ low_cpu_mem_usage=True,
+ is_immediate_packing=True,
+ is_immediate_saving=True,
+ ),
+ quantize_config=SimpleNamespace(data_type="mx_fp"),
+ output_dir="unused",
+ _ensure_shard_writer=lambda: pytest.fail("full-model export must not create a shard writer"),
+ )
+
+ BaseCompressor._adjust_immediate_packing_and_saving(compressor)
+
+ assert compressor.compress_context.is_immediate_packing is False
+ assert compressor.compress_context.is_immediate_saving is False
+
+
+def test_format_uses_flux_adapter_and_diffusers_weight_name(monkeypatch, tmp_path):
+ import auto_round.export.svdquant_nunchaku as exporter
+ from auto_round.export.svdquant_adapters.flux import FluxSVDQuantNunchakuAdapter
+
+ output_format = get_formats("svdquant_nunchaku", _mxfp4_compressor())[0]
+
+ class FluxModel(torch.nn.Module):
+ config = {"_class_name": "FluxTransformer2DModel", "num_layers": 0, "num_single_layers": 0}
+
+ def save_config(self, output_dir):
+ Path(output_dir).mkdir(parents=True, exist_ok=True)
+ Path(output_dir, "config.json").write_text(json.dumps(self.config), encoding="utf-8")
+
+ model = FluxModel()
+ captured = {}
+
+ def fake_export(export_model, output_path, *, config, residual_provider, adapter):
+ captured.update(
+ model=export_model,
+ output_path=output_path,
+ runtime_loadable=config.runtime_loadable,
+ residual_provider=residual_provider,
+ adapter=adapter,
+ )
+
+ monkeypatch.setattr(exporter, "save_svdquant_nunchaku_safetensors", fake_export)
+
+ result = output_format.save_quantized(tmp_path, model=model, model_adapter="flux", device="cpu")
+
+ assert result is model
+ assert captured["output_path"] == str(tmp_path / "diffusion_pytorch_model.safetensors")
+ assert captured["runtime_loadable"] is True
+ assert isinstance(captured["adapter"], FluxSVDQuantNunchakuAdapter)
+ assert (tmp_path / "config.json").is_file()
+
+
+def test_diffusion_save_exports_self_contained_nunchaku_pipeline(tmp_path):
+ from auto_round.compressors.diffusion_mixin import DiffusionMixin
+
+ class QuantizedTransformer(torch.nn.Module):
+ def save_config(self, output_dir):
+ output_dir = Path(output_dir)
+ output_dir.mkdir(parents=True, exist_ok=True)
+ (output_dir / "config.json").write_text('{"_class_name": "FluxTransformer2DModel"}', encoding="utf-8")
+
+ class Bf16Component:
+ def save_pretrained(self, output_dir):
+ output_dir = Path(output_dir)
+ output_dir.mkdir(parents=True, exist_ok=True)
+ (output_dir / "config.json").write_text("{}", encoding="utf-8")
+ (output_dir / "diffusion_pytorch_model.safetensors").touch()
+
+ transformer = QuantizedTransformer()
+ vae = Bf16Component()
+
+ class Pipeline:
+ def __init__(self):
+ self.transformer = transformer
+ self.vae = vae
+ self.components = {"transformer": transformer, "vae": vae}
+
+ def save_config(self, output_dir):
+ Path(output_dir, "model_index.json").write_text(
+ json.dumps(
+ {
+ "_class_name": "FluxPipeline",
+ "transformer": ["diffusers", "FluxTransformer2DModel"],
+ "vae": ["diffusers", "AutoencoderKL"],
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ class ExportParent:
+ def save_quantized(self, output_dir, **kwargs):
+ Path(output_dir).mkdir(parents=True, exist_ok=True)
+ save_file(
+ {"probe": torch.zeros(1)},
+ f"{output_dir}/diffusion_pytorch_model.safetensors",
+ metadata={"model_class": "NunchakuFluxTransformer2dModel"},
+ )
+ return self.model_context.model
+
+ class Compressor(DiffusionMixin, ExportParent):
+ pass
+
+ compressor = Compressor.__new__(Compressor)
+ compressor.formats = [SimpleNamespace(format_name="svdquant_nunchaku")]
+ compressor.model_context = SimpleNamespace(pipe=Pipeline(), model=transformer)
+ compressor.compress_context = SimpleNamespace(is_immediate_saving=False)
+
+ compressor.save_quantized(tmp_path)
+
+ assert (tmp_path / "transformer" / "diffusion_pytorch_model.safetensors").is_file()
+ assert (tmp_path / "vae" / "diffusion_pytorch_model.safetensors").is_file()
+ model_index = json.loads((tmp_path / "model_index.json").read_text(encoding="utf-8"))
+ assert model_index["transformer"] == ["diffusers", "FluxTransformer2DModel"]
+ assert model_index["vae"] == ["diffusers", "AutoencoderKL"]
diff --git a/test/unit/test_cpu/export/test_svdquant_w4a16.py b/test/unit/test_cpu/export/test_svdquant_w4a16.py
new file mode 100644
index 000000000..209fc23e8
--- /dev/null
+++ b/test/unit/test_cpu/export/test_svdquant_w4a16.py
@@ -0,0 +1,44 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from auto_round.export.svdquant_w4a16 import (
+ dequantize_adanorm_w4a16,
+ pack_adanorm_w4a16,
+ unpack_adanorm_w4a16,
+)
+
+
+def _representable_fixture(dtype=torch.float16):
+ rows = torch.arange(12).reshape(12, 1)
+ columns = torch.arange(1024).reshape(1, 1024)
+ signed = ((rows * 5 + columns * 3) % 15 - 7).to(torch.float32)
+ scales = (torch.arange(12 * 16).reshape(12, 16) % 13 + 1).to(dtype) / 8
+ weight = (signed.reshape(12, 16, 64) * scales.unsqueeze(-1)).reshape(12, 1024).to(dtype)
+ return weight, scales, signed.to(torch.int8)
+
+
+def test_adanorm_w4a16_pack_uses_runtime_layout_and_roundtrips_codes():
+ weight, scales, signed = _representable_fixture()
+
+ packed = pack_adanorm_w4a16(weight, scales, splits=3)
+
+ assert packed.qweight.shape == (3, 512)
+ assert packed.wscales.shape == (16, 12)
+ assert packed.wzeros.shape == (16, 12)
+ expected_codes = signed.reshape(3, 4, 1024).permute(1, 0, 2).reshape(12, 1024)
+ expected_weight = weight.reshape(3, 4, 1024).permute(1, 0, 2).reshape(12, 1024)
+ assert torch.equal(unpack_adanorm_w4a16(packed), expected_codes)
+ torch.testing.assert_close(dequantize_adanorm_w4a16(packed), expected_weight)
diff --git a/test/unit/test_cpu/models/test_diffusion.py b/test/unit/test_cpu/models/test_diffusion.py
index d61100f97..48075aabc 100644
--- a/test/unit/test_cpu/models/test_diffusion.py
+++ b/test/unit/test_cpu/models/test_diffusion.py
@@ -7,10 +7,69 @@ import torch
from packaging import version
from auto_round import AutoRound
+from auto_round.calibration.diffusion import _prepare_pipeline_for_calibration
flux_name_or_path = get_model_path("black-forest-labs/FLUX.1-dev")
+def test_low_gpu_memory_diffusion_calibration_uses_model_cpu_offload():
+ class Pipeline:
+ device = torch.device("cpu")
+
+ def __init__(self):
+ self.offload_device = None
+
+ def enable_model_cpu_offload(self, *, device):
+ self.offload_device = device
+
+ def to(self, _device):
+ raise AssertionError("low GPU memory calibration must not move the full pipeline")
+
+ pipe = Pipeline()
+
+ mode = _prepare_pipeline_for_calibration(pipe, "cuda:0", low_gpu_mem_usage=True)
+
+ assert mode == "model"
+ assert pipe.offload_device == torch.device("cuda:0")
+
+
+def test_regular_diffusion_calibration_moves_pipeline_to_device():
+ class Pipeline:
+ device = None
+
+ def __init__(self):
+ self.target_device = None
+
+ def to(self, device):
+ self.target_device = device
+
+ pipe = Pipeline()
+ target_device = torch.device("cpu")
+
+ mode = _prepare_pipeline_for_calibration(pipe, target_device, low_gpu_mem_usage=False)
+
+ assert mode is None
+ assert pipe.target_device == target_device
+
+
+def test_regular_diffusion_calibration_moves_all_components_when_pipeline_reports_target_device():
+ class Pipeline:
+ device = torch.device("cuda:0")
+
+ def __init__(self):
+ self.target_device = None
+
+ def to(self, device):
+ self.target_device = device
+
+ pipe = Pipeline()
+ target_device = torch.device("cuda:0")
+
+ _prepare_pipeline_for_calibration(pipe, target_device, low_gpu_mem_usage=False)
+
+ assert pipe.target_device == target_device
+
+
@pytest.fixture
def setup_flux():
"""Fixture to set up the Flux model and tokenizer."""
diff --git a/test/unit/test_cpu/utils/test_cli_usage.py b/test/unit/test_cpu/utils/test_cli_usage.py
index bcdfba09b..11b31f22d 100644
--- a/test/unit/test_cpu/utils/test_cli_usage.py
+++ b/test/unit/test_cpu/utils/test_cli_usage.py
@@ -339,6 +339,85 @@ def test_legacy_disable_flags_map_to_enable_bools():
assert args.enable_quanted_input is False
+def test_svdquant_cli_builds_hyphenated_options_before_rtn():
+ from auto_round.algorithms.transforms.svdquant.config import SVDQuantConfig
+ from auto_round.cli.algorithms import AlgorithmHandler
+ from auto_round.cli.parser import build_quantize_parser
+
+ parser = build_quantize_parser()
+ args = parser.parse_args(
+ [
+ "--model",
+ "dummy-model",
+ "--algorithm",
+ "svdquant,rtn",
+ "--svdquant-rank",
+ "16",
+ "--enable-svdquant-smooth",
+ "--svdquant-smooth-num-grids",
+ "39",
+ "--svdquant-smooth-max-calibration-calls",
+ "64",
+ "--svdquant-residual-iters",
+ "20",
+ "--enable-svdquant-residual-early-stop",
+ "--svdquant-low-rank-dtype",
+ "fp32",
+ "--svdquant-target-modules",
+ "attn,ff",
+ "--svdquant-exclude-modules",
+ "proj_out",
+ "--svdquant-model-adapter",
+ "flux",
+ "--disable_opt_rtn",
+ ]
+ )
+
+ configs = AlgorithmHandler.build_configs(args, {})
+
+ assert isinstance(configs[0], SVDQuantConfig)
+ assert configs[0].rank == 16
+ assert configs[0].smooth_enabled is True
+ assert configs[0].smooth_num_grids == 39
+ assert configs[0].smooth_max_calibration_calls == 64
+ assert configs[0].residual_iters == 20
+ assert configs[0].residual_early_stop is True
+ assert configs[0].low_rank_dtype == "fp32"
+ assert configs[0].target_modules == ["attn", "ff"]
+ assert configs[0].exclude_modules == ["proj_out"]
+ assert configs[0].model_adapter == "flux"
+ assert configs[1].__class__.__name__ == "RTNConfig"
+
+
+def test_svdquant_cli_rejects_underscore_option_aliases():
+ from auto_round.cli.parser import build_quantize_parser
+
+ with pytest.raises(SystemExit):
+ build_quantize_parser().parse_args(
+ ["--model", "dummy-model", "--algorithm", "svdquant,rtn", "--svdquant_rank", "16"]
+ )
+
+
+def test_svdquant_cli_defaults_compose_before_signround():
+ from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
+ from auto_round.algorithms.transforms.svdquant.config import SVDQuantConfig
+ from auto_round.cli.algorithms import AlgorithmHandler
+ from auto_round.cli.parser import build_quantize_parser
+
+ args = build_quantize_parser().parse_args(
+ ["--model", "dummy-model", "--algorithm", "svdquant,auto_round", "--iters", "200"]
+ )
+
+ configs = AlgorithmHandler.build_configs(args, {})
+
+ assert isinstance(configs[0], SVDQuantConfig)
+ assert configs[0].rank == 32
+ assert configs[0].smooth_enabled is False
+ assert configs[0].residual_iters == 1
+ assert configs[0].model_adapter == "auto"
+ assert isinstance(configs[1], SignRoundConfig)
+
+
def _normalize_options(raw):
if raw is None:
return None
diff --git a/test/unit/test_cpu/utils/test_utils.py b/test/unit/test_cpu/utils/test_utils.py
index f818e376e..795356083 100644
--- a/test/unit/test_cpu/utils/test_utils.py
+++ b/test/unit/test_cpu/utils/test_utils.py
@@ -2,6 +2,7 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
+import torch
import auto_round.utils.device as auto_round_utils
from auto_round.utils.common import (
@@ -9,7 +10,19 @@ from auto_round.utils.common import (
preserve_original_visual_block_name,
revert_checkpoint_conversion_mapping,
)
-from auto_round.utils.model import is_code_model
+from auto_round.utils.model import is_code_model, map_nested_tensors
+
+
+def test_map_nested_tensors_preserves_structure_without_mutating_input():
+ original = {"args": (torch.tensor([1.0]),), "metadata": ["unchanged"]}
+
+ mapped = map_nested_tensors(original, lambda tensor: tensor + 1)
+
+ assert mapped is not original
+ assert mapped["args"] is not original["args"]
+ torch.testing.assert_close(mapped["args"][0], torch.tensor([2.0]))
+ torch.testing.assert_close(original["args"][0], torch.tensor([1.0]))
+ assert mapped["metadata"] == ["unchanged"]
@pytest.mark.parametrize(