Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Support Nemotron-H Omni wrappers (Nemotron 3.5 Super VL, Bolt 74B-A9B) in plain AutoRound calls (#2512)

SergiioBec10a649b073
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index 09df68453..f2073847e 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -315,6 +315,52 @@ def get_tokenizer_function(tokenizer, seqlen, apply_chat_template=False, system_
return default_tokenizer_function
+@register_dataset(["nvidia/Nemotron-Pretraining-Dataset-sample", "nemotron-pretraining"])
+def get_nemotron_pretraining_dataset(
+ tokenizer,
+ seqlen,
+ dataset_name="nvidia/Nemotron-Pretraining-Dataset-sample",
+ split=None,
+ seed=42,
+ apply_chat_template=False,
+ system_prompt=None,
+):
+ """Calibration data from Nvidia's Nemotron pretraining corpus (public sample).
+
+ Uses the Nemotron-CC-High-Quality config — filtered high-quality web text from
+ the same distribution as Nemotron-H / Nemotron-Cascade-2 pretraining.
+ """
+ split = "train" if split is None else split
+ if isinstance(split, list):
+ if len(split) != 1:
+ raise ValueError("Nemotron-Pretraining-Dataset-sample supports only one split at a time.")
+ split = split[0]
+ tokenizer_function = get_tokenizer_function(
+ tokenizer, seqlen, apply_chat_template=apply_chat_template, system_prompt=system_prompt
+ )
+ try:
+ calib_dataset = load_dataset(
+ dataset_name,
+ name="Nemotron-CC-High-Quality",
+ split=split,
+ streaming=True,
+ )
+ calib_dataset = calib_dataset.shuffle(seed=seed).take(10000)
+ calib_dataset = Dataset.from_list(list(calib_dataset))
+ except Exception as e:
+ logger.error(f"Failed to load {dataset_name}: {e}")
+ sys.exit(1)
+ calib_dataset = calib_dataset.shuffle(seed=seed)
+ calib_dataset = calib_dataset.map(
+ tokenizer_function,
+ batched=True,
+ new_fingerprint=_make_map_fingerprint(
+ calib_dataset, tokenizer, seqlen, apply_chat_template, system_prompt, "text"
+ ),
+ )
+ return calib_dataset
+
+
@register_dataset(["NeelNanda/pile-10k", "pile-10k"])
def get_pile_dataset(
tokenizer,
diff --git a/auto_round/compressors/layer_config_resolver.py b/auto_round/compressors/layer_config_resolver.py
index 8d281adf5..490f64c59 100644
--- a/auto_round/compressors/layer_config_resolver.py
+++ b/auto_round/compressors/layer_config_resolver.py
@@ -233,8 +233,24 @@ def _resolve_layer_config_presets(
"fixed_by_user": True,
}
+ # 1b. Merge predefined model defaults; user entries win per-pattern and per-key.
+ from auto_round.special_model_handler import get_predefined_layer_config
+
+ predefined_only = set()
+ for _pattern, _overlay in get_predefined_layer_config(model).items():
+ _existing = layer_config.get(_pattern)
+ if isinstance(_existing, dict):
+ for _k, _v in _overlay.items():
+ _existing.setdefault(_k, _v)
+ elif _existing is None:
+ layer_config[_pattern] = dict(_overlay)
+ predefined_only.add(_pattern)
+
# 2. normalize
layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}
+ # Entries added only from predefined model defaults are not user choices.
+ for _pattern in predefined_only:
+ layer_config[_pattern]["fixed_by_user"] = False
# entries whose sym was set by the user (dict key or a QuantizationScheme
# object) rather than inherited from the default scheme below
explicit_sym = {name for name, cfg in layer_config.items() if "sym" in cfg}
diff --git a/auto_round/context/model.py b/auto_round/context/model.py
index a4c6c03a4..b5c1bb2a8 100644
--- a/auto_round/context/model.py
+++ b/auto_round/context/model.py
@@ -27,6 +27,7 @@ from auto_round import envs
from auto_round.compressors.utils import get_shared_keys
from auto_round.context.base import BaseContext
from auto_round.logger import logger
+from auto_round.modeling.nemotron_h_omni import apply_nemotron_h_omni_post_load
from auto_round.modeling.unfused_moe import apply_model_monkey_patches
from auto_round.special_model_handler import _handle_special_model, update_module
from auto_round.utils import (
@@ -121,7 +122,7 @@ class ModelContext(BaseContext):
# by the time BaseCompressor.post_init() runs.
self._load_model()
- if unsupported_meta_device(self.model):
+ if self._disk_stream_index is None and unsupported_meta_device(self.model):
raise RuntimeError(
"AutoRound does not support parameters on meta device. "
"Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."
@@ -196,13 +197,22 @@ class ModelContext(BaseContext):
self.model, self.tokenizer, self._disk_stream_index = self._build_disk_stream_model(
self.disk_stream_model_dir
)
+ loaded_via_meta = True
from transformers import AutoProcessor
- self.processor = AutoProcessor.from_pretrained(
- self.disk_stream_model_dir, trust_remote_code=self.trust_remote_code
- )
- self.image_processor = getattr(self.processor, "image_processor", None)
- loaded_via_meta = True
+ try:
+ self.processor = AutoProcessor.from_pretrained(
+ self.disk_stream_model_dir, trust_remote_code=self.trust_remote_code
+ )
+ self.image_processor = getattr(self.processor, "image_processor", None)
+ except Exception:
+ logger.warning(
+ "AutoProcessor failed for %s; keeping the meta skeleton (text-only calibration).",
+ self.disk_stream_model_dir,
+ exc_info=True,
+ )
+ self.processor = None
+ self.image_processor = None
except Exception:
logger.warning(
"Building a multimodal meta skeleton for %s failed; " "falling back to a full CPU load.",
@@ -269,6 +279,9 @@ class ModelContext(BaseContext):
elif self.tokenizer is None and not self.is_diffusion and self.need_calib:
raise ValueError("A tokenizer must be set for non-str model input")
+ # Nemotron-H Omni wrappers: restore FP32 SSM/router tensors and the gated-norm
+ # group_size after loading. No-op for every other model.
+ apply_nemotron_h_omni_post_load(self.model)
self._model_loaded = True
# Clear tuning_device from any previous quantization passes.
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index b2f2cf979..de5c8410f 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -53,6 +53,21 @@ from auto_round.utils import (
)
+def _quant_linear_accepts(qlinear_cls, kwarg_name: str) -> bool:
+ target_cls = qlinear_cls
+ if isinstance(target_cls, functools.partial):
+ target_cls = target_cls.func
+ try:
+ sig = inspect.signature(target_cls.__init__)
+ except (TypeError, ValueError):
+ return False
+ params = sig.parameters
+ if kwarg_name in params:
+ return True
+ return any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
+
+
def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_bits=16):
"""
Dynamically imports and returns the appropriate QuantLinear class based on the specified backend and parameters.
@@ -127,8 +142,18 @@ def pack_qact_layer(name, model):
out_features = layer.weight.shape[1]
bias = layer.bias is not None
use_pc = False
+ extra_kwargs: dict[str, torch.dtype] = {}
+ if isinstance(scale, torch.Tensor) and _quant_linear_accepts(QuantLinear, "scale_dtype"):
+ extra_kwargs["scale_dtype"] = scale.dtype
new_layer = QuantLinear( ##pylint: disable=E1123
- bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype, use_pc=use_pc
+ bits,
+ group_size,
+ in_features,
+ out_features,
+ bias,
+ weight_dtype=layer.weight.dtype,
+ use_pc=use_pc,
+ **extra_kwargs,
)
new_layer.device = device
set_module(model, name, new_layer)
@@ -217,8 +242,11 @@ def pack_layer(layer_name, model, backend, device=None):
bias = layer.bias is not None
+ extra_kwargs: dict[str, torch.dtype] = {}
+ if isinstance(scale, torch.Tensor) and _quant_linear_accepts(QuantLinear, "scale_dtype"):
+ extra_kwargs["scale_dtype"] = scale.dtype
new_layer = QuantLinear( ##pylint: disable=E1123
- bits, group_size, in_features, out_features, bias=bias, weight_dtype=layer.weight.dtype
+ bits, group_size, in_features, out_features, bias=bias, weight_dtype=layer.weight.dtype, **extra_kwargs
)
new_layer.device = orig_device
set_module(model, layer_name, new_layer)
diff --git a/auto_round/export/export_to_autoround/qlinear_triton_act.py b/auto_round/export/export_to_autoround/qlinear_triton_act.py
index 57deb924d..fe49d7a4a 100644
--- a/auto_round/export/export_to_autoround/qlinear_triton_act.py
+++ b/auto_round/export/export_to_autoround/qlinear_triton_act.py
@@ -49,7 +49,17 @@ logger = getLogger(__name__)
class QuantLinear(nn.Module):
QUANT_TYPE = "triton"
- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):
+ def __init__(
+ self,
+ bits,
+ group_size,
+ infeatures,
+ outfeatures,
+ bias,
+ trainable=False,
+ scale_dtype=torch.bfloat16,
+ **kwargs,
+ ):
super().__init__()
if bits not in [2, 4, 8]:
raise NotImplementedError("Only 2,4,8 bits are supported.")
@@ -60,6 +70,7 @@ class QuantLinear(nn.Module):
self.bits = bits
self.group_size = group_size if group_size != -1 else infeatures
self.maxq = 2**self.bits - 1
+ self.scale_dtype = scale_dtype
self.register_buffer(
"qweight",
@@ -79,7 +90,7 @@ class QuantLinear(nn.Module):
"scales",
torch.zeros(
(math.ceil(infeatures / self.group_size), outfeatures),
- dtype=torch.bfloat16,
+ dtype=self.scale_dtype,
),
)
use_pc = kwargs.pop("use_pc", False)
@@ -126,7 +137,7 @@ class QuantLinear(nn.Module):
self.w_bf16_to_fp8_scale.data.copy_(w_bf16_to_fp8_scale.squeeze().clone())
if linear.bias is not None:
self.bias = linear.bias.clone().half()
- self.scales = scales_t.clone().half()
+ self.scales = scales_t.clone().to(self.scales.dtype)
W = linear.weight.data.to(device).clone()
if type(linear) == nn.Conv2d:
diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.py
index 07ae7b67b..3b3db968e 100644
--- a/auto_round/modeling/fused_moe/moe_experts_interface.py
+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py
@@ -842,6 +842,12 @@ def prepare_model_for_moe_quantization(
impl_to_set = saved_impl if saved_impl in AUTO_ROUND_EXPERTS_IMPLS else implementation
model.config._experts_implementation = impl_to_set
logger.debug(f"Set model.config._experts_implementation = '{impl_to_set}'")
+ # Experts dispatch on their own ``self.config``; in a multimodal wrapper that is the
+ # nested text config (e.g. Nemotron-H Omni's ``llm_config``), not ``model.config``.
+ for name in unfused_modules:
+ expert_config = getattr(model.get_submodule(name), "config", None)
+ if expert_config is not None and expert_config is not model.config:
+ expert_config._experts_implementation = impl_to_set
memory_monitor.update()
memory_monitor.log_summary("After applying custom replacements")
diff --git a/auto_round/modeling/nemotron_h_omni.py b/auto_round/modeling/nemotron_h_omni.py
new file mode 100644
index 000000000..1f7cd4dc8
--- /dev/null
+++ b/auto_round/modeling/nemotron_h_omni.py
@@ -0,0 +1,250 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Load-time support for NVIDIA's Nemotron-H Omni wrapper checkpoints.
+
+``nemotron_h_omni`` (``NemotronH_Omni_Reasoning_V3``, remote code) wraps a Nemotron-H
+language model -- native ``nemotron_h`` (e.g. Nemotron 3.5 Super VL) or the remote
+``nemotron_h_puzzle`` variant (e.g. Nemotron 3.5 Bolt) -- plus a RADIO vision tower.
+Plain ``nemotron_h`` text checkpoints are not touched by anything in this module.
+"""
+
+import glob
+import json
+import os
+import re
+from collections.abc import Iterable
+
+import torch
+import torch.nn as nn
+
+from auto_round.logger import logger
+from auto_round.utils.model import get_module
+
+NEMOTRON_H_OMNI_MODEL_TYPE = "nemotron_h_omni"
+
+# On disk the wrapper's language-model trunk is ``language_model.backbone.*``; the module
+# tree exposes it as ``language_model.model.*``. The inner model's own ``backbone.`` ->
+# ``model.`` rule cannot do this rename for the nested trunk: reversed, ``model.`` also
+# matches the tail of ``language_model.``. Use the full prefix instead.
+_NESTED_TRUNK_SOURCE = "language_model.backbone."
+_NESTED_TRUNK_TARGET = "language_model.model."
+
+_SSM_CORE_PATTERNS = (r"\.mixer\.A_log$", r"\.mixer\.D$", r"\.mixer\.dt_bias$")
+_ROUTER_BIAS_PATTERNS = (r"\.mixer\.gate\.e_score_correction_bias$",)
+# Composition-level renames registered by the checkpoints' remote code
+# (``register_nemotron_h_omni_conversion_mapping`` in ``modeling_nemotron_h_omni.py``).
+_REMOTE_OMNI_RENAMES = (
+ (r"^mlp1\.0\.", r"vision_projector\.mlp1\.norm\."),
+ (r"^mlp1\.1\.", r"vision_projector\.mlp1\.linear1\."),
+ (r"^mlp1\.3\.", r"vision_projector\.mlp1\.linear2\."),
+ (r"^sound_encoder\.", r"sound_projector\.sound_encoder\."),
+ (r"^sound_projection\.", r"sound_projector\.sound_projection\."),
+)
+
+
+def _cfg_get(config, key, default=None):
+ if isinstance(config, dict):
+ return config.get(key, default)
+ return getattr(config, key, default)
+
+
+def is_nemotron_h_omni_config(config) -> bool:
+ """Whether ``config`` (object or ``config.json`` dict) is a Nemotron-H Omni wrapper."""
+ return config is not None and _cfg_get(config, "model_type") == NEMOTRON_H_OMNI_MODEL_TYPE
+
+
+def prefer_remote_omni_class(config, trust_remote_code: bool) -> bool:
+ """Whether to build the wrapper from the checkpoint's remote code instead of a native class.
+
+ Transformers >= 5.20 ships a native class with the same name, whose config rejects the
+ remote ``nemotron_h_puzzle`` language model (Bolt). Wrappers around a native ``nemotron_h``
+ (Super VL) keep the default class resolution.
+ """
+ if not (trust_remote_code and is_nemotron_h_omni_config(config)):
+ return False
+ llm_config = _cfg_get(config, "llm_config")
+ auto_map = _cfg_get(config, "auto_map") or {}
+ return _cfg_get(llm_config, "model_type") == "nemotron_h_puzzle" and "AutoModelForCausalLM" in auto_map
+
+
+def nemotron_h_omni_load_kwargs(config) -> dict:
+ """``from_pretrained`` kwargs for a full (non-streamed) load of an Omni wrapper.
+
+ Transformers scopes a sub-model's registered renames to that sub-model's prefix, so the
+ nested-trunk rename is passed as a root-level ``key_mapping`` instead.
+ """
+ if not is_nemotron_h_omni_config(config):
+ return {}
+ return {"key_mapping": {r"^language_model\.backbone\.": _NESTED_TRUNK_TARGET}}
+
+
+def register_nemotron_h_omni_checkpoint_mapping(config, trust_remote_code: bool = False) -> bool:
+ """Register the conversion rules an Omni wrapper needs; idempotent, no-op for other models.
+
+ * The nested-trunk rename is appended to the inner model type's mapping, so the upstream
+ rules stay in place; it only matches ``language_model.backbone.*`` names, which exist
+ only in Omni wrapper checkpoints.
+ * When the remote class is used (see :func:`prefer_remote_omni_class`), Transformers would
+ look its class name up first and find the *native* class's mapping (projector renamed to
+ ``multi_modal_projector``), so the projector would silently stay random. Map the class
+ name to the remote code's own composition-level renames instead.
+ """
+ if not is_nemotron_h_omni_config(config):
+ return False
+ try:
+ from transformers.conversion_mapping import (
+ get_checkpoint_conversion_mapping,
+ register_checkpoint_conversion_mapping,
+ )
+ from transformers.core_model_loading import WeightRenaming
+ except ImportError: # transformers < 5
+ return False
+ if prefer_remote_omni_class(config, trust_remote_code):
+ for class_name in _cfg_get(config, "architectures") or []:
+ if get_checkpoint_conversion_mapping(class_name) is not None:
+ register_checkpoint_conversion_mapping(
+ class_name, [WeightRenaming(src, dst) for src, dst in _REMOTE_OMNI_RENAMES], overwrite=True
+ )
+ llm_config = _cfg_get(config, "llm_config") or _cfg_get(config, "text_config")
+ text_model_type = _cfg_get(llm_config, "model_type") if llm_config is not None else None
+ if text_model_type is None:
+ return False
+ rules = list(get_checkpoint_conversion_mapping(text_model_type) or [])
+ if any(
+ getattr(rule, "source_patterns", None) == [_NESTED_TRUNK_SOURCE]
+ and getattr(rule, "target_patterns", None) == [_NESTED_TRUNK_TARGET]
+ for rule in rules
+ ):
+ return True
+ rules.append(WeightRenaming(_NESTED_TRUNK_SOURCE, _NESTED_TRUNK_TARGET))
+ register_checkpoint_conversion_mapping(text_model_type, rules, overwrite=True)
+ logger.info("Registered %s -> %s for %s", _NESTED_TRUNK_SOURCE, _NESTED_TRUNK_TARGET, text_model_type)
+ return True
+
+
+def _text_config(config):
+ for attr in ("text_config", "llm_config"):
+ inner = getattr(config, attr, None)
+ if inner is not None:
+ return inner
+ return config
+
+
+def _patch_zamba2_group_size(model: nn.Module) -> int:
+ """Restore ``Zamba2RMSNormGated.group_size``, which a meta/low-memory load can drop."""
+ config = _text_config(model.config)
+ try:
+ group_size = config.mamba_num_heads * config.mamba_head_dim // config.n_groups
+ except (AttributeError, ZeroDivisionError):
+ return 0
+ patched = 0
+ for module in model.modules():
+ if module.__class__.__name__ != "Zamba2RMSNormGated":
+ continue
+ if getattr(module, "group_size", None) != group_size:
+ object.__setattr__(module, "group_size", group_size)
+ patched += 1
+ return patched
+
+
+def _source_shard_map(source_dir: str) -> dict[str, str]:
+ """Map checkpoint tensor name -> shard path (index file, single file, or shard headers)."""
+ from safetensors import safe_open
+
+ index_path = os.path.join(source_dir, "model.safetensors.index.json")
+ if os.path.exists(index_path):
+ with open(index_path) as f:
+ weight_map = json.load(f).get("weight_map", {})
+ if weight_map:
+ return {k: os.path.join(source_dir, shard) for k, shard in weight_map.items()}
+ shards = [os.path.join(source_dir, "model.safetensors")]
+ if not os.path.exists(shards[0]):
+ shards = sorted(glob.glob(os.path.join(source_dir, "model-*.safetensors")))
+ source_to_shard = {}
+ for shard in shards:
+ with safe_open(shard, framework="pt", device="cpu") as f:
+ for k in f.keys():
+ source_to_shard[k] = shard
+ return source_to_shard
+
+
+def _resolve_source_dir(name_or_path) -> str | None:
+ """Local checkpoint directory for a path or an (already downloaded) hub id."""
+ if not name_or_path:
+ return None
+ if os.path.isdir(name_or_path):
+ return name_or_path
+ try:
+ from huggingface_hub import snapshot_download
+
+ return snapshot_download(name_or_path, allow_patterns=["*.safetensors", "*.safetensors.index.json"])
+ except Exception as exc: # noqa: BLE001 - offline / gated: keep the loaded dtype
+ logger.warning("nemotron_h_omni: cannot resolve %s (%s); FP32 restore skipped", name_or_path, exc)
+ return None
+
+
+def _restore_tensors_from_source(
+ model: nn.Module, source_dir: str, name_patterns: Iterable[str], dtype: torch.dtype
+) -> int:
+ """Reload matching tensors from the checkpoint as ``dtype`` (they are stored in FP32)."""
+ from safetensors import safe_open
+
+ compiled = [re.compile(p) for p in name_patterns]
+ by_shard: dict[str, list[str]] = {}
+ for key, shard in _source_shard_map(source_dir).items():
+ if any(c.search(key) for c in compiled):
+ by_shard.setdefault(shard, []).append(key)
+ restored = 0
+ for shard, keys in by_shard.items():
+ with safe_open(shard, framework="pt", device="cpu") as f:
+ for src_key in keys:
+ module_path = src_key.replace(_NESTED_TRUNK_SOURCE, _NESTED_TRUNK_TARGET, 1)
+ parent_path, attr = module_path.rsplit(".", 1)
+ parent = get_module(model, parent_path)
+ old = getattr(parent, attr, None) if parent is not None else None
+ if not isinstance(old, torch.Tensor):
+ continue
+ src = f.get_tensor(src_key)
+ if old.shape != src.shape:
+ logger.warning("nemotron_h_omni: shape mismatch for %s, skipping restore", module_path)
+ continue
+ src = src.to(dtype)
+ if isinstance(old, nn.Parameter):
+ setattr(parent, attr, nn.Parameter(src, requires_grad=old.requires_grad))
+ else:
+ persistent = attr not in getattr(parent, "_non_persistent_buffers_set", set())
+ parent.register_buffer(attr, src, persistent=persistent)
+ restored += 1
+ return restored
+
+
+def apply_nemotron_h_omni_post_load(model: nn.Module) -> dict:
+ """Post-load fix-ups for Omni wrappers; no-op for every other model.
+
+ Restores the Mamba2 recurrence parameters (``A_log``/``D``/``dt_bias``) and the router
+ ``e_score_correction_bias`` from the checkpoint in FP32 (the low-precision load
+ truncates them), and re-applies the gated-norm ``group_size``.
+ """
+ summary = {"zamba2_patched": 0, "high_precision_restored": 0}
+ config = getattr(model, "config", None)
+ if not is_nemotron_h_omni_config(config) or getattr(model, "_autoround_nh_post_load_applied", False):
+ return summary
+ summary["zamba2_patched"] = _patch_zamba2_group_size(model)
+ source_dir = _resolve_source_dir(_cfg_get(config, "_name_or_path"))
+ if source_dir is not None:
+ patterns = _SSM_CORE_PATTERNS + _ROUTER_BIAS_PATTERNS
+ summary["high_precision_restored"] = _restore_tensors_from_source(model, source_dir, patterns, torch.float32)
+ logger.info("nemotron_h_omni: restored %d tensor(s) in FP32", summary["high_precision_restored"])
+ model._autoround_nh_post_load_applied = True
+ return summary
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 41c66099f..ffb135acf 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -1167,6 +1167,20 @@ register_ignore_layers(
)
+# Architecture-specific layer_config defaults, keyed by ``config.model_type``.
+# Nemotron-H Omni wrappers (Super VL, Bolt) keep their Mamba2 ``out_proj`` scales in
+# BF16; every other model, including plain ``nemotron_h``, keeps the caller's scale_dtype.
+_PRE_DEFINED_LAYER_CONFIG = {
+ "nemotron_h_omni": {".*mixer.out_proj$": {"scale_dtype": torch.bfloat16}},
+}
+
+
+def get_predefined_layer_config(model: torch.nn.Module) -> dict:
+ """Return predefined layer_config patterns for the model; user entries take precedence."""
+ model_type = getattr(getattr(model, "config", None), "model_type", None)
+ return _PRE_DEFINED_LAYER_CONFIG.get(model_type, {})
+
+
def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:
layers = []
for rule in _PRE_DEFINED_IGNORE_LAYERS:
diff --git a/auto_round/utils/disk_stream_util.py b/auto_round/utils/disk_stream_util.py
index c9e687c1d..6e4204e17 100644
--- a/auto_round/utils/disk_stream_util.py
+++ b/auto_round/utils/disk_stream_util.py
@@ -464,7 +464,8 @@ def _resolve_checkpoint_name(index, full_name: str):
if index.has_tensor(candidate):
return candidate
for model_type in _index_model_types(index):
- for transform in _reverse_renamings_for(model_type):
+ transforms = _reverse_renamings_for(model_type)
+ for transform in transforms:
try:
candidate, _ = transform.rename_source_key(full_name)
except Exception as exc: # pragma: no cover - defensive, never fail a load here
@@ -472,6 +473,17 @@ def _resolve_checkpoint_name(index, full_name: str):
continue
if candidate != full_name and index.has_tensor(candidate):
return candidate
+ # Some names need several of a family's renames at once, e.g. RADIO's
+ # ``encoder.layer.N.attention.output.dense`` <- ``radio_model.model.blocks.N.attn.proj``.
+ candidate = full_name
+ for transform in transforms:
+ try:
+ candidate, _ = transform.rename_source_key(candidate)
+ except Exception as exc: # pragma: no cover - defensive, never fail a load here
+ logger.debug("chained rename %s via %s failed: %s", candidate, transform, exc)
+ continue
+ if candidate != full_name and index.has_tensor(candidate):
+ return candidate
return None
@@ -1143,8 +1155,16 @@ def build_meta_model(model_name: str, trust_remote_code: bool = True, unfuse_moe
# reap/layerwise_prune.py's disk-streamed model builder.
import transformers as _transformers
+ from auto_round.modeling.nemotron_h_omni import (
+ prefer_remote_omni_class,
+ register_nemotron_h_omni_checkpoint_mapping,
+ )
+
+ register_nemotron_h_omni_checkpoint_mapping(config, trust_remote_code)
archs = getattr(config, "architectures", None) or []
- model_cls = next((getattr(_transformers, a) for a in archs if hasattr(_transformers, a)), None)
+ model_cls = None
+ if not prefer_remote_omni_class(config, trust_remote_code):
+ model_cls = next((getattr(_transformers, a) for a in archs if hasattr(_transformers, a)), None)
with init_empty_weights():
if model_cls is not None:
model = model_cls(config)
diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.py
index b894a5eff..7bd494e81 100644
--- a/auto_round/utils/missing_tensors.py
+++ b/auto_round/utils/missing_tensors.py
@@ -319,11 +319,7 @@ def copy_missing_tensors_from_source(
shortcut_block_prefix: set = {name.split(".", 1)[1] for name in saved_block_prefix if "." in name}
saved_block_prefix.update(shortcut_block_prefix)
- def _is_truly_missing(name: str) -> bool:
- # Special case: Qwen/Qwen3-0.6B-FP8
- # lm_head is tied but still in source_dir → not missing
- if name == "lm_head.weight":
- return False
+ def _is_missing_by_name(name: str) -> bool:
if name in saved_tensor_names:
return False
parent = name.rsplit(".", 1)[0]
@@ -335,6 +331,23 @@ def copy_missing_tensors_from_source(
return block not in saved_block_prefix
return True
+ nested_trunk_source, nested_trunk_saved = "language_model.backbone.", "language_model.model."
+
+ def _is_truly_missing(name: str) -> bool:
+ # Special case: Qwen/Qwen3-0.6B-FP8
+ # lm_head is tied but still in source_dir → not missing
+ if name == "lm_head.weight":
+ return False
+ if not _is_missing_by_name(name):
+ return False
+ # Nemotron-H Omni checkpoints store the language-model trunk as
+ # ``language_model.backbone.*`` while the quantized model saves it as
+ # ``language_model.model.*``; without this alias every quantized LM block
+ # would be re-copied from the source in BF16.
+ if name.startswith(nested_trunk_source):
+ return _is_missing_by_name(nested_trunk_saved + name[len(nested_trunk_source) :])
+ return True
+
missing_tensor_names: list = [name for name in source_tensor_to_file if _is_truly_missing(name)]
if not missing_tensor_names:
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index d10196e7e..559224db6 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -655,6 +655,11 @@ def mllm_load_model(
model_dtype: str | None = None,
**kwargs,
):
+ from auto_round.modeling.nemotron_h_omni import (
+ nemotron_h_omni_load_kwargs,
+ prefer_remote_omni_class,
+ register_nemotron_h_omni_checkpoint_mapping,
+ )
from auto_round.special_model_handler import MISTRAL_3_2_MODELS
_check_accelerate_version()
@@ -725,6 +730,7 @@ def mllm_load_model(
model_type = config["model_type"]
else:
model_type = None
+ register_nemotron_h_omni_checkpoint_mapping(config, trust_remote_code)
if model_type == "qwen2_5_omni":
if version.parse(transformers.__version__) < version.parse("4.52.0"):
@@ -850,7 +856,9 @@ def mllm_load_model(
torch_dtype=torch_dtype,
)
else:
- if architectures.endswith("Model") and hasattr(
+ if prefer_remote_omni_class(config, trust_remote_code):
+ cls = AutoModelForCausalLM
+ elif architectures.endswith("Model") and hasattr(
base_lib, n := architectures.replace("Model", "ForConditionalGeneration")
):
cls = getattr(base_lib, n)
@@ -859,7 +867,7 @@ def mllm_load_model(
else:
cls = AutoModelForCausalLM
try:
- model_load_kwargs = {}
+ model_load_kwargs = nemotron_h_omni_load_kwargs(config)
if model_subfolder is not None:
model_load_kwargs["subfolder"] = model_subfolder
model = cls.from_pretrained(
diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.py
index cc364327c..869effd5b 100644
--- a/auto_round_extension/torch/qlinear_torch.py
+++ b/auto_round_extension/torch/qlinear_torch.py
@@ -115,7 +115,18 @@ class QuantLinear(GenericBitPackingMixin, nn.Module):
QUANT_TYPE = "torch"
- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, g_idx=False, **kwargs):
+ def __init__(
+ self,
+ bits,
+ group_size,
+ infeatures,
+ outfeatures,
+ bias,
+ trainable=False,
+ g_idx=False,
+ scale_dtype=torch.float16,
+ **kwargs,
+ ):
super().__init__()
if bits not in SUPPORTED_BITS:
raise NotImplementedError(f"Only {','.join(map(str, SUPPORTED_BITS))} bits are supported.")
@@ -124,6 +135,7 @@ class QuantLinear(GenericBitPackingMixin, nn.Module):
self.bits = bits
self.group_size = group_size if group_size != -1 else infeatures
self.maxq = 2**self.bits - 1
+ self.scale_dtype = scale_dtype
self.register_buffer(
"qweight",
@@ -143,7 +155,7 @@ class QuantLinear(GenericBitPackingMixin, nn.Module):
"scales",
torch.zeros(
(math.ceil(infeatures / self.group_size), outfeatures),
- dtype=torch.float16,
+ dtype=self.scale_dtype,
),
)
if bias:
@@ -181,7 +193,7 @@ class QuantLinear(GenericBitPackingMixin, nn.Module):
scales_t = scales.t().contiguous()
if linear.bias is not None:
self.bias = linear.bias.clone().half()
- self.scales = scales_t.clone().half()
+ self.scales = scales_t.clone().to(self.scales.dtype)
W = linear.weight.data.to(device).clone()
if type(linear) == nn.Conv2d:
@@ -245,7 +257,7 @@ class QuantLinear(GenericBitPackingMixin, nn.Module):
scales_t = scales.t().contiguous()
if linear.bias is not None:
self.bias = linear.bias.clone().half()
- self.scales = scales_t.clone().half()
+ self.scales = scales_t.clone().to(self.scales.dtype)
W = linear.weight.data.to(device).clone()
if type(linear) == nn.Conv2d:
diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.py
index 117997ce7..9b1e92942 100644
--- a/auto_round_extension/torch/qlinear_torch_zp.py
+++ b/auto_round_extension/torch/qlinear_torch_zp.py
@@ -38,7 +38,18 @@ class QuantLinear(GenericBitPackingMixin, nn.Module):
QUANT_TYPE = "torch"
- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, g_idx=False, **kwargs):
+ def __init__(
+ self,
+ bits,
+ group_size,
+ infeatures,
+ outfeatures,
+ bias,
+ trainable=False,
+ g_idx=False,
+ scale_dtype=torch.float16,
+ **kwargs,
+ ):
super().__init__()
if bits not in SUPPORTED_BITS:
raise NotImplementedError(f"Only {','.join(map(str, SUPPORTED_BITS))} bits are supported.")
@@ -47,6 +58,7 @@ class QuantLinear(GenericBitPackingMixin, nn.Module):
self.bits = bits
self.group_size = group_size if group_size != -1 else infeatures
self.maxq = 2**self.bits - 1
+ self.scale_dtype = scale_dtype
self.register_buffer(
"qweight",
@@ -71,7 +83,7 @@ class QuantLinear(GenericBitPackingMixin, nn.Module):
"scales",
torch.zeros(
(math.ceil(infeatures / self.group_size), outfeatures),
- dtype=torch.float16,
+ dtype=self.scale_dtype,
),
)
if bias:
@@ -104,7 +116,7 @@ class QuantLinear(GenericBitPackingMixin, nn.Module):
scales_t = scales.t().contiguous()
if linear.bias is not None:
self.bias = linear.bias.clone().half()
- self.scales = scales_t.clone().half()
+ self.scales = scales_t.clone().to(self.scales.dtype)
W = linear.weight.data.to(device).clone()
if type(linear) == nn.Conv2d:
@@ -175,7 +187,7 @@ class QuantLinear(GenericBitPackingMixin, nn.Module):
scales_t = scales.t().contiguous()
if linear.bias is not None:
self.bias = linear.bias.clone().half()
- self.scales = scales_t.clone().half()
+ self.scales = scales_t.clone().to(self.scales.dtype)
W = linear.weight.data.to(device).clone()
if type(linear) == nn.Conv2d:
diff --git a/test/unit/common/models/test_nemotron_h.py b/test/unit/common/models/test_nemotron_h.py
new file mode 100644
index 000000000..696e6abe6
--- /dev/null
+++ b/test/unit/common/models/test_nemotron_h.py
@@ -0,0 +1,294 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""CPU regressions for Nemotron-H Omni wrapper loading (Super VL / Bolt)."""
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+from safetensors.torch import save_file
+from transformers import NemotronHConfig, conversion_mapping
+
+from auto_round.modeling.nemotron_h_omni import (
+ apply_nemotron_h_omni_post_load,
+ prefer_remote_omni_class,
+ register_nemotron_h_omni_checkpoint_mapping,
+)
+from auto_round.modeling.unfused_moe import apply_model_monkey_patches
+
+_OMNI_AUTO_MAP = {"AutoModelForCausalLM": "modeling_nemotron_h_omni.NemotronH_Omni_Reasoning_V3"}
+
+
+def _omni_config(text_model_type="nemotron_h_puzzle"):
+ return {"model_type": "nemotron_h_omni", "auto_map": _OMNI_AUTO_MAP, "llm_config": {"model_type": text_model_type}}
+
+
+def mapping_registry(monkeypatch):
+ """Isolate transformers' global conversion-mapping registry."""
+ store = {}
+ monkeypatch.setattr(conversion_mapping, "get_checkpoint_conversion_mapping", lambda t: list(store.get(t, [])))
+
+ def register(model_type, mapping, overwrite=False):
+ store[model_type] = list(mapping)
+
+ monkeypatch.setattr(conversion_mapping, "register_checkpoint_conversion_mapping", register)
+ return store
+
+
[email protected]("text_model_type", ["nemotron_h_puzzle", "nemotron_h"])
+def test_omni_rename_is_appended_once(mapping_registry, text_model_type):
+ upstream = object()
+ mapping_registry[text_model_type] = [upstream]
+ assert register_nemotron_h_omni_checkpoint_mapping(_omni_config(text_model_type))
+ assert register_nemotron_h_omni_checkpoint_mapping(_omni_config(text_model_type))
+ rules = mapping_registry[text_model_type]
+ assert rules[0] is upstream and len(rules) == 2
+ assert rules[1].source_patterns == ["language_model.backbone."]
+ assert rules[1].target_patterns == ["language_model.model."]
+
+
[email protected]("config", [{"model_type": "nemotron_h"}, {"model_type": "qwen3_moe"}, None])
+def test_omni_rename_skips_other_models(mapping_registry, config):
+ assert not register_nemotron_h_omni_checkpoint_mapping(config)
+ assert mapping_registry == {}
+
+
+ "text_model_type,trust,overridden",
+ [("nemotron_h_puzzle", True, True), ("nemotron_h_puzzle", False, False), ("nemotron_h", True, False)],
+)
+def test_remote_omni_class_name_mapping(mapping_registry, text_model_type, trust, overridden):
+ """Transformers 5.20's native same-name class mapping must not rename the remote projector."""
+ native = object()
+ class_name = "NemotronH_Omni_Reasoning_V3"
+ mapping_registry[class_name] = [native]
+ config = {**_omni_config(text_model_type), "architectures": [class_name]}
+ register_nemotron_h_omni_checkpoint_mapping(config, trust_remote_code=trust)
+ rules = mapping_registry[class_name]
+ if not overridden:
+ assert rules == [native]
+ return
+ renamed = {}
+ for src in ("mlp1.0.weight", "mlp1.1.weight", "mlp1.3.weight"):
+ renamed[src] = next(r.rename_source_key(src)[0] for r in rules if r.rename_source_key(src)[0] != src)
+ assert renamed == {
+ "mlp1.0.weight": "vision_projector.mlp1.norm.weight",
+ "mlp1.1.weight": "vision_projector.mlp1.linear1.weight",
+ "mlp1.3.weight": "vision_projector.mlp1.linear2.weight",
+ }
+
+
+def test_remote_class_preferred_only_for_trusted_puzzle_omni():
+ assert prefer_remote_omni_class(_omni_config(), trust_remote_code=True)
+ assert prefer_remote_omni_class(SimpleNamespace(**_omni_config()), trust_remote_code=True)
+ assert not prefer_remote_omni_class(_omni_config(), trust_remote_code=False)
+ assert not prefer_remote_omni_class(_omni_config("nemotron_h"), trust_remote_code=True)
+ assert not prefer_remote_omni_class({"model_type": "nemotron_h_omni", "llm_config": {}}, trust_remote_code=True)
+ assert not prefer_remote_omni_class({"model_type": "nemotron_h", "auto_map": _OMNI_AUTO_MAP}, True)
+
+
+def test_full_load_passes_root_level_trunk_rename():
+ from auto_round.modeling.nemotron_h_omni import nemotron_h_omni_load_kwargs
+
+ assert nemotron_h_omni_load_kwargs(_omni_config("nemotron_h")) == {
+ "key_mapping": {r"^language_model\.backbone\.": "language_model.model."}
+ }
+ assert nemotron_h_omni_load_kwargs({"model_type": "nemotron_h"}) == {}
+
+
+def test_disk_stream_resolves_names_needing_several_renames(monkeypatch):
+ """RADIO's attention output projection needs two of its family's renames at once."""
+ from transformers.core_model_loading import WeightRenaming
+
+ from auto_round.utils import disk_stream_util
+
+ rules = (
+ WeightRenaming("radio_model.model.blocks", "encoder.layer").reverse_transform(),
+ WeightRenaming("attn.proj", "attention.output.dense").reverse_transform(),
+ )
+ monkeypatch.setattr(disk_stream_util, "_index_model_types", lambda index: ("radio",))
+ monkeypatch.setattr(disk_stream_util, "_reverse_renamings_for", lambda model_type: rules)
+ monkeypatch.setattr(disk_stream_util, "_renamed_expert_candidates", lambda index, name: ())
+ stored = {"vision_model.radio_model.model.blocks.0.attn.proj.weight", "vision_model.radio_model.model.blocks.0.x"}
+ index = SimpleNamespace(has_tensor=stored.__contains__)
+ resolve = disk_stream_util._resolve_checkpoint_name
+ assert resolve(index, "vision_model.encoder.layer.0.attention.output.dense.weight") == (
+ "vision_model.radio_model.model.blocks.0.attn.proj.weight"
+ )
+ assert resolve(index, "vision_model.encoder.layer.0.x") == "vision_model.radio_model.model.blocks.0.x"
+ assert resolve(index, "vision_model.encoder.layer.1.attention.output.dense.weight") is None
+
+
+def _wrapper(model_type, source_dir):
+ model = torch.nn.Module()
+ model.config = SimpleNamespace(
+ model_type=model_type,
+ _name_or_path=str(source_dir),
+ llm_config=SimpleNamespace(mamba_num_heads=4, mamba_head_dim=8, n_groups=2),
+ )
+ model.language_model = torch.nn.Module()
+ model.language_model.model = torch.nn.Module()
+ layer = torch.nn.Module()
+ layer.mixer = torch.nn.Module()
+ layer.mixer.A_log = torch.nn.Parameter(torch.zeros(4, dtype=torch.bfloat16))
+ layer.mixer.gate = torch.nn.Module()
+ layer.mixer.gate.register_buffer("e_score_correction_bias", torch.zeros(3, dtype=torch.bfloat16))
+ model.language_model.model.layers = torch.nn.ModuleList([layer])
+ return model
+
+
+def test_post_load_restores_fp32_tensors_for_omni(tmp_path):
+ a_log = torch.tensor([0.1, 0.2, 0.3, 0.4000001], dtype=torch.float32)
+ bias = torch.tensor([1e-4, 2e-4, 3e-4], dtype=torch.float32)
+ save_file(
+ {
+ "language_model.backbone.layers.0.mixer.A_log": a_log,
+ "language_model.backbone.layers.0.mixer.gate.e_score_correction_bias": bias,
+ "language_model.backbone.layers.0.mixer.in_proj.weight": torch.zeros(2, 2),
+ },
+ str(tmp_path / "model.safetensors"),
+ )
+ model = _wrapper("nemotron_h_omni", tmp_path)
+ assert apply_nemotron_h_omni_post_load(model)["high_precision_restored"] == 2
+ mixer = model.language_model.model.layers[0].mixer
+ assert mixer.A_log.dtype == torch.float32 and torch.equal(mixer.A_log.data, a_log)
+ assert torch.equal(mixer.gate.e_score_correction_bias, bias)
+ # idempotent
+ assert apply_nemotron_h_omni_post_load(model)["high_precision_restored"] == 0
+
+
+def test_post_load_sets_group_size_per_instance_only(tmp_path):
+ """The gated-norm group_size is restored on the instance; the class stays untouched."""
+
+ class Zamba2RMSNormGated(torch.nn.Module):
+ pass
+
+ save_file({"language_model.backbone.layers.0.mixer.A_log": torch.ones(4)}, str(tmp_path / "model.safetensors"))
+ model = _wrapper("nemotron_h_omni", tmp_path)
+ model.language_model.model.layers[0].mixer.norm = Zamba2RMSNormGated()
+ assert apply_nemotron_h_omni_post_load(model)["zamba2_patched"] == 1
+ assert model.language_model.model.layers[0].mixer.norm.group_size == 4 * 8 // 2
+ assert not hasattr(Zamba2RMSNormGated, "group_size")
+ assert not hasattr(Zamba2RMSNormGated(), "group_size")
+
+
[email protected]("model_type", ["nemotron_h", "nemotron_h_puzzle", "qwen3_moe"])
+def test_post_load_is_noop_for_other_models(tmp_path, model_type):
+ save_file({"language_model.backbone.layers.0.mixer.A_log": torch.ones(4)}, str(tmp_path / "model.safetensors"))
+ model = _wrapper(model_type, tmp_path)
+ assert apply_nemotron_h_omni_post_load(model) == {"zamba2_patched": 0, "high_precision_restored": 0}
+ assert model.language_model.model.layers[0].mixer.A_log.dtype == torch.bfloat16
+
+
+def test_plain_nemotron_h_text_model_is_not_patched(tmp_path):
+ """Plain nemotron_h (e.g. Lightning) keeps the default fused-MoE path: no unfused-MoE patch."""
+ NemotronHConfig(hidden_size=16, vocab_size=32, num_attention_heads=2, num_key_value_heads=2).save_pretrained(
+ tmp_path
+ )
+ assert not apply_model_monkey_patches(str(tmp_path))
+
+
+def test_nested_experts_config_gets_linear_experts_implementation():
+ """Unfused experts inside a multimodal wrapper dispatch on their own nested config."""
+ pytest.importorskip("transformers.integrations.moe")
+ from transformers import NemotronHForCausalLM
+
+ from auto_round.modeling.fused_moe.moe_experts_interface import (
+ AUTO_ROUND_EXPERTS_IMPLS,
+ prepare_model_for_moe_quantization,
+ )
+
+ text_config = NemotronHConfig(
+ hidden_size=16,
+ vocab_size=32,
+ num_attention_heads=2,
+ num_key_value_heads=2,
+ layers_block_type=["moe"],
+ num_hidden_layers=1,
+ moe_intermediate_size=12,
+ moe_shared_expert_intermediate_size=12,
+ n_routed_experts=4,
+ num_experts_per_tok=2,
+ n_group=1,
+ topk_group=1,
+ num_nextn_predict_layers=0,
+ )
+ wrapper = torch.nn.Module()
+ wrapper.config = SimpleNamespace(model_type="nemotron_h_omni")
+ wrapper.language_model = NemotronHForCausalLM(text_config)
+ assert prepare_model_for_moe_quantization(wrapper)
+ impl = wrapper.config._experts_implementation
+ assert impl in AUTO_ROUND_EXPERTS_IMPLS
+ assert wrapper.language_model.config._experts_implementation == impl
+ hidden = torch.randn(1, 3, 16)
+ wrapper.language_model.model.layers[0].mixer(hidden) # unfused experts run on the nested config
+
+
[email protected]("processor_fails", [False, True])
+def test_multimodal_disk_stream_keeps_meta_skeleton(monkeypatch, processor_fails):
+ from unittest.mock import Mock
+
+ from auto_round.context import model as context_module
+
+ monkeypatch.setattr(context_module.ModelContext, "_instances", {})
+
+ class Tiny(torch.nn.Module):
+ dtype = torch.float32
+
+ def __init__(self):
+ super().__init__()
+ self.block = torch.nn.Linear(2, 2, device="meta")
+ self.restored_ssm = torch.nn.Parameter(torch.ones(2))
+
+ model, tokenizer, index = Tiny(), object(), object()
+ processor = Mock(image_processor=object())
+ load_processor = Mock(side_effect=ValueError("incompatible processor") if processor_fails else None)
+ load_processor.return_value = processor
+ full_load = Mock(side_effect=AssertionError("must not materialize the full checkpoint"))
+ warning = Mock()
+ materialize = Mock()
+ monkeypatch.setattr(context_module, "is_diffusion_model", lambda *a, **kw: False)
+ monkeypatch.setattr(context_module, "is_mllm_model", lambda *a, **kw: True)
+ monkeypatch.setattr(context_module, "is_moe_model", lambda *a, **kw: False)
+ monkeypatch.setattr(context_module, "install_debug_layer_config_patch", lambda: None)
+ monkeypatch.setattr(context_module, "apply_nemotron_h_omni_post_load", lambda _: None)
+ monkeypatch.setattr(context_module, "mllm_load_model", full_load)
+ monkeypatch.setattr(context_module.logger, "warning", warning)
+ monkeypatch.setattr("transformers.AutoProcessor.from_pretrained", load_processor)
+ monkeypatch.setattr(context_module.ModelContext, "_peek_config", lambda _: None)
+ monkeypatch.setattr(context_module.ModelContext, "_should_use_meta_skeleton", lambda *a: True)
+ monkeypatch.setattr(context_module.ModelContext, "_build_disk_stream_model", lambda *a: (model, tokenizer, index))
+ monkeypatch.setattr(context_module.ModelContext, "_materialize_disk_stream_non_block_params", materialize)
+ monkeypatch.setattr(context_module.ModelContext, "_import_custom_moe_replacements", lambda *a: None)
+ monkeypatch.setattr(
+ context_module.ModelContext, "_set_amp_dtype", lambda self: setattr(self, "amp_dtype", torch.float32)
+ )
+ guard = Mock(side_effect=AssertionError("disk-stream skeleton must bypass the mixed-meta guard"))
+ monkeypatch.setattr(context_module, "unsupported_meta_device", guard)
+ context = context_module.ModelContext(model="tiny-nemotron", need_calib=False)
+ assert context.model is model and context.tokenizer is tokenizer and context._disk_stream_index is index
+ materialize.assert_called_once()
+ full_load.assert_not_called()
+ guard.assert_not_called()
+ if processor_fails:
+ assert context.processor is None and context.image_processor is None
+ assert warning.call_args.kwargs["exc_info"] is True
+ else:
+ assert context.processor is processor and context.image_processor is processor.image_processor
+
+
+def test_non_streamed_meta_model_still_rejected(monkeypatch):
+ from auto_round.context import model as context_module
+
+ monkeypatch.setattr(context_module.ModelContext, "_instances", {})
+
+ monkeypatch.setattr(context_module.ModelContext, "_load_model", lambda _: None)
+ monkeypatch.setattr(context_module, "unsupported_meta_device", lambda _: True)
+ with pytest.raises(RuntimeError, match="does not support parameters on meta"):
+ context_module.ModelContext(model=torch.nn.Linear(2, 2, device="meta"), need_calib=False)
diff --git a/test/unit/common/utils/test_calib_dataset_helpers.py b/test/unit/common/utils/test_calib_dataset_helpers.py
index f02d6a27b..7f273f556 100644
--- a/test/unit/common/utils/test_calib_dataset_helpers.py
+++ b/test/unit/common/utils/test_calib_dataset_helpers.py
@@ -185,6 +185,41 @@ class TestFineWebEduDataset:
dataset.take.assert_called_once_with(10000)
+class TestNemotronPretrainingDataset:
+ @pytest.mark.parametrize("split,expected", [(None, "train"), ("validation", "validation"), (["train"], "train")])
+ def test_split_is_forwarded(self, monkeypatch, split, expected):
+ import auto_round.calib_dataset as calib_dataset
+
+ streamed = MagicMock()
+ streamed.shuffle.return_value = streamed
+ streamed.take.return_value = [{"text": "sample text"}]
+ load_dataset = MagicMock(return_value=streamed)
+ monkeypatch.setattr(calib_dataset, "load_dataset", load_dataset)
+ monkeypatch.setattr(calib_dataset, "_make_map_fingerprint", lambda *a, **k: "nemotron-test")
+ tokenizer = MagicMock(
+ side_effect=lambda texts, **kwargs: {
+ "input_ids": [[1, 2, 3] for _ in texts],
+ "attention_mask": [[1, 1, 1] for _ in texts],
+ }
+ )
+
+ result = calib_dataset.get_nemotron_pretraining_dataset(tokenizer, 128, split=split)
+
+ assert len(result) == 1
+ load_dataset.assert_called_once_with(
+ "nvidia/Nemotron-Pretraining-Dataset-sample",
+ name="Nemotron-CC-High-Quality",
+ split=expected,
+ streaming=True,
+ )
+
+ def test_rejects_multiple_splits(self):
+ import auto_round.calib_dataset as calib_dataset
+
+ with pytest.raises(ValueError, match="one split"):
+ calib_dataset.get_nemotron_pretraining_dataset(MagicMock(), 128, split=["train", "validation"])
+
+
# ---------------------------------------------------------------------------
# _make_map_fingerprint
# ---------------------------------------------------------------------------
diff --git a/test/unit/common/utils/test_layer_config_resolver.py b/test/unit/common/utils/test_layer_config_resolver.py
index a1f1e44f6..9b08ff34e 100644
--- a/test/unit/common/utils/test_layer_config_resolver.py
+++ b/test/unit/common/utils/test_layer_config_resolver.py
@@ -12,6 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from types import SimpleNamespace
+
+import pytest
+import torch
import torch.nn as nn
from auto_round.compressors.config_resolution import ResolvedScheme
@@ -164,3 +168,75 @@ def test_extract_regex_config_honors_format_allowance(monkeypatch):
inner_supported_types=(),
format="auto_round",
)
+
+
+ "model_type,expected_dtype",
+ [
+ ("nemotron_h_omni", torch.bfloat16),
+ ("nemotron_h", torch.float16),
+ ("nemotron_h_puzzle", torch.float16),
+ ("qwen3_moe", torch.float16),
+ ("nemotron", torch.float16),
+ ("nemotron_h_other", torch.float16),
+ (None, torch.float16),
+ ],
+)
+def test_mamba_scale_default_is_only_for_nemotron_h_omni(model_type, expected_dtype):
+ from auto_round.special_model_handler import get_predefined_layer_config
+
+ model = nn.Module()
+ model.config = SimpleNamespace(model_type=model_type)
+ model.mixer = nn.Module()
+ model.mixer.out_proj = nn.Linear(32, 32)
+ model.mixer.in_proj = nn.Linear(32, 32)
+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(act_bits=16, act_data_type="float"))
+ resolved = resolve_layer_config(
+ model=model,
+ scheme=scheme,
+ layer_config={},
+ scale_dtype=torch.float16,
+ supported_types=(nn.Linear,),
+ inner_supported_types=(),
+ quant_block_list=[["mixer"]],
+ )
+ assert resolved["mixer.out_proj"]["scale_dtype"] is expected_dtype
+ assert resolved["mixer.in_proj"]["scale_dtype"] is torch.float16
+ if expected_dtype is torch.float16:
+ assert get_predefined_layer_config(model) == {}
+
+
+def test_nemotron_h_omni_scale_default_respects_explicit_user_dtype():
+ model = nn.Module()
+ model.config = SimpleNamespace(model_type="nemotron_h_omni")
+ model.mixer = nn.Module()
+ model.mixer.out_proj = nn.Linear(32, 32)
+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(act_bits=16, act_data_type="float"))
+ resolved = resolve_layer_config(
+ model=model,
+ scheme=scheme,
+ layer_config={".*mixer.out_proj$": {"scale_dtype": torch.float32}},
+ scale_dtype=torch.float16,
+ supported_types=(nn.Linear,),
+ inner_supported_types=(),
+ )
+ assert resolved["mixer.out_proj"]["scale_dtype"] is torch.float32
+ assert resolved["mixer.out_proj"]["fixed_by_user"] is True
+
+
+def test_nemotron_h_omni_scale_default_is_not_marked_user_fixed():
+ model = nn.Module()
+ model.config = SimpleNamespace(model_type="nemotron_h_omni")
+ model.mixer = nn.Module()
+ model.mixer.out_proj = nn.Linear(32, 32)
+ scheme = ResolvedScheme.from_scheme(QuantizationScheme(act_bits=16, act_data_type="float"))
+ resolved = resolve_layer_config(
+ model=model,
+ scheme=scheme,
+ layer_config={},
+ scale_dtype=torch.float16,
+ supported_types=(nn.Linear,),
+ inner_supported_types=(),
+ )
+ assert resolved["mixer.out_proj"]["scale_dtype"] is torch.bfloat16
+ assert resolved["mixer.out_proj"]["fixed_by_user"] is False
diff --git a/test/unit/common/utils/test_missing_tensors.py b/test/unit/common/utils/test_missing_tensors.py
index e727b4b8e..3601cde03 100644
--- a/test/unit/common/utils/test_missing_tensors.py
+++ b/test/unit/common/utils/test_missing_tensors.py
@@ -443,6 +443,25 @@ class TestCopyMissingTensorsFromSource:
copy_missing_tensors_from_source(src, tgt)
assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))
+ def test_nested_nemotron_omni_lm_not_recopied(self, tmp_path):
+ """Nemotron-H Omni: ``language_model.backbone.*`` on disk is saved as ``language_model.model.*``."""
+ src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")
+ os.makedirs(src)
+ os.makedirs(tgt)
+ _save_safetensors(
+ {"language_model.backbone.layers.0.mixer.experts.0.up_proj.weight": torch.randn(4, 4).bfloat16()},
+ os.path.join(src, "model.safetensors"),
+ )
+ packed = torch.randint(0, 16, (2, 4), dtype=torch.int32)
+ saved_name = "language_model.model.layers.0.mixer.experts.0.up_proj.qweight"
+ _save_safetensors({saved_name: packed}, os.path.join(tgt, "model.safetensors"))
+ _write_config(tgt)
+ copy_missing_tensors_from_source(src, tgt)
+ assert not os.path.exists(os.path.join(tgt, "model_extra_tensors.safetensors"))
+ result = _load_safetensors(os.path.join(tgt, "model.safetensors"))
+ assert set(result) == {saved_name}
+ torch.testing.assert_close(result[saved_name], packed)
+
def test_known_block_prefix_not_copied(self, tmp_path):
src, tgt = str(tmp_path / "src"), str(tmp_path / "tgt")
os.makedirs(src)