Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Nemotron-H fast_inference via vLLM weight sharing (fallback on older unsloth_zoo); drop vLLM-only kwargs on fallback (#13440)

Daniel Han8196cf968918
diff --git a/tests/fast_inference/test_nemotron_h_fallback.py b/tests/fast_inference/test_nemotron_h_fallback.py
new file mode 100644
index 00000000000..cccaee53645
--- /dev/null
+++ b/tests/fast_inference/test_nemotron_h_fallback.py
@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Nemotron-H with fast_inference = True falls back to Unsloth inference when the installed unsloth_zoo cannot rebuild it from vLLM (unslothai/unsloth-zoo#565)."""
+
+import importlib.util
+
+import pytest
+from real_accelerator import (
+ has_real_accelerator,
+) # tests/_shared, on sys.path via tests/conftest.py
+
+pytestmark = pytest.mark.gpu
+
+MODEL_NAME = "trl-internal-testing/tiny-NemotronHForCausalLM-nano"
+
+
[email protected](not has_real_accelerator(), reason = "needs a GPU")
+def test_nemotron_h_needs_a_zoo_that_rebuilds_it(monkeypatch):
+ import unsloth_zoo.empty_model as zoo_empty_model
+ from unsloth.models.loader import _vllm_unsupported_model_types
+
+ monkeypatch.setattr(zoo_empty_model, "extract_nemotron_h_mixer", object(), raising = False)
+ assert _vllm_unsupported_model_types(["nemotron_h"]) == []
+ assert _vllm_unsupported_model_types(["gpt_oss"]) == ["gpt_oss"]
+ monkeypatch.delattr(zoo_empty_model, "extract_nemotron_h_mixer")
+ assert _vllm_unsupported_model_types(["nemotron_h"]) == ["nemotron_h"]
+ assert _vllm_unsupported_model_types(["llama"]) == []
+
+
[email protected](not has_real_accelerator(), reason = "needs a GPU")
[email protected](importlib.util.find_spec("vllm") is None, reason = "needs vLLM installed")
+def test_nemotron_h_fast_inference_falls_back_on_an_older_zoo(capfd, monkeypatch):
+ import unsloth_zoo.empty_model as zoo_empty_model
+ from unsloth import FastLanguageModel
+
+ monkeypatch.delattr(zoo_empty_model, "extract_nemotron_h_mixer", raising = False)
+ model, tokenizer = FastLanguageModel.from_pretrained(
+ model_name = MODEL_NAME,
+ max_seq_length = 256,
+ load_in_4bit = False,
+ fast_inference = True,
+ gpu_memory_utilization = 0.1,
+ enforce_eager = True,
+ max_num_seqs = 8,
+ )
+ out = capfd.readouterr().out
+ assert "does not support nemotron_h yet - will switch to Unsloth inference" in out
+ assert "Ignoring vLLM-only arguments ['enforce_eager', 'max_num_seqs']" in out
+ assert getattr(model, "vllm_engine", None) is None
+
+ inputs = tokenizer(["Hello"], return_tensors = "pt").to(model.device)
+ out = model.fast_generate(**inputs, max_new_tokens = 4, min_new_tokens = 4, do_sample = False)
+ assert out.shape[1] == inputs["input_ids"].shape[1] + 4
diff --git a/tests/test_fastmodel_prevolta_fast_inference.py b/tests/test_fastmodel_prevolta_fast_inference.py
index 9901096d0a9..a64760f2129 100644
--- a/tests/test_fastmodel_prevolta_fast_inference.py
+++ b/tests/test_fastmodel_prevolta_fast_inference.py
@@ -40,6 +40,23 @@ def _fast_inference_blocks(cls_name):
raise AssertionError(f"{cls_name}.from_pretrained has no pre-Volta gate before the GB10 block")
+def _drop_helper():
+ # The real helper the fallbacks call, sliced the same way.
+ tree = ast.parse(LOADER_PATH.read_text(encoding = "utf-8"))
+ nodes = [
+ n
+ for n in tree.body
+ if (
+ isinstance(n, ast.Assign)
+ and any(getattr(t, "id", None) == "_VLLM_ENGINE_KWARGS" for t in n.targets)
+ )
+ or (isinstance(n, ast.FunctionDef) and n.name == "_drop_vllm_engine_kwargs")
+ ]
+ ns = {}
+ exec(compile(ast.Module(nodes, []), str(LOADER_PATH), "exec"), ns)
+ return ns["_drop_vllm_engine_kwargs"]
+
+
def _run(
cls_name,
device_type,
@@ -62,8 +79,12 @@ def _run(
"DEVICE_TYPE_TORCH": device_type_torch,
"DEVICE_COUNT": 1,
"torch": types.SimpleNamespace(cuda = cuda),
+ "kwargs": {"enforce_eager": True},
+ "_drop_vllm_engine_kwargs": _drop_helper(),
}
exec(compile(ast.Module(_fast_inference_blocks(cls_name), []), str(LOADER_PATH), "exec"), ns)
+ # A fallback drops vLLM-only kwargs; staying on vLLM keeps them.
+ assert ("enforce_eager" in ns["kwargs"]) is ns["fast_inference"]
return ns["fast_inference"]
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 8424aeaa94a..1b2f350f294 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -102,6 +102,20 @@ SUPPORTS_GPTOSS = transformers_version >= Version("4.55.0")
SUPPORTS_GEMMA4 = transformers_version >= Version("5.5.0")
# unsloth_zoo cannot share these models' vLLM weights with the training model.
VLLM_UNSUPPORTED_MODEL_TYPES = ("gpt_oss",)
+# model_type -> unsloth_zoo.empty_model symbol that rebuilds it from vLLM; an older zoo falls back.
+_VLLM_NEEDS_ZOO_SYMBOL = {"nemotron_h": "extract_nemotron_h_mixer"}
+
+
+def _vllm_unsupported_model_types(model_types):
+ unsupported = [t for t in model_types if t in VLLM_UNSUPPORTED_MODEL_TYPES]
+ for model_type, symbol in _VLLM_NEEDS_ZOO_SYMBOL.items():
+ if model_type in model_types:
+ import unsloth_zoo.empty_model as zoo_empty_model
+ if not hasattr(zoo_empty_model, symbol):
+ unsupported.append(model_type)
+ return unsupported
+
+
# Transformers v5 meta-device loading corrupts non-persistent buffers (inv_freq); see _fix_rope_inv_freq() below.
_NEEDS_ROPE_FIX = transformers_version >= Version("5.0.0")
if SUPPORTS_GEMMA:
@@ -958,6 +972,28 @@ def _vllm_unavailable_error():
)
+# load_vllm-only kwargs; transformers' from_pretrained rejects them.
+_VLLM_ENGINE_KWARGS = (
+ "max_loras",
+ "use_async",
+ "use_engine",
+ "enforce_eager",
+ "enable_prefix_caching",
+ "compilation_config",
+ "conservativeness",
+ "max_logprobs",
+ "max_num_seqs",
+ "return_args",
+ "training",
+)
+
+
+def _drop_vllm_engine_kwargs(kwargs):
+ dropped = [k for k in _VLLM_ENGINE_KWARGS if kwargs.pop(k, None) is not None]
+ if dropped:
+ print(f"Unsloth: Ignoring vLLM-only arguments {dropped} since vLLM is not used.")
+
+
class FastLanguageModel(FastLlamaModel):
@staticmethod
@tokenizers_v1_on_return
@@ -1092,6 +1128,7 @@ class FastLanguageModel(FastLlamaModel):
if fast_inference and DEVICE_TYPE == "cuda" and torch.cuda.get_device_capability()[0] < 7:
print("Unsloth: vLLM does not work on older GPUs - will switch to Unsloth inference!")
fast_inference = False
+ _drop_vllm_engine_kwargs(kwargs)
if fast_inference:
if importlib.util.find_spec("vllm") is None:
raise _vllm_unavailable_error()
@@ -1103,6 +1140,7 @@ class FastLanguageModel(FastLlamaModel):
"Defaulting to native Unsloth inference."
)
fast_inference = False
+ _drop_vllm_engine_kwargs(kwargs)
break
raise_if_fast_inference_under_fsdp2(fast_inference)
@@ -2020,6 +2058,7 @@ class FastModel(FastBaseModel):
if fast_inference and DEVICE_TYPE == "cuda" and torch.cuda.get_device_capability()[0] < 7:
print("Unsloth: vLLM does not work on older GPUs - will switch to Unsloth inference!")
fast_inference = False
+ _drop_vllm_engine_kwargs(kwargs)
if fast_inference:
if importlib.util.find_spec("vllm") is None:
raise _vllm_unavailable_error()
@@ -2031,6 +2070,7 @@ class FastModel(FastBaseModel):
"Defaulting to native Unsloth inference."
)
fast_inference = False
+ _drop_vllm_engine_kwargs(kwargs)
break
raise_if_fast_inference_under_fsdp2(fast_inference)
@@ -2286,13 +2326,14 @@ class FastModel(FastBaseModel):
model_types_all = ",".join(model_types) + ","
_maybe_advise_fla_install(model_types)
_raise_if_modeling_ignores_config(model_config, model_types)
- _vllm_unsupported = [t for t in model_types if t in VLLM_UNSUPPORTED_MODEL_TYPES]
+ _vllm_unsupported = _vllm_unsupported_model_types(model_types) if fast_inference else []
if fast_inference and _vllm_unsupported:
# Without this, vLLM loads the whole model first and weight sharing then crashes (unslothai/unsloth#4541).
print(
f"Unsloth: fast_inference (vLLM) does not support {_vllm_unsupported[0]} yet - will switch to Unsloth inference!"
)
fast_inference = False
+ _drop_vllm_engine_kwargs(kwargs)
# Text-diffusion models (DiffusionGemma) take a transformers-only slow path: a custom block-diffusion generate over a novel backbone, so Unsloth's autoregressive kernel/compile patching is skipped and the unmodified HF model is loaded, keeping 4bit/8bit and PEFT LoRA.
# Remote-code checkpoints can report their backbone's type (Nemotron-Labs-Diffusion says "nemotron"), so also match the raw config.
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 3366203e081..5ff6b30ba4e 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -4666,6 +4666,17 @@ class FastBaseModel:
target_parameters = packed_expert_target_parameters(
model, target_parameters, _packed_leaves
)
+ # vLLM serves Nemotron-H's relu2 experts only via the 2D FusedMoEWithLoRA; load_lora refuses a stacked adapter.
+ if (
+ target_parameters
+ and getattr(model, "vllm_engine", None) is not None
+ and getattr(getattr(model, "config", None), "model_type", None) == "nemotron_h"
+ ):
+ print(
+ "Unsloth: fast_inference (vLLM) cannot serve LoRA on Nemotron-H's MoE experts, so they stay frozen "
+ f"({target_parameters} not trained). Use fast_inference = False to train them."
+ )
+ target_parameters = None
if _moe_module_targets:
target_modules = add_moe_target_modules(target_modules, _moe_module_targets)