Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Nemotron-H fast_inference via vLLM weight sharing (fallback on older unsloth_zoo); drop vLLM-only kwargs on fallback (#13440)
Daniel Han8196cf968918
diff --git a/tests/fast_inference/test_nemotron_h_fallback.py b/tests/fast_inference/test_nemotron_h_fallback.pynew file mode 100644index 00000000000..cccaee53645--- /dev/null+++ b/tests/fast_inference/test_nemotron_h_fallback.py@@ -0,0 +1,54 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Nemotron-H with fast_inference = True falls back to Unsloth inference when the installed unsloth_zoo cannot rebuild it from vLLM (unslothai/unsloth-zoo#565)."""++import importlib.util++import pytest+from real_accelerator import (+ has_real_accelerator,+) # tests/_shared, on sys.path via tests/conftest.py++pytestmark = pytest.mark.gpu++MODEL_NAME = "trl-internal-testing/tiny-NemotronHForCausalLM-nano"++[email protected](not has_real_accelerator(), reason = "needs a GPU")+def test_nemotron_h_needs_a_zoo_that_rebuilds_it(monkeypatch):+ import unsloth_zoo.empty_model as zoo_empty_model+ from unsloth.models.loader import _vllm_unsupported_model_types++ monkeypatch.setattr(zoo_empty_model, "extract_nemotron_h_mixer", object(), raising = False)+ assert _vllm_unsupported_model_types(["nemotron_h"]) == []+ assert _vllm_unsupported_model_types(["gpt_oss"]) == ["gpt_oss"]+ monkeypatch.delattr(zoo_empty_model, "extract_nemotron_h_mixer")+ assert _vllm_unsupported_model_types(["nemotron_h"]) == ["nemotron_h"]+ assert _vllm_unsupported_model_types(["llama"]) == []++[email protected](not has_real_accelerator(), reason = "needs a GPU")[email protected](importlib.util.find_spec("vllm") is None, reason = "needs vLLM installed")+def test_nemotron_h_fast_inference_falls_back_on_an_older_zoo(capfd, monkeypatch):+ import unsloth_zoo.empty_model as zoo_empty_model+ from unsloth import FastLanguageModel++ monkeypatch.delattr(zoo_empty_model, "extract_nemotron_h_mixer", raising = False)+ model, tokenizer = FastLanguageModel.from_pretrained(+ model_name = MODEL_NAME,+ max_seq_length = 256,+ load_in_4bit = False,+ fast_inference = True,+ gpu_memory_utilization = 0.1,+ enforce_eager = True,+ max_num_seqs = 8,+ )+ out = capfd.readouterr().out+ assert "does not support nemotron_h yet - will switch to Unsloth inference" in out+ assert "Ignoring vLLM-only arguments ['enforce_eager', 'max_num_seqs']" in out+ assert getattr(model, "vllm_engine", None) is None++ inputs = tokenizer(["Hello"], return_tensors = "pt").to(model.device)+ out = model.fast_generate(**inputs, max_new_tokens = 4, min_new_tokens = 4, do_sample = False)+ assert out.shape[1] == inputs["input_ids"].shape[1] + 4diff --git a/tests/test_fastmodel_prevolta_fast_inference.py b/tests/test_fastmodel_prevolta_fast_inference.pyindex 9901096d0a9..a64760f2129 100644--- a/tests/test_fastmodel_prevolta_fast_inference.py+++ b/tests/test_fastmodel_prevolta_fast_inference.py@@ -40,6 +40,23 @@ def _fast_inference_blocks(cls_name):raise AssertionError(f"{cls_name}.from_pretrained has no pre-Volta gate before the GB10 block")+def _drop_helper():+ # The real helper the fallbacks call, sliced the same way.+ tree = ast.parse(LOADER_PATH.read_text(encoding = "utf-8"))+ nodes = [+ n+ for n in tree.body+ if (+ isinstance(n, ast.Assign)+ and any(getattr(t, "id", None) == "_VLLM_ENGINE_KWARGS" for t in n.targets)+ )+ or (isinstance(n, ast.FunctionDef) and n.name == "_drop_vllm_engine_kwargs")+ ]+ ns = {}+ exec(compile(ast.Module(nodes, []), str(LOADER_PATH), "exec"), ns)+ return ns["_drop_vllm_engine_kwargs"]++def _run(cls_name,device_type,@@ -62,8 +79,12 @@ def _run("DEVICE_TYPE_TORCH": device_type_torch,"DEVICE_COUNT": 1,"torch": types.SimpleNamespace(cuda = cuda),+ "kwargs": {"enforce_eager": True},+ "_drop_vllm_engine_kwargs": _drop_helper(),}exec(compile(ast.Module(_fast_inference_blocks(cls_name), []), str(LOADER_PATH), "exec"), ns)+ # A fallback drops vLLM-only kwargs; staying on vLLM keeps them.+ assert ("enforce_eager" in ns["kwargs"]) is ns["fast_inference"]return ns["fast_inference"]diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 8424aeaa94a..1b2f350f294 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -102,6 +102,20 @@ SUPPORTS_GPTOSS = transformers_version >= Version("4.55.0")SUPPORTS_GEMMA4 = transformers_version >= Version("5.5.0")# unsloth_zoo cannot share these models' vLLM weights with the training model.VLLM_UNSUPPORTED_MODEL_TYPES = ("gpt_oss",)+# model_type -> unsloth_zoo.empty_model symbol that rebuilds it from vLLM; an older zoo falls back.+_VLLM_NEEDS_ZOO_SYMBOL = {"nemotron_h": "extract_nemotron_h_mixer"}+++def _vllm_unsupported_model_types(model_types):+ unsupported = [t for t in model_types if t in VLLM_UNSUPPORTED_MODEL_TYPES]+ for model_type, symbol in _VLLM_NEEDS_ZOO_SYMBOL.items():+ if model_type in model_types:+ import unsloth_zoo.empty_model as zoo_empty_model+ if not hasattr(zoo_empty_model, symbol):+ unsupported.append(model_type)+ return unsupported++# Transformers v5 meta-device loading corrupts non-persistent buffers (inv_freq); see _fix_rope_inv_freq() below._NEEDS_ROPE_FIX = transformers_version >= Version("5.0.0")if SUPPORTS_GEMMA:@@ -958,6 +972,28 @@ def _vllm_unavailable_error():)+# load_vllm-only kwargs; transformers' from_pretrained rejects them.+_VLLM_ENGINE_KWARGS = (+ "max_loras",+ "use_async",+ "use_engine",+ "enforce_eager",+ "enable_prefix_caching",+ "compilation_config",+ "conservativeness",+ "max_logprobs",+ "max_num_seqs",+ "return_args",+ "training",+)+++def _drop_vllm_engine_kwargs(kwargs):+ dropped = [k for k in _VLLM_ENGINE_KWARGS if kwargs.pop(k, None) is not None]+ if dropped:+ print(f"Unsloth: Ignoring vLLM-only arguments {dropped} since vLLM is not used.")++class FastLanguageModel(FastLlamaModel):@staticmethod@tokenizers_v1_on_return@@ -1092,6 +1128,7 @@ class FastLanguageModel(FastLlamaModel):if fast_inference and DEVICE_TYPE == "cuda" and torch.cuda.get_device_capability()[0] < 7:print("Unsloth: vLLM does not work on older GPUs - will switch to Unsloth inference!")fast_inference = False+ _drop_vllm_engine_kwargs(kwargs)if fast_inference:if importlib.util.find_spec("vllm") is None:raise _vllm_unavailable_error()@@ -1103,6 +1140,7 @@ class FastLanguageModel(FastLlamaModel):"Defaulting to native Unsloth inference.")fast_inference = False+ _drop_vllm_engine_kwargs(kwargs)breakraise_if_fast_inference_under_fsdp2(fast_inference)@@ -2020,6 +2058,7 @@ class FastModel(FastBaseModel):if fast_inference and DEVICE_TYPE == "cuda" and torch.cuda.get_device_capability()[0] < 7:print("Unsloth: vLLM does not work on older GPUs - will switch to Unsloth inference!")fast_inference = False+ _drop_vllm_engine_kwargs(kwargs)if fast_inference:if importlib.util.find_spec("vllm") is None:raise _vllm_unavailable_error()@@ -2031,6 +2070,7 @@ class FastModel(FastBaseModel):"Defaulting to native Unsloth inference.")fast_inference = False+ _drop_vllm_engine_kwargs(kwargs)breakraise_if_fast_inference_under_fsdp2(fast_inference)@@ -2286,13 +2326,14 @@ class FastModel(FastBaseModel):model_types_all = ",".join(model_types) + ","_maybe_advise_fla_install(model_types)_raise_if_modeling_ignores_config(model_config, model_types)- _vllm_unsupported = [t for t in model_types if t in VLLM_UNSUPPORTED_MODEL_TYPES]+ _vllm_unsupported = _vllm_unsupported_model_types(model_types) if fast_inference else []if fast_inference and _vllm_unsupported:# Without this, vLLM loads the whole model first and weight sharing then crashes (unslothai/unsloth#4541).print(f"Unsloth: fast_inference (vLLM) does not support {_vllm_unsupported[0]} yet - will switch to Unsloth inference!")fast_inference = False+ _drop_vllm_engine_kwargs(kwargs)# Text-diffusion models (DiffusionGemma) take a transformers-only slow path: a custom block-diffusion generate over a novel backbone, so Unsloth's autoregressive kernel/compile patching is skipped and the unmodified HF model is loaded, keeping 4bit/8bit and PEFT LoRA.# Remote-code checkpoints can report their backbone's type (Nemotron-Labs-Diffusion says "nemotron"), so also match the raw config.diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 3366203e081..5ff6b30ba4e 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -4666,6 +4666,17 @@ class FastBaseModel:target_parameters = packed_expert_target_parameters(model, target_parameters, _packed_leaves)+ # vLLM serves Nemotron-H's relu2 experts only via the 2D FusedMoEWithLoRA; load_lora refuses a stacked adapter.+ if (+ target_parameters+ and getattr(model, "vllm_engine", None) is not None+ and getattr(getattr(model, "config", None), "model_type", None) == "nemotron_h"+ ):+ print(+ "Unsloth: fast_inference (vLLM) cannot serve LoRA on Nemotron-H's MoE experts, so they stay frozen "+ f"({target_parameters} not trained). Use fast_inference = False to train them."+ )+ target_parameters = Noneif _moe_module_targets:target_modules = add_moe_target_modules(target_modules, _moe_module_targets)