Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep import unsloth working when vLLM needs transformers 5 (#12268)
Daniel Han62db978f0afc
diff --git a/tests/test_vllm_broken_detection.py b/tests/test_vllm_broken_detection.pyindex 96e0f8b77e0..54dd339381b 100644--- a/tests/test_vllm_broken_detection.py+++ b/tests/test_vllm_broken_detection.py@@ -83,9 +83,11 @@ def _fake_vllm(saved_meta_path = list(sys.meta_path)saved_find_spec = importlib.util.find_specsaved_broken = import_fixes.VLLM_BROKEN+ saved_reason = import_fixes.VLLM_DISABLED_REASONsaved_modules = {n: sys.modules.get(n) for n in ("vllm", *submodules)}try:import_fixes.VLLM_BROKEN = False+ import_fixes.VLLM_DISABLED_REASON = Nonefake_vllm = types.ModuleType("vllm")fake_vllm.__path__ = []fake_vllm.__spec__ = importlib.machinery.ModuleSpec("vllm", loader = None, is_package = True)@@ -96,6 +98,7 @@ def _fake_vllm(yield import_fixesfinally:import_fixes.VLLM_BROKEN = saved_broken+ import_fixes.VLLM_DISABLED_REASON = saved_reasonsys.meta_path[:] = saved_meta_pathimportlib.util.find_spec = saved_find_specfor name, module in saved_modules.items():@@ -162,5 +165,51 @@ def test_disable_broken_vllm_keeps_healthy_vllm_enabled(present):assert importlib.util.find_spec("vllm") is not None+# vllm/transformers_utils/config.py raises this at import on vLLM >= 0.24 under transformers < 5.+_NEEDS_TRANSFORMERS_V5 = (+ "Support for Transformers v4 is deprecated and was removed in vLLM v0.24.0. "+ "Please upgrade to Transformers v5: pip install --upgrade transformers"+)+++def _chained(outer, inner):+ try:+ try:+ raise inner+ except Exception as error:+ raise outer from error+ except Exception as error:+ return error+++ "error",+ [+ ImportError(_NEEDS_TRANSFORMERS_V5),+ _chained(RuntimeError("vllm import failed"), ImportError(_NEEDS_TRANSFORMERS_V5)),+ ],+ ids = ["direct", "chained"],+)+def test_disable_broken_vllm_disables_vllm_that_needs_transformers_v5(error):+ # vLLM >= 0.24 refuses transformers 4.x at import; that used to escape and fail `import unsloth`.+ with _fake_vllm(present = set(), broken = set()) as import_fixes:+ assert import_fixes.disable_broken_vllm(error) is True+ assert import_fixes.VLLM_BROKEN is True+ assert importlib.util.find_spec("vllm") is None+ assert "needs transformers >= 5.0" in import_fixes.VLLM_DISABLED_REASON++ from unsloth.models.loader import _vllm_unavailable_error++ message = str(_vllm_unavailable_error())+ assert "needs transformers >= 5.0" in message and "pip install vllm`" not in message+++def test_disable_broken_vllm_leaves_unrelated_import_errors_alone():+ with _fake_vllm(present = set(), broken = set()) as import_fixes:+ assert import_fixes.disable_broken_vllm(ImportError("No module named 'xformers'")) is False+ assert import_fixes.VLLM_BROKEN is False+ assert import_fixes.VLLM_DISABLED_REASON is None++if __name__ == "__main__":raise SystemExit(pytest.main([__file__, "-v"]))diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex a0467ad8c58..725ecd37182 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -8267,6 +8267,7 @@ CAUSAL_CONV1D_BROKEN = False_CAUSAL_CONV1D_PREFIX = "causal_conv1d"_CAUSAL_CONV1D_BLOCKER_SENTINEL = "_unsloth_causal_conv1d_blocker"VLLM_BROKEN = False+VLLM_DISABLED_REASON = None # the warning logged when vLLM was disabled, for fast_inference errors_VLLM_PREFIX = "vllm"_VLLM_BLOCKER_SENTINEL = "_unsloth_vllm_blocker"_ROCM_ENV_HINT_KEYS = (@@ -8781,6 +8782,19 @@ def _is_broken_vllm_error(error) -> bool:return False+def _is_vllm_needs_transformers_v5_error(error) -> bool:+ # vLLM >= 0.24 raises ImportError at import under transformers < 5 (vllm/transformers_utils/config.py).+ checked = set()+ current = error+ while current is not None and id(current) not in checked:+ checked.add(id(current))+ message = str(current).lower()+ if "support for transformers v4" in message and "removed in vllm" in message:+ return True+ current = getattr(current, "__cause__", None) or getattr(current, "__context__", None)+ return False++_VLLM_RELEASES_URL = "https://github.com/vllm-project/vllm/releases"_VLLM_INSTALL_DOCS_URL = "https://docs.vllm.ai/en/latest/getting_started/installation/gpu/"@@ -9142,8 +9156,8 @@ _VLLM_COMPILED_EXTENSIONS = (def disable_broken_vllm(error = None):- """Disable vLLM dynamically when its shared library is ABI-broken."""- global VLLM_BROKEN+ """Disable vLLM dynamically when its shared library is ABI-broken or it refuses this transformers."""+ global VLLM_BROKEN, VLLM_DISABLED_REASONif VLLM_BROKEN:_install_vllm_blocker()return True@@ -9170,22 +9184,34 @@ def disable_broken_vllm(error = None):except Exception as import_error:failure = import_error- if not _is_broken_vllm_error(failure):+ needs_transformers_v5 = _is_vllm_needs_transformers_v5_error(failure)+ if not needs_transformers_v5 and not _is_broken_vllm_error(failure):return FalseVLLM_BROKEN = True_clear_vllm_modules()_install_vllm_blocker()- cuda_msg = _get_vllm_cuda_mismatch_message(failure)- if cuda_msg:- logger.warning(cuda_msg)+ cuda_msg = None if needs_transformers_v5 else _get_vllm_cuda_mismatch_message(failure)+ if needs_transformers_v5:+ try:+ vllm_version = importlib_version("vllm")+ except Exception:+ vllm_version = "unknown"+ VLLM_DISABLED_REASON = (+ f"Unsloth: vLLM {vllm_version} needs transformers >= 5.0, so vLLM is disabled and "+ "fast_inference is unavailable; everything else still works.\n"+ 'To use fast_inference, upgrade transformers or install "vllm<0.24".'+ )+ elif cuda_msg:+ VLLM_DISABLED_REASON = cuda_msgelse:- logger.warning(+ VLLM_DISABLED_REASON = ("Unsloth: Detected broken vLLM binary extension; ""disabling vLLM imports and continuing import.\n""Please reinstall via `uv pip install unsloth vllm torchvision torchaudio ""--torch-backend=auto`.")+ logger.warning(VLLM_DISABLED_REASON)return Truediff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex dfa5b525567..5a8b3cbd87e 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -728,6 +728,19 @@ def _fix_rope_inv_freq(model):return model+def _vllm_unavailable_error():+ # vLLM installed but disabled at import (ABI break, needs transformers 5) is not "not installed".+ from unsloth import import_fixes+ if import_fixes.VLLM_DISABLED_REASON:+ return ImportError(+ f"{import_fixes.VLLM_DISABLED_REASON}\n`fast_inference = True` needs a working vLLM."+ )+ return ImportError(+ "Unsloth: Please install vLLM before enabling `fast_inference`!\n"+ "You can do this in a terminal via `pip install vllm`"+ )++class FastLanguageModel(FastLlamaModel):@staticmethod@_offline_aware_load@@ -859,10 +872,7 @@ class FastLanguageModel(FastLlamaModel):if fast_inference:if importlib.util.find_spec("vllm") is None:- raise ImportError(- "Unsloth: Please install vLLM before enabling `fast_inference`!\n"- "You can do this in a terminal via `pip install vllm`"- )+ raise _vllm_unavailable_error()if DEVICE_TYPE_TORCH == "cuda":for i in range(DEVICE_COUNT):if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():@@ -1726,10 +1736,7 @@ class FastModel(FastBaseModel):if fast_inference:if importlib.util.find_spec("vllm") is None:- raise ImportError(- "Unsloth: Please install vLLM before enabling `fast_inference`!\n"- "You can do this in a terminal via `pip install vllm`"- )+ raise _vllm_unavailable_error()if DEVICE_TYPE_TORCH == "cuda":for i in range(DEVICE_COUNT):if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():