Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep import unsloth working when vLLM needs transformers 5 (#12268)

Daniel Han62db978f0afc
diff --git a/tests/test_vllm_broken_detection.py b/tests/test_vllm_broken_detection.py
index 96e0f8b77e0..54dd339381b 100644
--- a/tests/test_vllm_broken_detection.py
+++ b/tests/test_vllm_broken_detection.py
@@ -83,9 +83,11 @@ def _fake_vllm(
saved_meta_path = list(sys.meta_path)
saved_find_spec = importlib.util.find_spec
saved_broken = import_fixes.VLLM_BROKEN
+ saved_reason = import_fixes.VLLM_DISABLED_REASON
saved_modules = {n: sys.modules.get(n) for n in ("vllm", *submodules)}
try:
import_fixes.VLLM_BROKEN = False
+ import_fixes.VLLM_DISABLED_REASON = None
fake_vllm = types.ModuleType("vllm")
fake_vllm.__path__ = []
fake_vllm.__spec__ = importlib.machinery.ModuleSpec("vllm", loader = None, is_package = True)
@@ -96,6 +98,7 @@ def _fake_vllm(
yield import_fixes
finally:
import_fixes.VLLM_BROKEN = saved_broken
+ import_fixes.VLLM_DISABLED_REASON = saved_reason
sys.meta_path[:] = saved_meta_path
importlib.util.find_spec = saved_find_spec
for name, module in saved_modules.items():
@@ -162,5 +165,51 @@ def test_disable_broken_vllm_keeps_healthy_vllm_enabled(present):
assert importlib.util.find_spec("vllm") is not None
+# vllm/transformers_utils/config.py raises this at import on vLLM >= 0.24 under transformers < 5.
+_NEEDS_TRANSFORMERS_V5 = (
+ "Support for Transformers v4 is deprecated and was removed in vLLM v0.24.0. "
+ "Please upgrade to Transformers v5: pip install --upgrade transformers"
+)
+
+
+def _chained(outer, inner):
+ try:
+ try:
+ raise inner
+ except Exception as error:
+ raise outer from error
+ except Exception as error:
+ return error
+
+
+ "error",
+ [
+ ImportError(_NEEDS_TRANSFORMERS_V5),
+ _chained(RuntimeError("vllm import failed"), ImportError(_NEEDS_TRANSFORMERS_V5)),
+ ],
+ ids = ["direct", "chained"],
+)
+def test_disable_broken_vllm_disables_vllm_that_needs_transformers_v5(error):
+ # vLLM >= 0.24 refuses transformers 4.x at import; that used to escape and fail `import unsloth`.
+ with _fake_vllm(present = set(), broken = set()) as import_fixes:
+ assert import_fixes.disable_broken_vllm(error) is True
+ assert import_fixes.VLLM_BROKEN is True
+ assert importlib.util.find_spec("vllm") is None
+ assert "needs transformers >= 5.0" in import_fixes.VLLM_DISABLED_REASON
+
+ from unsloth.models.loader import _vllm_unavailable_error
+
+ message = str(_vllm_unavailable_error())
+ assert "needs transformers >= 5.0" in message and "pip install vllm`" not in message
+
+
+def test_disable_broken_vllm_leaves_unrelated_import_errors_alone():
+ with _fake_vllm(present = set(), broken = set()) as import_fixes:
+ assert import_fixes.disable_broken_vllm(ImportError("No module named 'xformers'")) is False
+ assert import_fixes.VLLM_BROKEN is False
+ assert import_fixes.VLLM_DISABLED_REASON is None
+
+
if __name__ == "__main__":
raise SystemExit(pytest.main([__file__, "-v"]))
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index a0467ad8c58..725ecd37182 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -8267,6 +8267,7 @@ CAUSAL_CONV1D_BROKEN = False
_CAUSAL_CONV1D_PREFIX = "causal_conv1d"
_CAUSAL_CONV1D_BLOCKER_SENTINEL = "_unsloth_causal_conv1d_blocker"
VLLM_BROKEN = False
+VLLM_DISABLED_REASON = None # the warning logged when vLLM was disabled, for fast_inference errors
_VLLM_PREFIX = "vllm"
_VLLM_BLOCKER_SENTINEL = "_unsloth_vllm_blocker"
_ROCM_ENV_HINT_KEYS = (
@@ -8781,6 +8782,19 @@ def _is_broken_vllm_error(error) -> bool:
return False
+def _is_vllm_needs_transformers_v5_error(error) -> bool:
+ # vLLM >= 0.24 raises ImportError at import under transformers < 5 (vllm/transformers_utils/config.py).
+ checked = set()
+ current = error
+ while current is not None and id(current) not in checked:
+ checked.add(id(current))
+ message = str(current).lower()
+ if "support for transformers v4" in message and "removed in vllm" in message:
+ return True
+ current = getattr(current, "__cause__", None) or getattr(current, "__context__", None)
+ return False
+
+
_VLLM_RELEASES_URL = "https://github.com/vllm-project/vllm/releases"
_VLLM_INSTALL_DOCS_URL = "https://docs.vllm.ai/en/latest/getting_started/installation/gpu/"
@@ -9142,8 +9156,8 @@ _VLLM_COMPILED_EXTENSIONS = (
def disable_broken_vllm(error = None):
- """Disable vLLM dynamically when its shared library is ABI-broken."""
- global VLLM_BROKEN
+ """Disable vLLM dynamically when its shared library is ABI-broken or it refuses this transformers."""
+ global VLLM_BROKEN, VLLM_DISABLED_REASON
if VLLM_BROKEN:
_install_vllm_blocker()
return True
@@ -9170,22 +9184,34 @@ def disable_broken_vllm(error = None):
except Exception as import_error:
failure = import_error
- if not _is_broken_vllm_error(failure):
+ needs_transformers_v5 = _is_vllm_needs_transformers_v5_error(failure)
+ if not needs_transformers_v5 and not _is_broken_vllm_error(failure):
return False
VLLM_BROKEN = True
_clear_vllm_modules()
_install_vllm_blocker()
- cuda_msg = _get_vllm_cuda_mismatch_message(failure)
- if cuda_msg:
- logger.warning(cuda_msg)
+ cuda_msg = None if needs_transformers_v5 else _get_vllm_cuda_mismatch_message(failure)
+ if needs_transformers_v5:
+ try:
+ vllm_version = importlib_version("vllm")
+ except Exception:
+ vllm_version = "unknown"
+ VLLM_DISABLED_REASON = (
+ f"Unsloth: vLLM {vllm_version} needs transformers >= 5.0, so vLLM is disabled and "
+ "fast_inference is unavailable; everything else still works.\n"
+ 'To use fast_inference, upgrade transformers or install "vllm<0.24".'
+ )
+ elif cuda_msg:
+ VLLM_DISABLED_REASON = cuda_msg
else:
- logger.warning(
+ VLLM_DISABLED_REASON = (
"Unsloth: Detected broken vLLM binary extension; "
"disabling vLLM imports and continuing import.\n"
"Please reinstall via `uv pip install unsloth vllm torchvision torchaudio "
"--torch-backend=auto`."
)
+ logger.warning(VLLM_DISABLED_REASON)
return True
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index dfa5b525567..5a8b3cbd87e 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -728,6 +728,19 @@ def _fix_rope_inv_freq(model):
return model
+def _vllm_unavailable_error():
+ # vLLM installed but disabled at import (ABI break, needs transformers 5) is not "not installed".
+ from unsloth import import_fixes
+ if import_fixes.VLLM_DISABLED_REASON:
+ return ImportError(
+ f"{import_fixes.VLLM_DISABLED_REASON}\n`fast_inference = True` needs a working vLLM."
+ )
+ return ImportError(
+ "Unsloth: Please install vLLM before enabling `fast_inference`!\n"
+ "You can do this in a terminal via `pip install vllm`"
+ )
+
+
class FastLanguageModel(FastLlamaModel):
@staticmethod
@_offline_aware_load
@@ -859,10 +872,7 @@ class FastLanguageModel(FastLlamaModel):
if fast_inference:
if importlib.util.find_spec("vllm") is None:
- raise ImportError(
- "Unsloth: Please install vLLM before enabling `fast_inference`!\n"
- "You can do this in a terminal via `pip install vllm`"
- )
+ raise _vllm_unavailable_error()
if DEVICE_TYPE_TORCH == "cuda":
for i in range(DEVICE_COUNT):
if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():
@@ -1726,10 +1736,7 @@ class FastModel(FastBaseModel):
if fast_inference:
if importlib.util.find_spec("vllm") is None:
- raise ImportError(
- "Unsloth: Please install vLLM before enabling `fast_inference`!\n"
- "You can do this in a terminal via `pip install vllm`"
- )
+ raise _vllm_unavailable_error()
if DEVICE_TYPE_TORCH == "cuda":
for i in range(DEVICE_COUNT):
if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():