Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Add a vLLM backend to unsloth eval (#14228)

Daniel Hand3a82576a430
diff --git a/unsloth_cli/commands/eval.py b/unsloth_cli/commands/eval.py
index b97b8b91ee1..2c77c3b0012 100644
--- a/unsloth_cli/commands/eval.py
+++ b/unsloth_cli/commands/eval.py
@@ -8,6 +8,7 @@ import re
import shutil
import sys
import tempfile
+from importlib.metadata import PackageNotFoundError, version
from importlib.util import find_spec
from pathlib import Path
from typing import List, Optional, Tuple
@@ -22,8 +23,8 @@ _YAML_SUFFIXES = {".yaml", ".yml"}
@contextlib.contextmanager
-def _silence():
- """Send fd 1/2 to devnull (lm-eval/transformers progress bars); yield a Console on the real stdout."""
+def _silence(log: Optional[Path] = None):
+ """Send fd 1/2 to devnull or `log` (lm-eval/transformers progress bars); yield a Console on the real stdout."""
from rich.console import Console
sys.stdout.flush()
@@ -37,7 +38,11 @@ def _silence():
errors = "replace",
)
saved_out, saved_err = os.dup(1), os.dup(2)
- devnull_fd = os.open(os.devnull, os.O_WRONLY)
+ devnull_fd = (
+ os.open(log, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
+ if log
+ else os.open(os.devnull, os.O_WRONLY)
+ )
try:
os.dup2(devnull_fd, 1)
os.dup2(devnull_fd, 2)
@@ -185,6 +190,64 @@ def _hflm_4bit(HFLM):
return _HFLM4bit
+# vLLM's accepted max_lora_rank values (vllm/config/lora.py MaxLoRARanks)
+_VLLM_LORA_RANKS = (16, 32, 64, 128, 256, 320, 512)
+_BNB_SUFFIX_RE = re.compile(r"(-unsloth)?-bnb-4bit$")
+
+
+def _vllm_error() -> Optional[str]:
+ if sys.platform == "darwin":
+ return "--backend vllm needs an NVIDIA or AMD GPU. On a Mac use --backend hf."
+ if find_spec("vllm") is None:
+ return (
+ "--backend vllm needs vllm (`pip install vllm`). "
+ "Use --backend unsloth to evaluate without it."
+ )
+ try:
+ lm_eval_version = version("lm_eval")
+ except PackageNotFoundError:
+ lm_eval_version = None
+ # older lm-eval's vllm wrapper breaks on current vLLM (imports ray, passes the removed swap_space)
+ installed = tuple(int(n) for n in re.findall(r"\d+", lm_eval_version or "")[:3])
+ if installed and installed < (0, 4, 12):
+ return (
+ f"--backend vllm needs lm-eval 0.4.12 or newer, but {lm_eval_version} is installed. "
+ "Upgrade it with `pip install -U lm-eval`."
+ )
+ # as vLLM does on import: the default probe initialises CUDA, which its forked engine cannot reuse
+ os.environ["PYTORCH_NVML_BASED_CUDA_CHECK"] = "1"
+ import torch
+
+ if not torch.cuda.is_available():
+ return (
+ "--backend vllm needs an NVIDIA or AMD GPU, but none is visible. "
+ "Use --backend hf to evaluate on CPU."
+ )
+ return None
+
+
+def _vllm_base(model: str) -> str:
+ # vLLM has no bitsandbytes loader
+ stripped = _BNB_SUFFIX_RE.sub("", model)
+ if stripped != model and model.lower().startswith("unsloth/"):
+ typer.echo(f"Note: vLLM cannot load '{model}', using its 16-bit repo '{stripped}'.")
+ return stripped
+ return model
+
+
+def _local_adapter(model: str) -> str:
+ if Path(model).is_dir():
+ return str(Path(model).resolve())
+ from huggingface_hub import snapshot_download
+ return snapshot_download(model)
+
+
+def _adapter_rank(adapter_dir: str) -> int:
+ config = _read_config(Path(adapter_dir) / "adapter_config.json") or {}
+ ranks = [config.get("r")] + list((config.get("rank_pattern") or {}).values())
+ return max((r for r in ranks if isinstance(r, int)), default = 0)
+
+
def _base_vocab_size(model: str) -> int:
from transformers import AutoConfig
config = AutoConfig.from_pretrained(model)
@@ -425,7 +488,8 @@ def evaluate(
"unsloth",
"--backend",
help = "Model backend: 'unsloth' (fast kernels; needs an NVIDIA/AMD/Intel "
- "GPU) or 'hf' (plain transformers; works on CPU/MPS/Mac). "
+ "GPU), 'hf' (plain transformers; works on CPU/MPS/Mac) or 'vllm' (fastest on "
+ "generation tasks; needs vllm and an NVIDIA/AMD GPU, loads 16-bit weights). "
"Auto-falls back to 'hf' on Apple Silicon.",
),
device: Optional[str] = typer.Option(
@@ -433,6 +497,11 @@ def evaluate(
"--device",
help = "Device for the hf backend (e.g. cpu, mps, cuda). Default: auto.",
),
+ gpu_memory_utilization: Optional[float] = typer.Option(
+ None,
+ "--gpu-memory-utilization",
+ help = "Fraction of GPU memory vLLM may use (vllm backend). Default: vLLM's own.",
+ ),
input_key: str = typer.Option(
"question", "--input-key", help = "Prompt field for a .jsonl/.csv dataset task."
),
@@ -458,8 +527,10 @@ def evaluate(
bs = int(batch_size) if batch_size.isdigit() else 0
if bs <= 0:
_fail("--batch-size must be a positive integer or 'auto'.")
- if backend not in ("unsloth", "hf"):
- _fail(f"--backend must be 'unsloth' or 'hf', got '{backend}'.")
+ if backend not in ("unsloth", "hf", "vllm"):
+ _fail(f"--backend must be 'unsloth', 'hf' or 'vllm', got '{backend}'.")
+ if gpu_memory_utilization is not None and not 0 < gpu_memory_utilization <= 1:
+ _fail("--gpu-memory-utilization must be between 0 and 1.")
if num_fewshot is not None and num_fewshot < 0:
_fail("--num-fewshot must be >= 0.")
if limit is not None:
@@ -473,17 +544,25 @@ def evaluate(
if "lm_eval" not in sys.modules and find_spec("lm_eval") is None:
_fail("evaluation requires lm-eval. Install it with `pip install unsloth[eval]`.", code = 1)
+ if backend == "vllm":
+ if (error := _vllm_error()) is not None:
+ _fail(error)
+ if device is not None:
+ typer.echo("Note: --backend vllm ignores --device; set CUDA_VISIBLE_DEVICES instead.")
if backend == "unsloth":
with _silence():
import unsloth
if getattr(unsloth, "DEVICE_TYPE", None) == "mlx":
typer.echo("Note: Apple Silicon (MLX) detected, falling back to --backend hf.")
backend = "hf"
+ # only the vllm branch forwards it, and a silently dropped memory cap reads as an OOM
+ if backend != "vllm" and gpu_memory_utilization is not None:
+ typer.echo(f"Note: --gpu-memory-utilization applies to --backend vllm, not {backend}.")
# a pre-loaded model makes lm-eval single-process, so every rank would run every task
- if backend == "unsloth" and os.environ.get("WORLD_SIZE", "1") not in ("", "1"):
+ if backend in ("unsloth", "vllm") and os.environ.get("WORLD_SIZE", "1") not in ("", "1"):
_fail(
"multi-process launches (accelerate/torchrun) are not supported with "
- "--backend unsloth. Use --backend hf for multi-GPU evaluation."
+ f"--backend {backend}. Use --backend hf for multi-GPU evaluation."
)
import inspect
@@ -495,6 +574,10 @@ def evaluate(
if hf_token:
os.environ["HF_TOKEN"] = hf_token
effective_base = base_model or resolve_base_model(model)
+ if backend == "vllm":
+ if load_in_4bit:
+ typer.echo("Note: --backend vllm loads 16-bit weights; --load-in-4bit has no effect.")
+ load_in_4bit = False
if load_in_4bit is None:
load_in_4bit = _trained_in_4bit(model)
if not load_in_4bit:
@@ -566,6 +649,26 @@ def evaluate(
)
else:
eval_kwargs.update(model = "hf", model_args = model_args, batch_size = bs, device = device)
+ elif backend == "vllm":
+ model_args = {"pretrained": model, "max_model_len": max_seq_length}
+ if effective_base:
+ model_args["pretrained"] = _vllm_base(effective_base)
+ model_args["lora_local_path"] = _local_adapter(model)
+ rank = _adapter_rank(model_args["lora_local_path"])
+ if rank > 16:
+ model_args["max_lora_rank"] = next(
+ (r for r in _VLLM_LORA_RANKS if r >= rank), _VLLM_LORA_RANKS[-1]
+ )
+ if _has_tokenizer_files(model):
+ model_args["tokenizer"] = model_args["lora_local_path"]
+ typer.echo(
+ f"Evaluating adapter '{model}' on base '{model_args['pretrained']}' with vLLM."
+ )
+ else:
+ model_args["pretrained"] = _vllm_base(model)
+ if gpu_memory_utilization is not None:
+ model_args["gpu_memory_utilization"] = gpu_memory_utilization
+ eval_kwargs.update(model = "vllm", model_args = model_args, batch_size = bs)
else:
from unsloth import FastLanguageModel
@@ -611,10 +714,18 @@ def evaluate(
pretrained = lmodel, tokenizer = tokenizer, batch_size = bs, max_length = max_seq_length
)
- with _silence() as ui:
- from rich.status import Status
- with Status(f"Evaluating {', '.join(task_names)}…", console = ui, spinner = "dots"):
- results = lm_eval.simple_evaluate(**eval_kwargs)
+ engine_log = tmp_dir / "vllm.log" if backend == "vllm" else None
+ try:
+ with _silence(engine_log) as ui:
+ from rich.status import Status
+ with Status(f"Evaluating {', '.join(task_names)}…", console = ui, spinner = "dots"):
+ results = lm_eval.simple_evaluate(**eval_kwargs)
+ except Exception:
+ # vLLM's engine runs in its own process and logs the real cause there, not in the traceback
+ if engine_log is not None and engine_log.exists():
+ lines = engine_log.read_text(encoding = "utf-8", errors = "replace").splitlines()
+ typer.echo("\n".join(line.rsplit("\r", 1)[-1] for line in lines[-40:]), err = True)
+ raise
finally:
shutil.rmtree(tmp_dir, ignore_errors = True)
diff --git a/unsloth_cli/tests/test_eval.py b/unsloth_cli/tests/test_eval.py
index db67268872c..160c4cf6c77 100644
--- a/unsloth_cli/tests/test_eval.py
+++ b/unsloth_cli/tests/test_eval.py
@@ -577,7 +577,8 @@ def test_unsafe_code_flag_is_forwarded(env, tmp_path):
["--batch-size", "0"],
["--batch-size", "-1"],
["--batch-size", "big"],
- ["--backend", "vllm"],
+ ["--backend", "tgi"],
+ ["--backend", "vllm", "--gpu-memory-utilization", "1.5"],
["--num-fewshot", "-1"],
["--limit", "0"],
["--limit", "2.5"],
@@ -609,6 +610,142 @@ def test_unsloth_backend_refuses_multi_process(env, monkeypatch):
assert result.exit_code == 2 and "--backend hf" in result.output
+def vllm_env(env, monkeypatch):
+ sys.modules["torch"].cuda = SimpleNamespace(is_available = lambda: True, device_count = lambda: 1)
+ monkeypatch.setattr(evalmod.sys, "platform", "linux")
+ monkeypatch.setattr(evalmod, "version", lambda name: "0.4.13", raising = False)
+ return env
+
+
+def test_vllm_backend_builds_lm_eval_vllm(vllm_env, tmp_path):
+ result = _run(
+ "fake/model",
+ "--tasks",
+ "gsm8k",
+ "--backend",
+ "vllm",
+ "--max-seq-length",
+ "4096",
+ "--gpu-memory-utilization",
+ "0.5",
+ "-o",
+ str(tmp_path),
+ )
+ assert result.exit_code == 0, result.output
+ assert vllm_env["model"] == "vllm"
+ assert vllm_env["model_args"] == {
+ "pretrained": "fake/model",
+ "max_model_len": 4096,
+ "gpu_memory_utilization": 0.5,
+ }
+ assert vllm_env["kwargs"]["batch_size"] == "auto"
+ assert "model_name" not in vllm_env and "hflm" not in vllm_env
+
+
+def test_vllm_backend_serves_adapter_as_lora(vllm_env, tmp_path):
+ adapter = tmp_path / "lora"
+ adapter.mkdir()
+ (adapter / "adapter_config.json").write_text(
+ json.dumps({"base_model_name_or_path": "unsloth/Qwen3-8B-unsloth-bnb-4bit", "r": 64})
+ )
+ (adapter / "tokenizer.json").write_text("{}")
+ result = _run(
+ str(adapter), "--tasks", "gsm8k", "--backend", "vllm", "-b", "8", "-o", str(tmp_path / "o")
+ )
+ assert result.exit_code == 0, result.output
+ args = vllm_env["model_args"]
+ assert args["pretrained"] == "unsloth/Qwen3-8B"
+ assert args["lora_local_path"] == args["tokenizer"] == str(adapter.resolve())
+ assert args["max_lora_rank"] == 64
+ assert vllm_env["kwargs"]["batch_size"] == 8
+
+
+def test_vllm_backend_keeps_default_lora_rank_and_ignores_4bit(vllm_env, tmp_path):
+ adapter = _adapter(tmp_path)
+ result = _run(
+ str(adapter),
+ "--tasks",
+ "gsm8k",
+ "--backend",
+ "vllm",
+ "--load-in-4bit",
+ "-o",
+ str(tmp_path / "o"),
+ )
+ assert result.exit_code == 0, result.output
+ assert vllm_env["model_args"]["pretrained"] == "org/base"
+ assert (
+ "max_lora_rank" not in vllm_env["model_args"] and "tokenizer" not in vllm_env["model_args"]
+ )
+ assert "16-bit" in result.output
+
+
+def test_vllm_backend_refuses_multi_process(vllm_env, monkeypatch):
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ result = _run("fake/model", "--tasks", "gsm8k", "--backend", "vllm")
+ assert result.exit_code == 2 and "--backend hf" in result.output
+ assert "model" not in vllm_env
+
+
[email protected]("installed, ok", [("0.4.11", False), ("0.4.12", True)])
+def test_vllm_backend_needs_lm_eval_0_4_12(vllm_env, monkeypatch, tmp_path, installed, ok):
+ monkeypatch.setattr(evalmod, "version", lambda name: installed)
+ result = _run("fake/model", "--tasks", "gsm8k", "--backend", "vllm", "-o", str(tmp_path))
+ if ok:
+ assert result.exit_code == 0, result.output
+ else:
+ assert result.exit_code == 2 and "model" not in vllm_env
+ assert installed in result.output and "pip install -U lm-eval" in result.output
+
+
+def test_vllm_engine_failure_prints_the_engine_log(vllm_env, monkeypatch, tmp_path):
+ def _fail_in_engine(**kwargs):
+ os.write(2, b"ValueError: No available memory for the cache blocks.\n")
+ raise RuntimeError("Engine core initialization failed. See root cause above.")
+
+ monkeypatch.setattr(sys.modules["lm_eval"], "simple_evaluate", _fail_in_engine)
+ result = _run("fake/model", "--tasks", "gsm8k", "--backend", "vllm", "-o", str(tmp_path))
+ assert result.exit_code != 0
+ assert "No available memory for the cache blocks" in result.output
+
+
+def test_vllm_gpu_probe_leaves_cuda_uninitialised(vllm_env, monkeypatch, tmp_path):
+ monkeypatch.delenv("PYTORCH_NVML_BASED_CUDA_CHECK", raising = False)
+ seen = []
+ sys.modules["torch"].cuda = SimpleNamespace(
+ is_available = lambda: seen.append(os.environ.get("PYTORCH_NVML_BASED_CUDA_CHECK")) or True,
+ device_count = lambda: 1,
+ )
+ result = _run("fake/model", "--tasks", "gsm8k", "--backend", "vllm", "-o", str(tmp_path))
+ assert result.exit_code == 0, result.output
+ assert seen == ["1"]
+
+
+ "platform, cuda, vllm_installed, hint",
+ [
+ ("linux", True, False, "pip install vllm"),
+ ("linux", False, True, "--backend hf"),
+ ("darwin", True, True, "On a Mac use --backend hf"),
+ ],
+)
+def test_vllm_backend_refuses_where_it_cannot_run(
+ env, monkeypatch, platform, cuda, vllm_installed, hint
+):
+ sys.modules["torch"].cuda = SimpleNamespace(is_available = lambda: cuda, device_count = lambda: 1)
+ monkeypatch.setattr(evalmod.sys, "platform", platform)
+ monkeypatch.setattr(evalmod, "version", lambda name: "0.4.13", raising = False)
+ monkeypatch.setattr(
+ evalmod,
+ "find_spec",
+ lambda name: None if name == "vllm" and not vllm_installed else object(),
+ )
+ result = _run("fake/model", "--tasks", "gsm8k", "--backend", "vllm")
+ assert result.exit_code == 2, result.output
+ assert hint in result.output and "model" not in env
+
+
def test_silence_keeps_the_reconfigured_stdout_encoding(tmp_path):
import subprocess
@@ -626,3 +763,23 @@ def test_silence_keeps_the_reconfigured_stdout_encoding(tmp_path):
)
assert proc.returncode == 0, proc.stderr.decode(errors = "replace")
assert "Evaluating gsm8k\u2026" in proc.stdout.decode("utf-8")
+
+
[email protected]("backend", ["unsloth", "hf"])
+def test_gpu_memory_utilization_says_it_only_applies_to_vllm(env, monkeypatch, tmp_path, backend):
+ """Only the vllm branch forwards it, so a user capping memory on another backend would
+ otherwise read the dropped cap as an OOM."""
+ result = _run(
+ "org/model",
+ "--tasks",
+ "gsm8k",
+ "--backend",
+ backend,
+ "--gpu-memory-utilization",
+ "0.5",
+ "-o",
+ str(tmp_path / "o"),
+ )
+ assert result.exit_code == 0, result.output
+ assert "--gpu-memory-utilization applies to --backend vllm" in result.output
+ assert "gpu_memory_utilization" not in (env["model_args"] or {})