Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Add a vLLM backend to unsloth eval (#14228)
Daniel Hand3a82576a430
diff --git a/unsloth_cli/commands/eval.py b/unsloth_cli/commands/eval.pyindex b97b8b91ee1..2c77c3b0012 100644--- a/unsloth_cli/commands/eval.py+++ b/unsloth_cli/commands/eval.py@@ -8,6 +8,7 @@ import reimport shutilimport sysimport tempfile+from importlib.metadata import PackageNotFoundError, versionfrom importlib.util import find_specfrom pathlib import Pathfrom typing import List, Optional, Tuple@@ -22,8 +23,8 @@ _YAML_SUFFIXES = {".yaml", ".yml"}@contextlib.contextmanager-def _silence():- """Send fd 1/2 to devnull (lm-eval/transformers progress bars); yield a Console on the real stdout."""+def _silence(log: Optional[Path] = None):+ """Send fd 1/2 to devnull or `log` (lm-eval/transformers progress bars); yield a Console on the real stdout."""from rich.console import Consolesys.stdout.flush()@@ -37,7 +38,11 @@ def _silence():errors = "replace",)saved_out, saved_err = os.dup(1), os.dup(2)- devnull_fd = os.open(os.devnull, os.O_WRONLY)+ devnull_fd = (+ os.open(log, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)+ if log+ else os.open(os.devnull, os.O_WRONLY)+ )try:os.dup2(devnull_fd, 1)os.dup2(devnull_fd, 2)@@ -185,6 +190,64 @@ def _hflm_4bit(HFLM):return _HFLM4bit+# vLLM's accepted max_lora_rank values (vllm/config/lora.py MaxLoRARanks)+_VLLM_LORA_RANKS = (16, 32, 64, 128, 256, 320, 512)+_BNB_SUFFIX_RE = re.compile(r"(-unsloth)?-bnb-4bit$")+++def _vllm_error() -> Optional[str]:+ if sys.platform == "darwin":+ return "--backend vllm needs an NVIDIA or AMD GPU. On a Mac use --backend hf."+ if find_spec("vllm") is None:+ return (+ "--backend vllm needs vllm (`pip install vllm`). "+ "Use --backend unsloth to evaluate without it."+ )+ try:+ lm_eval_version = version("lm_eval")+ except PackageNotFoundError:+ lm_eval_version = None+ # older lm-eval's vllm wrapper breaks on current vLLM (imports ray, passes the removed swap_space)+ installed = tuple(int(n) for n in re.findall(r"\d+", lm_eval_version or "")[:3])+ if installed and installed < (0, 4, 12):+ return (+ f"--backend vllm needs lm-eval 0.4.12 or newer, but {lm_eval_version} is installed. "+ "Upgrade it with `pip install -U lm-eval`."+ )+ # as vLLM does on import: the default probe initialises CUDA, which its forked engine cannot reuse+ os.environ["PYTORCH_NVML_BASED_CUDA_CHECK"] = "1"+ import torch++ if not torch.cuda.is_available():+ return (+ "--backend vllm needs an NVIDIA or AMD GPU, but none is visible. "+ "Use --backend hf to evaluate on CPU."+ )+ return None+++def _vllm_base(model: str) -> str:+ # vLLM has no bitsandbytes loader+ stripped = _BNB_SUFFIX_RE.sub("", model)+ if stripped != model and model.lower().startswith("unsloth/"):+ typer.echo(f"Note: vLLM cannot load '{model}', using its 16-bit repo '{stripped}'.")+ return stripped+ return model+++def _local_adapter(model: str) -> str:+ if Path(model).is_dir():+ return str(Path(model).resolve())+ from huggingface_hub import snapshot_download+ return snapshot_download(model)+++def _adapter_rank(adapter_dir: str) -> int:+ config = _read_config(Path(adapter_dir) / "adapter_config.json") or {}+ ranks = [config.get("r")] + list((config.get("rank_pattern") or {}).values())+ return max((r for r in ranks if isinstance(r, int)), default = 0)++def _base_vocab_size(model: str) -> int:from transformers import AutoConfigconfig = AutoConfig.from_pretrained(model)@@ -425,7 +488,8 @@ def evaluate("unsloth","--backend",help = "Model backend: 'unsloth' (fast kernels; needs an NVIDIA/AMD/Intel "- "GPU) or 'hf' (plain transformers; works on CPU/MPS/Mac). "+ "GPU), 'hf' (plain transformers; works on CPU/MPS/Mac) or 'vllm' (fastest on "+ "generation tasks; needs vllm and an NVIDIA/AMD GPU, loads 16-bit weights). ""Auto-falls back to 'hf' on Apple Silicon.",),device: Optional[str] = typer.Option(@@ -433,6 +497,11 @@ def evaluate("--device",help = "Device for the hf backend (e.g. cpu, mps, cuda). Default: auto.",),+ gpu_memory_utilization: Optional[float] = typer.Option(+ None,+ "--gpu-memory-utilization",+ help = "Fraction of GPU memory vLLM may use (vllm backend). Default: vLLM's own.",+ ),input_key: str = typer.Option("question", "--input-key", help = "Prompt field for a .jsonl/.csv dataset task."),@@ -458,8 +527,10 @@ def evaluate(bs = int(batch_size) if batch_size.isdigit() else 0if bs <= 0:_fail("--batch-size must be a positive integer or 'auto'.")- if backend not in ("unsloth", "hf"):- _fail(f"--backend must be 'unsloth' or 'hf', got '{backend}'.")+ if backend not in ("unsloth", "hf", "vllm"):+ _fail(f"--backend must be 'unsloth', 'hf' or 'vllm', got '{backend}'.")+ if gpu_memory_utilization is not None and not 0 < gpu_memory_utilization <= 1:+ _fail("--gpu-memory-utilization must be between 0 and 1.")if num_fewshot is not None and num_fewshot < 0:_fail("--num-fewshot must be >= 0.")if limit is not None:@@ -473,17 +544,25 @@ def evaluate(if "lm_eval" not in sys.modules and find_spec("lm_eval") is None:_fail("evaluation requires lm-eval. Install it with `pip install unsloth[eval]`.", code = 1)+ if backend == "vllm":+ if (error := _vllm_error()) is not None:+ _fail(error)+ if device is not None:+ typer.echo("Note: --backend vllm ignores --device; set CUDA_VISIBLE_DEVICES instead.")if backend == "unsloth":with _silence():import unslothif getattr(unsloth, "DEVICE_TYPE", None) == "mlx":typer.echo("Note: Apple Silicon (MLX) detected, falling back to --backend hf.")backend = "hf"+ # only the vllm branch forwards it, and a silently dropped memory cap reads as an OOM+ if backend != "vllm" and gpu_memory_utilization is not None:+ typer.echo(f"Note: --gpu-memory-utilization applies to --backend vllm, not {backend}.")# a pre-loaded model makes lm-eval single-process, so every rank would run every task- if backend == "unsloth" and os.environ.get("WORLD_SIZE", "1") not in ("", "1"):+ if backend in ("unsloth", "vllm") and os.environ.get("WORLD_SIZE", "1") not in ("", "1"):_fail("multi-process launches (accelerate/torchrun) are not supported with "- "--backend unsloth. Use --backend hf for multi-GPU evaluation."+ f"--backend {backend}. Use --backend hf for multi-GPU evaluation.")import inspect@@ -495,6 +574,10 @@ def evaluate(if hf_token:os.environ["HF_TOKEN"] = hf_tokeneffective_base = base_model or resolve_base_model(model)+ if backend == "vllm":+ if load_in_4bit:+ typer.echo("Note: --backend vllm loads 16-bit weights; --load-in-4bit has no effect.")+ load_in_4bit = Falseif load_in_4bit is None:load_in_4bit = _trained_in_4bit(model)if not load_in_4bit:@@ -566,6 +649,26 @@ def evaluate()else:eval_kwargs.update(model = "hf", model_args = model_args, batch_size = bs, device = device)+ elif backend == "vllm":+ model_args = {"pretrained": model, "max_model_len": max_seq_length}+ if effective_base:+ model_args["pretrained"] = _vllm_base(effective_base)+ model_args["lora_local_path"] = _local_adapter(model)+ rank = _adapter_rank(model_args["lora_local_path"])+ if rank > 16:+ model_args["max_lora_rank"] = next(+ (r for r in _VLLM_LORA_RANKS if r >= rank), _VLLM_LORA_RANKS[-1]+ )+ if _has_tokenizer_files(model):+ model_args["tokenizer"] = model_args["lora_local_path"]+ typer.echo(+ f"Evaluating adapter '{model}' on base '{model_args['pretrained']}' with vLLM."+ )+ else:+ model_args["pretrained"] = _vllm_base(model)+ if gpu_memory_utilization is not None:+ model_args["gpu_memory_utilization"] = gpu_memory_utilization+ eval_kwargs.update(model = "vllm", model_args = model_args, batch_size = bs)else:from unsloth import FastLanguageModel@@ -611,10 +714,18 @@ def evaluate(pretrained = lmodel, tokenizer = tokenizer, batch_size = bs, max_length = max_seq_length)- with _silence() as ui:- from rich.status import Status- with Status(f"Evaluating {', '.join(task_names)}…", console = ui, spinner = "dots"):- results = lm_eval.simple_evaluate(**eval_kwargs)+ engine_log = tmp_dir / "vllm.log" if backend == "vllm" else None+ try:+ with _silence(engine_log) as ui:+ from rich.status import Status+ with Status(f"Evaluating {', '.join(task_names)}…", console = ui, spinner = "dots"):+ results = lm_eval.simple_evaluate(**eval_kwargs)+ except Exception:+ # vLLM's engine runs in its own process and logs the real cause there, not in the traceback+ if engine_log is not None and engine_log.exists():+ lines = engine_log.read_text(encoding = "utf-8", errors = "replace").splitlines()+ typer.echo("\n".join(line.rsplit("\r", 1)[-1] for line in lines[-40:]), err = True)+ raisefinally:shutil.rmtree(tmp_dir, ignore_errors = True)diff --git a/unsloth_cli/tests/test_eval.py b/unsloth_cli/tests/test_eval.pyindex db67268872c..160c4cf6c77 100644--- a/unsloth_cli/tests/test_eval.py+++ b/unsloth_cli/tests/test_eval.py@@ -577,7 +577,8 @@ def test_unsafe_code_flag_is_forwarded(env, tmp_path):["--batch-size", "0"],["--batch-size", "-1"],["--batch-size", "big"],- ["--backend", "vllm"],+ ["--backend", "tgi"],+ ["--backend", "vllm", "--gpu-memory-utilization", "1.5"],["--num-fewshot", "-1"],["--limit", "0"],["--limit", "2.5"],@@ -609,6 +610,142 @@ def test_unsloth_backend_refuses_multi_process(env, monkeypatch):assert result.exit_code == 2 and "--backend hf" in result.output+def vllm_env(env, monkeypatch):+ sys.modules["torch"].cuda = SimpleNamespace(is_available = lambda: True, device_count = lambda: 1)+ monkeypatch.setattr(evalmod.sys, "platform", "linux")+ monkeypatch.setattr(evalmod, "version", lambda name: "0.4.13", raising = False)+ return env+++def test_vllm_backend_builds_lm_eval_vllm(vllm_env, tmp_path):+ result = _run(+ "fake/model",+ "--tasks",+ "gsm8k",+ "--backend",+ "vllm",+ "--max-seq-length",+ "4096",+ "--gpu-memory-utilization",+ "0.5",+ "-o",+ str(tmp_path),+ )+ assert result.exit_code == 0, result.output+ assert vllm_env["model"] == "vllm"+ assert vllm_env["model_args"] == {+ "pretrained": "fake/model",+ "max_model_len": 4096,+ "gpu_memory_utilization": 0.5,+ }+ assert vllm_env["kwargs"]["batch_size"] == "auto"+ assert "model_name" not in vllm_env and "hflm" not in vllm_env+++def test_vllm_backend_serves_adapter_as_lora(vllm_env, tmp_path):+ adapter = tmp_path / "lora"+ adapter.mkdir()+ (adapter / "adapter_config.json").write_text(+ json.dumps({"base_model_name_or_path": "unsloth/Qwen3-8B-unsloth-bnb-4bit", "r": 64})+ )+ (adapter / "tokenizer.json").write_text("{}")+ result = _run(+ str(adapter), "--tasks", "gsm8k", "--backend", "vllm", "-b", "8", "-o", str(tmp_path / "o")+ )+ assert result.exit_code == 0, result.output+ args = vllm_env["model_args"]+ assert args["pretrained"] == "unsloth/Qwen3-8B"+ assert args["lora_local_path"] == args["tokenizer"] == str(adapter.resolve())+ assert args["max_lora_rank"] == 64+ assert vllm_env["kwargs"]["batch_size"] == 8+++def test_vllm_backend_keeps_default_lora_rank_and_ignores_4bit(vllm_env, tmp_path):+ adapter = _adapter(tmp_path)+ result = _run(+ str(adapter),+ "--tasks",+ "gsm8k",+ "--backend",+ "vllm",+ "--load-in-4bit",+ "-o",+ str(tmp_path / "o"),+ )+ assert result.exit_code == 0, result.output+ assert vllm_env["model_args"]["pretrained"] == "org/base"+ assert (+ "max_lora_rank" not in vllm_env["model_args"] and "tokenizer" not in vllm_env["model_args"]+ )+ assert "16-bit" in result.output+++def test_vllm_backend_refuses_multi_process(vllm_env, monkeypatch):+ monkeypatch.setenv("WORLD_SIZE", "2")+ result = _run("fake/model", "--tasks", "gsm8k", "--backend", "vllm")+ assert result.exit_code == 2 and "--backend hf" in result.output+ assert "model" not in vllm_env++[email protected]("installed, ok", [("0.4.11", False), ("0.4.12", True)])+def test_vllm_backend_needs_lm_eval_0_4_12(vllm_env, monkeypatch, tmp_path, installed, ok):+ monkeypatch.setattr(evalmod, "version", lambda name: installed)+ result = _run("fake/model", "--tasks", "gsm8k", "--backend", "vllm", "-o", str(tmp_path))+ if ok:+ assert result.exit_code == 0, result.output+ else:+ assert result.exit_code == 2 and "model" not in vllm_env+ assert installed in result.output and "pip install -U lm-eval" in result.output+++def test_vllm_engine_failure_prints_the_engine_log(vllm_env, monkeypatch, tmp_path):+ def _fail_in_engine(**kwargs):+ os.write(2, b"ValueError: No available memory for the cache blocks.\n")+ raise RuntimeError("Engine core initialization failed. See root cause above.")++ monkeypatch.setattr(sys.modules["lm_eval"], "simple_evaluate", _fail_in_engine)+ result = _run("fake/model", "--tasks", "gsm8k", "--backend", "vllm", "-o", str(tmp_path))+ assert result.exit_code != 0+ assert "No available memory for the cache blocks" in result.output+++def test_vllm_gpu_probe_leaves_cuda_uninitialised(vllm_env, monkeypatch, tmp_path):+ monkeypatch.delenv("PYTORCH_NVML_BASED_CUDA_CHECK", raising = False)+ seen = []+ sys.modules["torch"].cuda = SimpleNamespace(+ is_available = lambda: seen.append(os.environ.get("PYTORCH_NVML_BASED_CUDA_CHECK")) or True,+ device_count = lambda: 1,+ )+ result = _run("fake/model", "--tasks", "gsm8k", "--backend", "vllm", "-o", str(tmp_path))+ assert result.exit_code == 0, result.output+ assert seen == ["1"]+++ "platform, cuda, vllm_installed, hint",+ [+ ("linux", True, False, "pip install vllm"),+ ("linux", False, True, "--backend hf"),+ ("darwin", True, True, "On a Mac use --backend hf"),+ ],+)+def test_vllm_backend_refuses_where_it_cannot_run(+ env, monkeypatch, platform, cuda, vllm_installed, hint+):+ sys.modules["torch"].cuda = SimpleNamespace(is_available = lambda: cuda, device_count = lambda: 1)+ monkeypatch.setattr(evalmod.sys, "platform", platform)+ monkeypatch.setattr(evalmod, "version", lambda name: "0.4.13", raising = False)+ monkeypatch.setattr(+ evalmod,+ "find_spec",+ lambda name: None if name == "vllm" and not vllm_installed else object(),+ )+ result = _run("fake/model", "--tasks", "gsm8k", "--backend", "vllm")+ assert result.exit_code == 2, result.output+ assert hint in result.output and "model" not in env++def test_silence_keeps_the_reconfigured_stdout_encoding(tmp_path):import subprocess@@ -626,3 +763,23 @@ def test_silence_keeps_the_reconfigured_stdout_encoding(tmp_path):)assert proc.returncode == 0, proc.stderr.decode(errors = "replace")assert "Evaluating gsm8k\u2026" in proc.stdout.decode("utf-8")++[email protected]("backend", ["unsloth", "hf"])+def test_gpu_memory_utilization_says_it_only_applies_to_vllm(env, monkeypatch, tmp_path, backend):+ """Only the vllm branch forwards it, so a user capping memory on another backend would+ otherwise read the dropped cap as an OOM."""+ result = _run(+ "org/model",+ "--tasks",+ "gsm8k",+ "--backend",+ backend,+ "--gpu-memory-utilization",+ "0.5",+ "-o",+ str(tmp_path / "o"),+ )+ assert result.exit_code == 0, result.output+ assert "--gpu-memory-utilization applies to --backend vllm" in result.output+ assert "gpu_memory_utilization" not in (env["model_args"] or {})