Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Add `unsloth eval` CLI command (#6824)

Nilay3617ea62638b
diff --git a/pyproject.toml b/pyproject.toml
index 21d73fbdae4..20d2eaa9ce9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -339,6 +339,10 @@ windows = [
base = [
"unsloth[huggingface]",
]
+eval = [
+ "unsloth[huggingface]",
+ "lm_eval>=0.4.4",
+]
cu118only = [
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)",
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)",
diff --git a/unsloth_cli/__init__.py b/unsloth_cli/__init__.py
index e0498e44a8a..515c98e6a02 100644
--- a/unsloth_cli/__init__.py
+++ b/unsloth_cli/__init__.py
@@ -83,6 +83,7 @@ else:
from unsloth_cli.commands.chat import chat
from unsloth_cli.commands.start import start_app
from unsloth_cli.commands.export import export, list_checkpoints
+ from unsloth_cli.commands.eval import evaluate as eval_command
from unsloth_cli.commands.studio import (
run as studio_run,
studio_app,
@@ -216,6 +217,7 @@ if not _windows_studio_mutation_entry:
app.command()(inference)
app.command()(chat)
app.command()(export)
+ app.command("eval")(eval_command)
app.command("list-checkpoints")(list_checkpoints)
app.add_typer(
start_app,
diff --git a/unsloth_cli/commands/eval.py b/unsloth_cli/commands/eval.py
new file mode 100644
index 00000000000..72d3d2000bf
--- /dev/null
+++ b/unsloth_cli/commands/eval.py
@@ -0,0 +1,578 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import contextlib
+import json
+import os
+import re
+import shutil
+import sys
+import tempfile
+from importlib.util import find_spec
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+import typer
+import yaml
+
+_HUB_REPO_RE = re.compile(r"[\w.\-]+/[\w.\-]+")
+_TOKENIZER_FILES = ("tokenizer_config.json", "tokenizer.json", "tokenizer.model")
+_DATASET_SUFFIXES = {".jsonl", ".json", ".csv"}
+_YAML_SUFFIXES = {".yaml", ".yml"}
+
+
+def _silence():
+ """Send fd 1/2 to devnull (lm-eval/transformers progress bars); yield a Console on the real stdout."""
+ from rich.console import Console
+
+ sys.stdout.flush()
+ sys.stderr.flush()
+ # keep the entry point's UTF-8 reconfigure (Windows legacy code pages); fdopen would reset it
+ real = os.fdopen(
+ os.dup(1),
+ "w",
+ closefd = True,
+ encoding = getattr(sys.stdout, "encoding", None) or "utf-8",
+ errors = "replace",
+ )
+ saved_out, saved_err = os.dup(1), os.dup(2)
+ devnull_fd = os.open(os.devnull, os.O_WRONLY)
+ try:
+ os.dup2(devnull_fd, 1)
+ os.dup2(devnull_fd, 2)
+ yield Console(file = real)
+ finally:
+ sys.stdout.flush()
+ sys.stderr.flush()
+ os.dup2(saved_out, 1)
+ os.dup2(saved_err, 2)
+ for fd in (saved_out, saved_err, devnull_fd):
+ os.close(fd)
+ real.close()
+
+
+def _is_hub_id(model: str) -> bool:
+ return not Path(model).exists() and bool(_HUB_REPO_RE.fullmatch(model))
+
+
+def resolve_base_model(model: str) -> Optional[str]:
+ """base_model_name_or_path of a LoRA adapter (local dir or Hub repo), else None."""
+ path = Path(model)
+ config = path / "adapter_config.json"
+ if not path.is_dir():
+ if not _is_hub_id(model):
+ return None
+ try:
+ from huggingface_hub import hf_hub_download
+ config = Path(hf_hub_download(model, "adapter_config.json"))
+ except Exception:
+ return None
+ try:
+ data = json.loads(config.read_text(encoding = "utf-8"))
+ except (ValueError, OSError):
+ return None
+ return data.get("base_model_name_or_path") if isinstance(data, dict) else None
+
+
+def _has_tokenizer_files(model: str) -> bool:
+ path = Path(model)
+ if path.is_dir():
+ return any((path / name).exists() for name in _TOKENIZER_FILES)
+ if not _is_hub_id(model):
+ return False
+ try:
+ from huggingface_hub import list_repo_files
+ files = set(list_repo_files(model))
+ except Exception:
+ return False
+ return any(name in files for name in _TOKENIZER_FILES)
+
+
+def _hf_device_error(device: str) -> Optional[str]:
+ # HFLM silently falls back to its default device on any string outside its device_list
+ match = re.fullmatch(r"(cpu|cuda|mps|npu|xpu|hpu)(?::(0|[1-9]\d*))?", device)
+ kind, index = (match.group(1), match.group(2)) if match else (None, None)
+ if (
+ kind is None
+ or (kind == "cpu" and index is not None)
+ or (kind == "mps" and index not in (None, "0"))
+ or (kind in ("npu", "xpu", "hpu") and index is None)
+ ):
+ return (
+ f"invalid --device '{device}': use 'cpu', 'cuda[:<index>]', 'mps', "
+ "or '<npu|xpu|hpu>:<index>'."
+ )
+ if kind == "cpu":
+ return None
+ import torch
+
+ backend = getattr(torch.backends, "mps", None) if kind == "mps" else getattr(torch, kind, None)
+ try:
+ available = bool(backend is not None and backend.is_available())
+ except Exception:
+ available = False
+ if not available:
+ return f"--device {device} requested but {kind.upper()} is not available."
+ if index is not None and kind != "mps":
+ try:
+ count = int(backend.device_count())
+ except Exception:
+ count = 0
+ if int(index) >= count:
+ return f"--device {device} requested but only {count} {kind.upper()} device(s) are available."
+ return None
+
+
+def _hflm_4bit(HFLM):
+ # transformers 5 removed from_pretrained(load_in_4bit=...), which HFLM forwards verbatim
+ class _HFLM4bit(HFLM):
+ def _create_model(
+ self,
+ *args,
+ quantization_config = None,
+ load_in_4bit = False,
+ **kwargs,
+ ):
+ if load_in_4bit and quantization_config is None:
+ from transformers import BitsAndBytesConfig
+ quantization_config = BitsAndBytesConfig(load_in_4bit = True)
+ return super()._create_model(*args, quantization_config = quantization_config, **kwargs)
+
+ return _HFLM4bit
+
+
+def _base_vocab_size(model: str) -> int:
+ from transformers import AutoConfig
+ config = AutoConfig.from_pretrained(model)
+ return getattr(getattr(config, "text_config", None) or config, "vocab_size", 0) or 0
+
+
+class _TaskYamlLoader(yaml.SafeLoader):
+ """safe_load that tolerates lm-eval's custom tags (!function utils.fn)."""
+
+
+_TaskYamlLoader.add_multi_constructor(
+ "!", lambda loader, suffix, node: getattr(node, "value", None)
+)
+
+
+def _doc_column(key: str) -> str:
+ # Jinja stringifies the value but cannot parse non-identifier keys; lm-eval reads a bare column name raw
+ import keyword
+ if key.isidentifier() and not keyword.iskeyword(key) and key not in ("true", "false", "none"):
+ return "{{" + key + "}}"
+ return key
+
+
+def make_dataset_task(
+ data_file: Path, input_key: str, target_key: str, out_dir: Path, name: str
+) -> None:
+ """Write an exact-match generate_until task over a .jsonl/.json/.csv file."""
+ builder = "json" if data_file.suffix.lower() in {".json", ".jsonl"} else "csv"
+ spec = {
+ "task": name,
+ "dataset_path": builder,
+ "dataset_kwargs": {"data_files": str(data_file.resolve())},
+ "test_split": "train",
+ "fewshot_split": "train",
+ "output_type": "generate_until",
+ "doc_to_text": _doc_column(input_key),
+ "doc_to_target": _doc_column(target_key),
+ "generation_kwargs": {"until": ["\n"]},
+ # strip so " 2" matches gold "2"; with one capture group re.findall yields the group text
+ "filter_list": [
+ {
+ "name": "strip",
+ "filter": [
+ {"function": "regex", "regex_pattern": r"^\s*(.*?)\s*$", "group_select": 0},
+ {"function": "take_first"},
+ ],
+ },
+ ],
+ "metric_list": [{"metric": "exact_match", "aggregation": "mean", "higher_is_better": True}],
+ }
+ out_dir.mkdir(parents = True, exist_ok = True)
+ (out_dir / f"{name}.yaml").write_text(yaml.safe_dump(spec, sort_keys = False), encoding = "utf-8")
+
+
+def _yaml_task_name(path: Path) -> Tuple[str, bool]:
+ """(registered name, needs its directory on the include path)."""
+ text = path.read_text(encoding = "utf-8")
+ try:
+ spec = yaml.load(text, Loader = _TaskYamlLoader)
+ except yaml.YAMLError as e:
+ raise ValueError(f"Invalid YAML in custom task file '{path}': {e}") from e
+ if not isinstance(spec, dict):
+ raise ValueError(f"Custom task file '{path}' must define a YAML mapping.")
+ is_group = isinstance(spec.get("task"), list)
+ name = spec.get("group") if is_group else spec.get("task")
+ if not name or not isinstance(name, str):
+ raise ValueError(
+ f"Custom task file '{path}' needs a top-level 'task:' name ('group:' for a task list)."
+ )
+ references_siblings = is_group or "include" in spec or "!function" in text
+ if references_siblings and path.suffix.lower() == ".yml":
+ raise ValueError(
+ f"Custom task file '{path}' references sibling files but lm-eval only indexes "
+ ".yaml files. Rename it (and the files it references) to .yaml."
+ )
+ return name, references_siblings
+
+
+def resolve_tasks(
+ tasks: str, input_key: str, target_key: str, tmp_dir: Path
+) -> Tuple[List[str], List[str]]:
+ """Split --tasks into lm-eval task names plus the include paths custom files need."""
+ entries = [e.strip() for e in tasks.split(",") if e.strip()]
+ if not entries:
+ raise ValueError("No tasks provided. Pass --tasks with at least one task.")
+ plain = {e for e in entries if Path(e).suffix.lower() not in _DATASET_SUFFIXES | _YAML_SUFFIXES}
+ names: List[str] = []
+ include_paths: List[str] = []
+
+ def add(name: str, include: Path) -> None:
+ if name in names:
+ raise ValueError(f"Duplicate task '{name}' in --tasks.")
+ names.append(name)
+ if str(include) not in include_paths:
+ include_paths.append(str(include))
+
+ for entry in entries:
+ path = Path(entry)
+ suffix = path.suffix.lower()
+ if suffix in _YAML_SUFFIXES:
+ if not path.exists():
+ raise FileNotFoundError(f"Custom task file not found: {entry}")
+ name, needs_dir = _yaml_task_name(path)
+ if needs_dir:
+ add(name, path.resolve().parent)
+ else:
+ # copied alone so an unrelated broken yaml beside it cannot break lm-eval's index
+ custom_dir = (tmp_dir / "custom").resolve()
+ custom_dir.mkdir(parents = True, exist_ok = True)
+ shutil.copy2(path, custom_dir / f"{name}.yaml")
+ add(name, custom_dir)
+ elif suffix in _DATASET_SUFFIXES:
+ if not path.exists():
+ raise FileNotFoundError(f"Dataset file not found: {entry}")
+ # an include-path task overrides a registered one of the same name, so keep clear of
+ # names requested alongside (gsm8k,./gsm8k.jsonl)
+ name, counter = path.stem, 2
+ while name in names or name in plain:
+ name, counter = f"{path.stem}_{counter}", counter + 1
+ if name != path.stem:
+ typer.echo(f"Note: running dataset '{path.name}' as task '{name}'.")
+ gen_dir = (tmp_dir / "generated").resolve()
+ make_dataset_task(path, input_key, target_key, gen_dir, name)
+ add(name, gen_dir)
+ else:
+ if entry in names:
+ raise ValueError(f"Duplicate task '{entry}' in --tasks.")
+ names.append(entry)
+ return names, include_paths
+
+
+def _expand_tasks(task_manager, names: List[str]) -> List[str]:
+ """Expand globs (mmlu_*) like lm-eval's CLI and reject unknown names."""
+ known = set(getattr(task_manager, "all_tasks", None) or [])
+ if not known:
+ return names
+ expanded: List[str] = []
+ for name in names:
+ if any(ch in name for ch in "*?["):
+ matches = task_manager.match_tasks([name])
+ if not matches:
+ raise ValueError(f"no tasks match pattern '{name}'.")
+ expanded.extend(m for m in matches if m not in expanded)
+ elif name not in known:
+ raise ValueError(
+ f"unknown task '{name}'. Pass a built-in task name, a .yaml task file, "
+ "or a .jsonl/.csv dataset."
+ )
+ elif name not in expanded:
+ expanded.append(name)
+ return expanded
+
+
+def _metric_number(value):
+ # numpy float32/int64 are not int/float subclasses: unwrap scalars via item()
+ if not isinstance(value, (int, float)) and callable(getattr(value, "item", None)):
+ try:
+ value = value.item()
+ except Exception:
+ return None
+ return value if isinstance(value, (int, float)) else None
+
+
+def _json_default(value):
+ try:
+ return value.tolist()
+ except Exception:
+ return str(value)
+
+
+def _render_results(results: dict) -> None:
+ from rich.console import Console
+ from rich.table import Table
+
+ table = Table(title = "Evaluation results")
+ table.add_column("Task", style = "cyan")
+ table.add_column("Metric")
+ table.add_column("Value", justify = "right")
+ table.add_column("± stderr", justify = "right")
+ rows = dict(results.get("results") or {})
+ for task, metrics in (results.get("groups") or {}).items():
+ rows.setdefault(task, metrics)
+ for task, metrics in rows.items():
+ for key, raw_value in metrics.items():
+ value = _metric_number(raw_value)
+ if key == "alias" or "_stderr" in key or value is None:
+ continue
+ metric, _, flt = key.partition(",")
+ stderr = _metric_number(
+ metrics.get(f"{metric}_stderr,{flt}" if flt else f"{metric}_stderr")
+ )
+ table.add_row(task, key, f"{value:.4f}", f"{stderr:.4f}" if stderr is not None else "—")
+ Console().print(table)
+
+
+def _fail(message: str, code: int = 2):
+ typer.echo(f"Error: {message}", err = True)
+ raise typer.Exit(code = code)
+
+
+def evaluate(
+ model: str = typer.Argument(
+ ..., help = "Path to a checkpoint/adapter directory or a HuggingFace model id."
+ ),
+ tasks: str = typer.Option(
+ ...,
+ "--tasks",
+ "-t",
+ help = "Comma-separated built-in task names (e.g. mmlu,gsm8k), or a path to a "
+ "custom .yaml task or a .jsonl/.csv dataset.",
+ ),
+ base_model: Optional[str] = typer.Option(
+ None,
+ "--base-model",
+ help = "Base model for a LoRA adapter. Auto-detected from adapter_config.json; "
+ "set this to override a moved/renamed base.",
+ ),
+ num_fewshot: Optional[int] = typer.Option(
+ None, "--num-fewshot", "-n", help = "Few-shot examples (default: per-task)."
+ ),
+ limit: Optional[float] = typer.Option(
+ None,
+ "--limit",
+ help = "Cap examples per task (for quick smoke tests): a whole count, or a "
+ "fraction between 0 and 1 for a proportion of each task.",
+ ),
+ batch_size: str = typer.Option("auto", "--batch-size", "-b", help = "Batch size, or 'auto'."),
+ max_seq_length: int = typer.Option(
+ 2048, "--max-seq-length", help = "Max sequence length for the model."
+ ),
+ load_in_4bit: bool = typer.Option(
+ True, "--load-in-4bit/--no-load-in-4bit", help = "Load the model in 4-bit."
+ ),
+ backend: str = typer.Option(
+ "unsloth",
+ "--backend",
+ help = "Model backend: 'unsloth' (fast kernels; needs an NVIDIA/AMD/Intel "
+ "GPU) or 'hf' (plain transformers; works on CPU/MPS/Mac). "
+ "Auto-falls back to 'hf' on Apple Silicon.",
+ ),
+ device: Optional[str] = typer.Option(
+ None,
+ "--device",
+ help = "Device for the hf backend (e.g. cpu, mps, cuda). Default: auto.",
+ ),
+ input_key: str = typer.Option(
+ "question", "--input-key", help = "Prompt field for a .jsonl/.csv dataset task."
+ ),
+ target_key: str = typer.Option(
+ "answer", "--target-key", help = "Answer field for a .jsonl/.csv dataset task."
+ ),
+ output_dir: Path = typer.Option(
+ Path("./eval_results"), "--output-dir", "-o", help = "Directory for results.json."
+ ),
+ confirm_run_unsafe_code: bool = typer.Option(
+ False,
+ "--confirm-run-unsafe-code",
+ help = "Allow tasks that lm-eval marks unsafe (e.g. humaneval): they execute "
+ "model-generated code on this machine. Off by default.",
+ ),
+ hf_token: Optional[str] = typer.Option(
+ None, "--hf-token", envvar = "HF_TOKEN", help = "HuggingFace token if needed."
+ ),
+):
+ """Evaluate a checkpoint or LoRA adapter using lm-eval-harness."""
+ bs = batch_size
+ if batch_size != "auto":
+ bs = int(batch_size) if batch_size.isdigit() else 0
+ if bs <= 0:
+ _fail("--batch-size must be a positive integer or 'auto'.")
+ if backend not in ("unsloth", "hf"):
+ _fail(f"--backend must be 'unsloth' or 'hf', got '{backend}'.")
+ if num_fewshot is not None and num_fewshot < 0:
+ _fail("--num-fewshot must be >= 0.")
+ if limit is not None:
+ # lm-eval reads < 1 as a fraction and int()s counts, so 2.5 would silently run 2
+ if limit <= 0 or (limit >= 1 and not limit.is_integer()):
+ _fail("--limit must be a whole count or a fraction between 0 and 1.")
+ limit = int(limit) if limit >= 1 else limit
+ if max_seq_length <= 0:
+ _fail("--max-seq-length must be a positive integer.")
+ # find_spec, not import: lm_eval imports transformers, which must come after unsloth
+ if "lm_eval" not in sys.modules and find_spec("lm_eval") is None:
+ _fail("evaluation requires lm-eval. Install it with `pip install unsloth[eval]`.", code = 1)
+
+ if backend == "unsloth":
+ with _silence():
+ import unsloth
+ if getattr(unsloth, "DEVICE_TYPE", None) == "mlx":
+ typer.echo("Note: Apple Silicon (MLX) detected, falling back to --backend hf.")
+ backend = "hf"
+ # a pre-loaded model makes lm-eval single-process, so every rank would run every task
+ if backend == "unsloth" and os.environ.get("WORLD_SIZE", "1") not in ("", "1"):
+ _fail(
+ "multi-process launches (accelerate/torchrun) are not supported with "
+ "--backend unsloth. Use --backend hf for multi-GPU evaluation."
+ )
+
+ import inspect
+
+ import lm_eval
+ from lm_eval.models.huggingface import HFLM
+ from lm_eval.tasks import TaskManager
+
+ if hf_token:
+ os.environ["HF_TOKEN"] = hf_token
+ effective_base = base_model or resolve_base_model(model)
+
+ tmp_dir = Path(tempfile.mkdtemp(prefix = "unsloth_eval_"))
+ try:
+ try:
+ task_names, include_paths = resolve_tasks(tasks, input_key, target_key, tmp_dir)
+ task_manager = TaskManager(include_path = include_paths or None)
+ task_names = _expand_tasks(task_manager, task_names)
+ except (FileNotFoundError, ValueError) as e:
+ _fail(str(e))
+
+ typer.echo(f"Running tasks: {', '.join(task_names)} (backend: {backend})")
+ eval_kwargs = dict(
+ tasks = task_names,
+ num_fewshot = num_fewshot,
+ limit = limit,
+ task_manager = task_manager,
+ log_samples = False,
+ )
+ # lm-eval < 0.4.8 has no unsafe-code gate
+ if "confirm_run_unsafe_code" in inspect.signature(lm_eval.simple_evaluate).parameters:
+ eval_kwargs["confirm_run_unsafe_code"] = confirm_run_unsafe_code
+ elif confirm_run_unsafe_code:
+ typer.echo(
+ "Note: this lm-eval version has no unsafe-code gate; the flag has no effect."
+ )
+
+ if backend == "hf":
+ import torch
+
+ if device is None:
+ mps = getattr(torch.backends, "mps", None)
+ device = (
+ "cuda"
+ if torch.cuda.is_available()
+ else "mps"
+ if mps and mps.is_available()
+ else "cpu"
+ )
+ elif (device_error := _hf_device_error(device)) is not None:
+ _fail(device_error)
+ if bs == "auto" and not device.startswith("cuda"):
+ typer.echo(
+ "Note: batch size 'auto' is slow on CPU/MPS, using 1 (override with --batch-size)."
+ )
+ bs = 1
+ model_args = {"pretrained": model}
+ if effective_base:
+ model_args = {"pretrained": effective_base, "peft": model}
+ if _has_tokenizer_files(model):
+ model_args["tokenizer"] = model
+ typer.echo(f"Evaluating adapter '{model}' on base '{effective_base}'.")
+ model_args["max_length"] = max_seq_length
+ if load_in_4bit and device.startswith("cuda"):
+ if find_spec("bitsandbytes") is not None:
+ model_args["load_in_4bit"] = True
+ else:
+ typer.echo("Note: bitsandbytes is not installed, loading in full precision.")
+ if model_args.get("load_in_4bit"):
+ with _silence():
+ eval_kwargs["model"] = _hflm_4bit(HFLM)(
+ **model_args, batch_size = bs, device = device
+ )
+ else:
+ eval_kwargs.update(model = "hf", model_args = model_args, batch_size = bs, device = device)
+ else:
+ from unsloth import FastLanguageModel
+
+ if load_in_4bit and find_spec("bitsandbytes") is None:
+ typer.echo("Note: bitsandbytes is not installed, loading in full precision.")
+ load_in_4bit = False
+ load_kwargs = dict(
+ max_seq_length = max_seq_length, load_in_4bit = load_in_4bit, token = hf_token or None
+ )
+ adapter_vocab = None
+ if effective_base and _has_tokenizer_files(model):
+ from transformers import AutoTokenizer
+ adapter_vocab = len(AutoTokenizer.from_pretrained(model))
+ typer.echo(
+ f"Loading base model '{effective_base}' with adapter '{model}'..."
+ if effective_base
+ else f"Loading model: {model}"
+ )
+ with _silence():
+ if effective_base and not base_model:
+ # Unsloth's own adapter path applies its PEFT hooks (grouped-linear LoRA, Gemma 4)
+ if adapter_vocab and adapter_vocab > _base_vocab_size(effective_base):
+ load_kwargs["resize_model_vocab"] = adapter_vocab
+ lmodel, tokenizer = FastLanguageModel.from_pretrained(
+ model_name = model, **load_kwargs
+ )
+ else:
+ lmodel, tokenizer = FastLanguageModel.from_pretrained(
+ model_name = effective_base or model, **load_kwargs
+ )
+ if effective_base and base_model:
+ # --base-model overrides adapter_config.json, which Unsloth's adapter path always reads
+ if adapter_vocab:
+ from transformers import AutoTokenizer
+ tokenizer = AutoTokenizer.from_pretrained(model)
+ # grow only: padded vocabularies (Qwen3) have more rows than tokens
+ if adapter_vocab > lmodel.get_input_embeddings().weight.shape[0]:
+ lmodel.resize_token_embeddings(adapter_vocab)
+ from peft import PeftModel
+ lmodel = PeftModel.from_pretrained(lmodel, model)
+ FastLanguageModel.for_inference(lmodel)
+ eval_kwargs["model"] = HFLM(
+ pretrained = lmodel, tokenizer = tokenizer, batch_size = bs, max_length = max_seq_length
+ )
+
+ with _silence() as ui:
+ from rich.status import Status
+ with Status(f"Evaluating {', '.join(task_names)}…", console = ui, spinner = "dots"):
+ results = lm_eval.simple_evaluate(**eval_kwargs)
+ finally:
+ shutil.rmtree(tmp_dir, ignore_errors = True)
+
+ if results is None:
+ # lm-eval returns None on non-zero ranks
+ if os.environ.get("RANK", "0") != "0" or os.environ.get("LOCAL_RANK", "0") != "0":
+ return
+ _fail("evaluation returned no results.", code = 1)
+
+ _render_results(results)
+ output_dir.mkdir(parents = True, exist_ok = True)
+ results_path = output_dir / "results.json"
+ results_path.write_text(json.dumps(results, indent = 2, default = _json_default), encoding = "utf-8")
+ typer.echo(f"Saved results to: {results_path}")
diff --git a/unsloth_cli/tests/test_eval.py b/unsloth_cli/tests/test_eval.py
new file mode 100644
index 00000000000..91fd75c751e
--- /dev/null
+++ b/unsloth_cli/tests/test_eval.py
@@ -0,0 +1,534 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import types
+from pathlib import Path
+from types import SimpleNamespace
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(_REPO_ROOT) not in sys.path:
+ sys.path.insert(0, str(_REPO_ROOT))
+
+import pytest
+import typer
+import yaml
+from typer.testing import CliRunner
+
+import unsloth_cli.commands.eval as evalmod
+
+
+def _eval_app():
+ cli = typer.Typer()
+ cli.command()(evalmod.evaluate)
+ return cli
+
+
+def _run(*args):
+ return CliRunner().invoke(_eval_app(), list(args))
+
+
+ "content, expected",
+ [
+ (json.dumps({"base_model_name_or_path": "unsloth/Llama-3.2-1B"}), "unsloth/Llama-3.2-1B"),
+ ("not json", None),
+ ("[1, 2]", None),
+ ],
+)
+def test_resolve_base_model_from_local_adapter(tmp_path, content, expected):
+ (tmp_path / "adapter_config.json").write_text(content)
+ assert evalmod.resolve_base_model(str(tmp_path)) == expected
+
+
+def test_resolve_base_model_plain_dir_and_non_hub_name(tmp_path):
+ assert evalmod.resolve_base_model(str(tmp_path)) is None
+ assert evalmod.resolve_base_model("not a repo id") is None
+
+
+ "key, expected",
+ [
+ ("question", "{{question}}"),
+ ("expected answer", "expected answer"),
+ ("if", "if"),
+ ("none", "none"),
+ ],
+)
+def test_doc_column(key, expected):
+ assert evalmod._doc_column(key) == expected
+
+
+def test_resolve_tasks_plain_names():
+ names, includes = evalmod.resolve_tasks("gsm8k, mmlu ,", "q", "a", Path("/unused"))
+ assert names == ["gsm8k", "mmlu"]
+ assert includes == []
+
+
[email protected]("tasks", ["", " , ", "gsm8k,gsm8k"])
+def test_resolve_tasks_rejects_empty_and_duplicates(tasks, tmp_path):
+ with pytest.raises(ValueError):
+ evalmod.resolve_tasks(tasks, "q", "a", tmp_path)
+
+
+def test_resolve_tasks_missing_files(tmp_path):
+ for missing in ("nope.jsonl", "nope.yaml"):
+ with pytest.raises(FileNotFoundError):
+ evalmod.resolve_tasks(str(tmp_path / missing), "q", "a", tmp_path)
+
+
+def test_dataset_task_yaml(tmp_path):
+ data = tmp_path / "qa.jsonl"
+ data.write_text('{"question": "1+1?", "answer": "2"}\n')
+ names, includes = evalmod.resolve_tasks(str(data), "question", "answer", tmp_path / "tmp")
+ assert names == ["qa"]
+ spec = yaml.safe_load(Path(includes[0], "qa.yaml").read_text())
+ assert spec["dataset_path"] == "json"
+ assert spec["dataset_kwargs"] == {"data_files": str(data.resolve())}
+ assert spec["doc_to_text"] == "{{question}}"
+ assert spec["fewshot_split"] == "train"
+ assert spec["metric_list"][0]["metric"] == "exact_match"
+
+
+def test_dataset_task_stays_clear_of_names_requested_alongside(tmp_path):
+ (tmp_path / "gsm8k.jsonl").write_text("{}\n")
+ (tmp_path / "sub").mkdir()
+ (tmp_path / "sub" / "gsm8k.csv").write_text("question,answer\n")
+ names, _ = evalmod.resolve_tasks(
+ f"gsm8k,{tmp_path / 'gsm8k.jsonl'},{tmp_path / 'sub' / 'gsm8k.csv'}",
+ "q",
+ "a",
+ tmp_path / "tmp",
+ )
+ assert names == ["gsm8k", "gsm8k_2", "gsm8k_3"]
+
+
+def test_standalone_yaml_is_copied_alone(tmp_path):
+ (tmp_path / "broken.yaml").write_text("task: [unclosed\n")
+ task = tmp_path / "mine.yml"
+ task.write_text("task: my_task\ndataset_path: json\n")
+ names, includes = evalmod.resolve_tasks(str(task), "q", "a", tmp_path / "tmp")
+ assert names == ["my_task"]
+ assert sorted(p.name for p in Path(includes[0]).iterdir()) == ["my_task.yaml"]
+
+
+def test_group_yaml_puts_its_directory_on_the_include_path(tmp_path):
+ group = tmp_path / "suite.yaml"
+ group.write_text("group: suite\ntask: [a, b]\n")
+ names, includes = evalmod.resolve_tasks(str(group), "q", "a", tmp_path / "tmp")
+ assert names == ["suite"]
+ assert includes == [str(tmp_path.resolve())]
+
+
+def test_function_tag_yaml_uses_its_directory(tmp_path):
+ task = tmp_path / "fn.yaml"
+ task.write_text("task: fn_task\ndoc_to_text: !function utils.doc_to_text\n")
+ names, includes = evalmod.resolve_tasks(str(task), "q", "a", tmp_path / "tmp")
+ assert names == ["fn_task"]
+ assert includes == [str(tmp_path.resolve())]
+
+
+ "filename, content",
+ [
+ ("bad.yaml", "task: [unclosed\n"),
+ ("list.yaml", "- a\n- b\n"),
+ ("noname.yaml", "dataset_path: json\n"),
+ ("group_no_name.yaml", "task: [a, b]\n"),
+ ("sibling.yml", "group: g\ntask: [a]\n"),
+ ],
+)
+def test_invalid_yaml_tasks(tmp_path, filename, content):
+ path = tmp_path / filename
+ path.write_text(content)
+ with pytest.raises(ValueError):
+ evalmod.resolve_tasks(str(path), "q", "a", tmp_path / "tmp")
+
+
+def _fake_torch(
+ cuda_count = 0,
+ mps = False,
+ xpu_count = 0,
+):
+ xpu = SimpleNamespace(is_available = lambda: xpu_count > 0, device_count = lambda: xpu_count)
+ return SimpleNamespace(
+ cuda = SimpleNamespace(is_available = lambda: cuda_count > 0, device_count = lambda: cuda_count),
+ backends = SimpleNamespace(mps = SimpleNamespace(is_available = lambda: mps)),
+ xpu = xpu,
+ )
+
+
+ "device, ok",
+ [
+ ("cpu", True),
+ ("cuda", True),
+ ("cuda:1", True),
+ ("cuda:2", False),
+ ("cuda:01", False),
+ ("cpu:0", False),
+ ("mps", False),
+ ("xpu", False),
+ ("xpu:0", True),
+ ("npu:0", False),
+ ("tpu", False),
+ ],
+)
+def test_hf_device_error(monkeypatch, device, ok):
+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(cuda_count = 2, xpu_count = 1))
+ assert (evalmod._hf_device_error(device) is None) is ok
+
+
+def test_hflm_4bit_passes_a_quantization_config_not_load_in_4bit():
+ # transformers without torch cannot build BitsAndBytesConfig (macOS runners)
+ pytest.importorskip("torch")
+ pytest.importorskip("transformers")
+ from transformers import BitsAndBytesConfig
+
+ class _Base:
+ def _create_model(
+ self,
+ pretrained,
+ quantization_config = None,
+ **kwargs,
+ ):
+ self.seen = dict(kwargs, quantization_config = quantization_config)
+
+ lm = evalmod._hflm_4bit(_Base)()
+ lm._create_model(pretrained = "x", load_in_4bit = True, quantization_config = None)
+ assert "load_in_4bit" not in lm.seen
+ assert isinstance(lm.seen["quantization_config"], BitsAndBytesConfig)
+ assert lm.seen["quantization_config"].load_in_4bit is True
+
+ prequantized = object()
+ lm._create_model(pretrained = "x", load_in_4bit = True, quantization_config = prequantized)
+ assert lm.seen["quantization_config"] is prequantized
+
+
+def test_metric_number_and_render(capsys):
+ np = pytest.importorskip("numpy")
+ assert evalmod._metric_number(np.float32(0.5)) == 0.5
+ assert evalmod._metric_number("x") is None
+ evalmod._render_results(
+ {
+ "results": {
+ "gsm8k": {"exact_match,strict": np.float32(0.3), "exact_match_stderr,strict": 0.01}
+ },
+ "groups": {"mmlu": {"acc,none": 0.45, "alias": "mmlu"}},
+ }
+ )
+ out = capsys.readouterr().out
+ assert "0.3000" in out and "0.0100" in out and "0.4500" in out
+
+
+def test_eval_missing_lm_eval_shows_hint(monkeypatch):
+ monkeypatch.delitem(sys.modules, "lm_eval", raising = False)
+ monkeypatch.setattr(evalmod, "find_spec", lambda name: None)
+ result = _run("fake/model", "--tasks", "gsm8k")
+ assert result.exit_code == 1, result.output
+ assert "pip install unsloth[eval]" in result.output
+
+
+def env(monkeypatch):
+ calls = {"task_managers": 0}
+
+ class _FakeFLM:
+ @classmethod
+ def from_pretrained(
+ cls,
+ model_name = None,
+ **kw,
+ ):
+ calls["model_name"] = model_name
+ calls["load_kwargs"] = kw
+ model = SimpleNamespace(
+ get_input_embeddings = lambda: SimpleNamespace(weight = SimpleNamespace(shape = (10, 4))),
+ resize_token_embeddings = lambda n: calls.__setitem__("resized", n),
+ )
+ return model, SimpleNamespace(name = "tok")
+
+ @classmethod
+ def for_inference(cls, model):
+ calls["for_inference"] = True
+
+ class _FakeHFLM:
+ def __init__(
+ self,
+ pretrained = None,
+ tokenizer = None,
+ batch_size = None,
+ max_length = None,
+ **kw,
+ ):
+ calls["hflm"] = dict(
+ kw, pretrained = pretrained, batch_size = batch_size, max_length = max_length
+ )
+
+ class _FakeTaskManager:
+ def __init__(self, include_path = None):
+ calls["task_managers"] += 1
+ calls["include_path"] = include_path
+ self.all_tasks = ["gsm8k", "mmlu", "mmlu_a", "mmlu_b"]
+ for directory in include_path or []:
+ self.all_tasks += [p.stem for p in Path(directory).glob("*.yaml")]
+
+ def match_tasks(self, patterns):
+ import fnmatch
+ return [t for p in patterns for t in self.all_tasks if fnmatch.fnmatch(t, p)]
+
+ def _simple_evaluate(
+ model = None,
+ model_args = None,
+ tasks = None,
+ confirm_run_unsafe_code = False,
+ **kw,
+ ):
+ calls.update(model = model, model_args = model_args, tasks = tasks, kwargs = kw)
+ calls["confirm_run_unsafe_code"] = confirm_run_unsafe_code
+ np = pytest.importorskip("numpy")
+ return {"results": {"gsm8k": {"exact_match,strict": np.float64(0.42)}}, "configs": {}}
+
+ unsloth_mod = types.ModuleType("unsloth")
+ unsloth_mod.FastLanguageModel = _FakeFLM
+ lm_eval_mod = types.ModuleType("lm_eval")
+ lm_eval_mod.simple_evaluate = _simple_evaluate
+ hf_mod = types.ModuleType("lm_eval.models.huggingface")
+ hf_mod.HFLM = _FakeHFLM
+ tasks_mod = types.ModuleType("lm_eval.tasks")
+ tasks_mod.TaskManager = _FakeTaskManager
+ peft_mod = types.ModuleType("peft")
+ peft_mod.PeftModel = SimpleNamespace(
+ from_pretrained = lambda m, path: calls.__setitem__("peft", path) or m
+ )
+ hub_mod = types.ModuleType("huggingface_hub")
+ hub_mod.hf_hub_download = hub_mod.list_repo_files = lambda *a, **k: (_ for _ in ()).throw(
+ OSError()
+ )
+ for name, mod in {
+ "unsloth": unsloth_mod,
+ "torch": _fake_torch(),
+ "huggingface_hub": hub_mod,
+ "peft": peft_mod,
+ "lm_eval": lm_eval_mod,
+ "lm_eval.models": types.ModuleType("lm_eval.models"),
+ "lm_eval.models.huggingface": hf_mod,
+ "lm_eval.tasks": tasks_mod,
+ }.items():
+ monkeypatch.setitem(sys.modules, name, mod)
+ monkeypatch.setattr(evalmod, "find_spec", lambda name: object())
+ monkeypatch.delenv("WORLD_SIZE", raising = False)
+ return calls
+
+
+def test_unsloth_backend_writes_results(env, tmp_path):
+ result = _run("fake/model", "--tasks", "gsm8k", "-o", str(tmp_path / "out"))
+ assert result.exit_code == 0, result.output
+ assert env["model_name"] == "fake/model"
+ assert env["load_kwargs"]["load_in_4bit"] is True
+ assert env["for_inference"]
+ assert env["hflm"]["max_length"] == 2048 and env["hflm"]["batch_size"] == "auto"
+ assert env["tasks"] == ["gsm8k"] and env["task_managers"] == 1
+ assert env["kwargs"]["log_samples"] is False
+ saved = json.loads((tmp_path / "out" / "results.json").read_text())
+ assert saved["results"]["gsm8k"]["exact_match,strict"] == 0.42
+
+
+def _adapter(tmp_path, tokenizer = False):
+ adapter = tmp_path / "lora"
+ adapter.mkdir()
+ (adapter / "adapter_config.json").write_text(
+ json.dumps({"base_model_name_or_path": "org/base"})
+ )
+ if tokenizer:
+ (adapter / "tokenizer.json").write_text("{}")
+ return adapter
+
+
+def test_unsloth_backend_loads_adapter_through_unsloth(env, tmp_path):
+ adapter = _adapter(tmp_path)
+ result = _run(str(adapter), "--tasks", "gsm8k", "-o", str(tmp_path / "out"))
+ assert result.exit_code == 0, result.output
+ assert env["model_name"] == str(adapter)
+ assert "peft" not in env and "resize_model_vocab" not in env["load_kwargs"]
+
+
+def test_unsloth_backend_grows_vocab_for_added_tokens(env, tmp_path, monkeypatch):
+ adapter = _adapter(tmp_path, tokenizer = True)
+ tok = types.ModuleType("transformers")
+ tok.AutoTokenizer = SimpleNamespace(from_pretrained = lambda path: [0] * 12)
+ monkeypatch.setitem(sys.modules, "transformers", tok)
+ monkeypatch.setattr(evalmod, "_base_vocab_size", lambda model: 10)
+ assert _run(str(adapter), "--tasks", "gsm8k", "-o", str(tmp_path / "out")).exit_code == 0
+ assert env["load_kwargs"]["resize_model_vocab"] == 12
+ monkeypatch.setattr(evalmod, "_base_vocab_size", lambda model: 16)
+ assert _run(str(adapter), "--tasks", "gsm8k", "-o", str(tmp_path / "out")).exit_code == 0
+ assert "resize_model_vocab" not in env["load_kwargs"]
+
+
+def test_base_model_override_attaches_adapter_manually(env, tmp_path, monkeypatch):
+ adapter = _adapter(tmp_path, tokenizer = True)
+ tok = types.ModuleType("transformers")
+ tok.AutoTokenizer = SimpleNamespace(from_pretrained = lambda path: [0] * 8)
+ monkeypatch.setitem(sys.modules, "transformers", tok)
+ args = (
+ str(adapter),
+ "--tasks",
+ "gsm8k",
+ "--base-model",
+ "org/other",
+ "-o",
+ str(tmp_path / "o"),
+ )
+ result = _run(*args)
+ assert result.exit_code == 0, result.output
+ assert env["model_name"] == "org/other" and env["peft"] == str(adapter)
+ assert "resized" not in env
+ tok.AutoTokenizer = SimpleNamespace(from_pretrained = lambda path: [0] * 11)
+ assert _run(*args).exit_code == 0
+ assert env["resized"] == 11
+
+
+def test_unsloth_backend_reports_missing_bitsandbytes(env, tmp_path, monkeypatch):
+ monkeypatch.setattr(
+ evalmod, "find_spec", lambda name: None if name == "bitsandbytes" else object()
+ )
+ result = _run("fake/model", "--tasks", "gsm8k", "-o", str(tmp_path))
+ assert result.exit_code == 0, result.output
+ assert "bitsandbytes is not installed" in result.output
+ assert env["load_kwargs"]["load_in_4bit"] is False
+
+
+def test_custom_dataset_builds_one_task_manager(env, tmp_path):
+ data = tmp_path / "qa.jsonl"
+ data.write_text('{"question": "q", "answer": "a"}\n')
+ result = _run("fake/model", "--tasks", f"gsm8k,{data}", "-o", str(tmp_path / "out"))
+ assert result.exit_code == 0, result.output
+ assert env["tasks"] == ["gsm8k", "qa"]
+ assert env["task_managers"] == 1 and len(env["include_path"]) == 1
+
+
+def test_glob_expands_and_unknown_task_errors(env, tmp_path):
+ assert _run("fake/model", "--tasks", "mmlu_*", "-o", str(tmp_path)).exit_code == 0
+ assert env["tasks"] == ["mmlu_a", "mmlu_b"]
+ for tasks in ("nope", "zzz_*"):
+ result = _run("fake/model", "--tasks", tasks)
+ assert result.exit_code == 2 and "Error:" in result.output
+
+
+def test_hf_backend_cpu(env, tmp_path):
+ result = _run("fake/model", "--tasks", "gsm8k", "--backend", "hf", "-o", str(tmp_path))
+ assert result.exit_code == 0, result.output
+ assert env["model"] == "hf"
+ assert env["model_args"] == {"pretrained": "fake/model", "max_length": 2048}
+ assert env["kwargs"]["device"] == "cpu" and env["kwargs"]["batch_size"] == 1
+ assert "model_name" not in env
+
+
+def test_hf_backend_cuda_4bit_builds_quantizing_hflm(env, tmp_path):
+ sys.modules["torch"].cuda = SimpleNamespace(is_available = lambda: True, device_count = lambda: 1)
+ result = _run(
+ "fake/model",
+ "--tasks",
+ "gsm8k",
+ "--backend",
+ "hf",
+ "--device",
+ "cuda:0",
+ "-o",
+ str(tmp_path),
+ )
+ assert result.exit_code == 0, result.output
+ assert type(env["model"]).__name__ == "_HFLM4bit"
+ assert env["hflm"]["load_in_4bit"] is True and env["hflm"]["device"] == "cuda:0"
+
+
+def test_hf_backend_adapter_uses_peft_arg(env, tmp_path):
+ adapter = tmp_path / "lora"
+ adapter.mkdir()
+ (adapter / "adapter_config.json").write_text(
+ json.dumps({"base_model_name_or_path": "org/base"})
+ )
+ (adapter / "tokenizer.json").write_text("{}")
+ result = _run(str(adapter), "--tasks", "gsm8k", "--backend", "hf", "-o", str(tmp_path / "o"))
+ assert result.exit_code == 0, result.output
+ assert env["model_args"]["pretrained"] == "org/base"
+ assert env["model_args"]["peft"] == env["model_args"]["tokenizer"] == str(adapter)
+
+
+def test_mlx_falls_back_to_hf(env, tmp_path):
+ sys.modules["unsloth"].DEVICE_TYPE = "mlx"
+ result = _run("fake/model", "--tasks", "gsm8k", "-o", str(tmp_path))
+ assert result.exit_code == 0, result.output
+ assert "falling back" in result.output and env["model"] == "hf"
+
+
+def test_unsafe_code_flag_is_forwarded(env, tmp_path):
+ assert (
+ _run(
+ "fake/model", "--tasks", "gsm8k", "--confirm-run-unsafe-code", "-o", str(tmp_path)
+ ).exit_code
+ == 0
+ )
+ assert env["confirm_run_unsafe_code"] is True
+
+
+ "args",
+ [
+ ["--batch-size", "0"],
+ ["--batch-size", "-1"],
+ ["--batch-size", "big"],
+ ["--backend", "vllm"],
+ ["--num-fewshot", "-1"],
+ ["--limit", "0"],
+ ["--limit", "2.5"],
+ ["--max-seq-length", "0"],
+ ["--backend", "hf", "--device", "cuda:9"],
+ ],
+)
+def test_invalid_arguments_exit_2(env, args):
+ result = _run("fake/model", "--tasks", "gsm8k", *args)
+ assert result.exit_code == 2, result.output
+ assert "model" not in env
+
+
+def test_fractional_and_whole_limits(env, tmp_path):
+ assert (
+ _run("fake/model", "--tasks", "gsm8k", "--limit", "0.25", "-o", str(tmp_path)).exit_code
+ == 0
+ )
+ assert env["kwargs"]["limit"] == 0.25
+ assert (
+ _run("fake/model", "--tasks", "gsm8k", "--limit", "5", "-o", str(tmp_path)).exit_code == 0
+ )
+ assert env["kwargs"]["limit"] == 5 and isinstance(env["kwargs"]["limit"], int)
+
+
+def test_unsloth_backend_refuses_multi_process(env, monkeypatch):
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ result = _run("fake/model", "--tasks", "gsm8k")
+ assert result.exit_code == 2 and "--backend hf" in result.output
+
+
+def test_silence_keeps_the_reconfigured_stdout_encoding(tmp_path):
+ import subprocess
+
+ script = tmp_path / "enc.py"
+ script.write_text(
+ "import sys\n"
+ "sys.stdout.reconfigure(encoding='utf-8')\n"
+ "from unsloth_cli.commands.eval import _silence\n"
+ "with _silence() as c:\n"
+ " c.print('Evaluating gsm8k\\u2026')\n"
+ )
+ env = dict(os.environ, PYTHONPATH = str(_REPO_ROOT), PYTHONCOERCECLOCALE = "0", LC_ALL = "C")
+ proc = subprocess.run(
+ [sys.executable, "-X", "utf8=0", str(script)], capture_output = True, env = env
+ )
+ assert proc.returncode == 0, proc.stderr.decode(errors = "replace")
+ assert "Evaluating gsm8k\u2026" in proc.stdout.decode("utf-8")