Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Add `unsloth eval` CLI command (#6824)
Nilay3617ea62638b
diff --git a/pyproject.toml b/pyproject.tomlindex 21d73fbdae4..20d2eaa9ce9 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -339,6 +339,10 @@ windows = [base = ["unsloth[huggingface]",]+eval = [+ "unsloth[huggingface]",+ "lm_eval>=0.4.4",+]cu118only = ["xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)","xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)",diff --git a/unsloth_cli/__init__.py b/unsloth_cli/__init__.pyindex e0498e44a8a..515c98e6a02 100644--- a/unsloth_cli/__init__.py+++ b/unsloth_cli/__init__.py@@ -83,6 +83,7 @@ else:from unsloth_cli.commands.chat import chatfrom unsloth_cli.commands.start import start_appfrom unsloth_cli.commands.export import export, list_checkpoints+ from unsloth_cli.commands.eval import evaluate as eval_commandfrom unsloth_cli.commands.studio import (run as studio_run,studio_app,@@ -216,6 +217,7 @@ if not _windows_studio_mutation_entry:app.command()(inference)app.command()(chat)app.command()(export)+ app.command("eval")(eval_command)app.command("list-checkpoints")(list_checkpoints)app.add_typer(start_app,diff --git a/unsloth_cli/commands/eval.py b/unsloth_cli/commands/eval.pynew file mode 100644index 00000000000..72d3d2000bf--- /dev/null+++ b/unsloth_cli/commands/eval.py@@ -0,0 +1,578 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import contextlib+import json+import os+import re+import shutil+import sys+import tempfile+from importlib.util import find_spec+from pathlib import Path+from typing import List, Optional, Tuple++import typer+import yaml++_HUB_REPO_RE = re.compile(r"[\w.\-]+/[\w.\-]+")+_TOKENIZER_FILES = ("tokenizer_config.json", "tokenizer.json", "tokenizer.model")+_DATASET_SUFFIXES = {".jsonl", ".json", ".csv"}+_YAML_SUFFIXES = {".yaml", ".yml"}+++def _silence():+ """Send fd 1/2 to devnull (lm-eval/transformers progress bars); yield a Console on the real stdout."""+ from rich.console import Console++ sys.stdout.flush()+ sys.stderr.flush()+ # keep the entry point's UTF-8 reconfigure (Windows legacy code pages); fdopen would reset it+ real = os.fdopen(+ os.dup(1),+ "w",+ closefd = True,+ encoding = getattr(sys.stdout, "encoding", None) or "utf-8",+ errors = "replace",+ )+ saved_out, saved_err = os.dup(1), os.dup(2)+ devnull_fd = os.open(os.devnull, os.O_WRONLY)+ try:+ os.dup2(devnull_fd, 1)+ os.dup2(devnull_fd, 2)+ yield Console(file = real)+ finally:+ sys.stdout.flush()+ sys.stderr.flush()+ os.dup2(saved_out, 1)+ os.dup2(saved_err, 2)+ for fd in (saved_out, saved_err, devnull_fd):+ os.close(fd)+ real.close()+++def _is_hub_id(model: str) -> bool:+ return not Path(model).exists() and bool(_HUB_REPO_RE.fullmatch(model))+++def resolve_base_model(model: str) -> Optional[str]:+ """base_model_name_or_path of a LoRA adapter (local dir or Hub repo), else None."""+ path = Path(model)+ config = path / "adapter_config.json"+ if not path.is_dir():+ if not _is_hub_id(model):+ return None+ try:+ from huggingface_hub import hf_hub_download+ config = Path(hf_hub_download(model, "adapter_config.json"))+ except Exception:+ return None+ try:+ data = json.loads(config.read_text(encoding = "utf-8"))+ except (ValueError, OSError):+ return None+ return data.get("base_model_name_or_path") if isinstance(data, dict) else None+++def _has_tokenizer_files(model: str) -> bool:+ path = Path(model)+ if path.is_dir():+ return any((path / name).exists() for name in _TOKENIZER_FILES)+ if not _is_hub_id(model):+ return False+ try:+ from huggingface_hub import list_repo_files+ files = set(list_repo_files(model))+ except Exception:+ return False+ return any(name in files for name in _TOKENIZER_FILES)+++def _hf_device_error(device: str) -> Optional[str]:+ # HFLM silently falls back to its default device on any string outside its device_list+ match = re.fullmatch(r"(cpu|cuda|mps|npu|xpu|hpu)(?::(0|[1-9]\d*))?", device)+ kind, index = (match.group(1), match.group(2)) if match else (None, None)+ if (+ kind is None+ or (kind == "cpu" and index is not None)+ or (kind == "mps" and index not in (None, "0"))+ or (kind in ("npu", "xpu", "hpu") and index is None)+ ):+ return (+ f"invalid --device '{device}': use 'cpu', 'cuda[:<index>]', 'mps', "+ "or '<npu|xpu|hpu>:<index>'."+ )+ if kind == "cpu":+ return None+ import torch++ backend = getattr(torch.backends, "mps", None) if kind == "mps" else getattr(torch, kind, None)+ try:+ available = bool(backend is not None and backend.is_available())+ except Exception:+ available = False+ if not available:+ return f"--device {device} requested but {kind.upper()} is not available."+ if index is not None and kind != "mps":+ try:+ count = int(backend.device_count())+ except Exception:+ count = 0+ if int(index) >= count:+ return f"--device {device} requested but only {count} {kind.upper()} device(s) are available."+ return None+++def _hflm_4bit(HFLM):+ # transformers 5 removed from_pretrained(load_in_4bit=...), which HFLM forwards verbatim+ class _HFLM4bit(HFLM):+ def _create_model(+ self,+ *args,+ quantization_config = None,+ load_in_4bit = False,+ **kwargs,+ ):+ if load_in_4bit and quantization_config is None:+ from transformers import BitsAndBytesConfig+ quantization_config = BitsAndBytesConfig(load_in_4bit = True)+ return super()._create_model(*args, quantization_config = quantization_config, **kwargs)++ return _HFLM4bit+++def _base_vocab_size(model: str) -> int:+ from transformers import AutoConfig+ config = AutoConfig.from_pretrained(model)+ return getattr(getattr(config, "text_config", None) or config, "vocab_size", 0) or 0+++class _TaskYamlLoader(yaml.SafeLoader):+ """safe_load that tolerates lm-eval's custom tags (!function utils.fn)."""+++_TaskYamlLoader.add_multi_constructor(+ "!", lambda loader, suffix, node: getattr(node, "value", None)+)+++def _doc_column(key: str) -> str:+ # Jinja stringifies the value but cannot parse non-identifier keys; lm-eval reads a bare column name raw+ import keyword+ if key.isidentifier() and not keyword.iskeyword(key) and key not in ("true", "false", "none"):+ return "{{" + key + "}}"+ return key+++def make_dataset_task(+ data_file: Path, input_key: str, target_key: str, out_dir: Path, name: str+) -> None:+ """Write an exact-match generate_until task over a .jsonl/.json/.csv file."""+ builder = "json" if data_file.suffix.lower() in {".json", ".jsonl"} else "csv"+ spec = {+ "task": name,+ "dataset_path": builder,+ "dataset_kwargs": {"data_files": str(data_file.resolve())},+ "test_split": "train",+ "fewshot_split": "train",+ "output_type": "generate_until",+ "doc_to_text": _doc_column(input_key),+ "doc_to_target": _doc_column(target_key),+ "generation_kwargs": {"until": ["\n"]},+ # strip so " 2" matches gold "2"; with one capture group re.findall yields the group text+ "filter_list": [+ {+ "name": "strip",+ "filter": [+ {"function": "regex", "regex_pattern": r"^\s*(.*?)\s*$", "group_select": 0},+ {"function": "take_first"},+ ],+ },+ ],+ "metric_list": [{"metric": "exact_match", "aggregation": "mean", "higher_is_better": True}],+ }+ out_dir.mkdir(parents = True, exist_ok = True)+ (out_dir / f"{name}.yaml").write_text(yaml.safe_dump(spec, sort_keys = False), encoding = "utf-8")+++def _yaml_task_name(path: Path) -> Tuple[str, bool]:+ """(registered name, needs its directory on the include path)."""+ text = path.read_text(encoding = "utf-8")+ try:+ spec = yaml.load(text, Loader = _TaskYamlLoader)+ except yaml.YAMLError as e:+ raise ValueError(f"Invalid YAML in custom task file '{path}': {e}") from e+ if not isinstance(spec, dict):+ raise ValueError(f"Custom task file '{path}' must define a YAML mapping.")+ is_group = isinstance(spec.get("task"), list)+ name = spec.get("group") if is_group else spec.get("task")+ if not name or not isinstance(name, str):+ raise ValueError(+ f"Custom task file '{path}' needs a top-level 'task:' name ('group:' for a task list)."+ )+ references_siblings = is_group or "include" in spec or "!function" in text+ if references_siblings and path.suffix.lower() == ".yml":+ raise ValueError(+ f"Custom task file '{path}' references sibling files but lm-eval only indexes "+ ".yaml files. Rename it (and the files it references) to .yaml."+ )+ return name, references_siblings+++def resolve_tasks(+ tasks: str, input_key: str, target_key: str, tmp_dir: Path+) -> Tuple[List[str], List[str]]:+ """Split --tasks into lm-eval task names plus the include paths custom files need."""+ entries = [e.strip() for e in tasks.split(",") if e.strip()]+ if not entries:+ raise ValueError("No tasks provided. Pass --tasks with at least one task.")+ plain = {e for e in entries if Path(e).suffix.lower() not in _DATASET_SUFFIXES | _YAML_SUFFIXES}+ names: List[str] = []+ include_paths: List[str] = []++ def add(name: str, include: Path) -> None:+ if name in names:+ raise ValueError(f"Duplicate task '{name}' in --tasks.")+ names.append(name)+ if str(include) not in include_paths:+ include_paths.append(str(include))++ for entry in entries:+ path = Path(entry)+ suffix = path.suffix.lower()+ if suffix in _YAML_SUFFIXES:+ if not path.exists():+ raise FileNotFoundError(f"Custom task file not found: {entry}")+ name, needs_dir = _yaml_task_name(path)+ if needs_dir:+ add(name, path.resolve().parent)+ else:+ # copied alone so an unrelated broken yaml beside it cannot break lm-eval's index+ custom_dir = (tmp_dir / "custom").resolve()+ custom_dir.mkdir(parents = True, exist_ok = True)+ shutil.copy2(path, custom_dir / f"{name}.yaml")+ add(name, custom_dir)+ elif suffix in _DATASET_SUFFIXES:+ if not path.exists():+ raise FileNotFoundError(f"Dataset file not found: {entry}")+ # an include-path task overrides a registered one of the same name, so keep clear of+ # names requested alongside (gsm8k,./gsm8k.jsonl)+ name, counter = path.stem, 2+ while name in names or name in plain:+ name, counter = f"{path.stem}_{counter}", counter + 1+ if name != path.stem:+ typer.echo(f"Note: running dataset '{path.name}' as task '{name}'.")+ gen_dir = (tmp_dir / "generated").resolve()+ make_dataset_task(path, input_key, target_key, gen_dir, name)+ add(name, gen_dir)+ else:+ if entry in names:+ raise ValueError(f"Duplicate task '{entry}' in --tasks.")+ names.append(entry)+ return names, include_paths+++def _expand_tasks(task_manager, names: List[str]) -> List[str]:+ """Expand globs (mmlu_*) like lm-eval's CLI and reject unknown names."""+ known = set(getattr(task_manager, "all_tasks", None) or [])+ if not known:+ return names+ expanded: List[str] = []+ for name in names:+ if any(ch in name for ch in "*?["):+ matches = task_manager.match_tasks([name])+ if not matches:+ raise ValueError(f"no tasks match pattern '{name}'.")+ expanded.extend(m for m in matches if m not in expanded)+ elif name not in known:+ raise ValueError(+ f"unknown task '{name}'. Pass a built-in task name, a .yaml task file, "+ "or a .jsonl/.csv dataset."+ )+ elif name not in expanded:+ expanded.append(name)+ return expanded+++def _metric_number(value):+ # numpy float32/int64 are not int/float subclasses: unwrap scalars via item()+ if not isinstance(value, (int, float)) and callable(getattr(value, "item", None)):+ try:+ value = value.item()+ except Exception:+ return None+ return value if isinstance(value, (int, float)) else None+++def _json_default(value):+ try:+ return value.tolist()+ except Exception:+ return str(value)+++def _render_results(results: dict) -> None:+ from rich.console import Console+ from rich.table import Table++ table = Table(title = "Evaluation results")+ table.add_column("Task", style = "cyan")+ table.add_column("Metric")+ table.add_column("Value", justify = "right")+ table.add_column("± stderr", justify = "right")+ rows = dict(results.get("results") or {})+ for task, metrics in (results.get("groups") or {}).items():+ rows.setdefault(task, metrics)+ for task, metrics in rows.items():+ for key, raw_value in metrics.items():+ value = _metric_number(raw_value)+ if key == "alias" or "_stderr" in key or value is None:+ continue+ metric, _, flt = key.partition(",")+ stderr = _metric_number(+ metrics.get(f"{metric}_stderr,{flt}" if flt else f"{metric}_stderr")+ )+ table.add_row(task, key, f"{value:.4f}", f"{stderr:.4f}" if stderr is not None else "—")+ Console().print(table)+++def _fail(message: str, code: int = 2):+ typer.echo(f"Error: {message}", err = True)+ raise typer.Exit(code = code)+++def evaluate(+ model: str = typer.Argument(+ ..., help = "Path to a checkpoint/adapter directory or a HuggingFace model id."+ ),+ tasks: str = typer.Option(+ ...,+ "--tasks",+ "-t",+ help = "Comma-separated built-in task names (e.g. mmlu,gsm8k), or a path to a "+ "custom .yaml task or a .jsonl/.csv dataset.",+ ),+ base_model: Optional[str] = typer.Option(+ None,+ "--base-model",+ help = "Base model for a LoRA adapter. Auto-detected from adapter_config.json; "+ "set this to override a moved/renamed base.",+ ),+ num_fewshot: Optional[int] = typer.Option(+ None, "--num-fewshot", "-n", help = "Few-shot examples (default: per-task)."+ ),+ limit: Optional[float] = typer.Option(+ None,+ "--limit",+ help = "Cap examples per task (for quick smoke tests): a whole count, or a "+ "fraction between 0 and 1 for a proportion of each task.",+ ),+ batch_size: str = typer.Option("auto", "--batch-size", "-b", help = "Batch size, or 'auto'."),+ max_seq_length: int = typer.Option(+ 2048, "--max-seq-length", help = "Max sequence length for the model."+ ),+ load_in_4bit: bool = typer.Option(+ True, "--load-in-4bit/--no-load-in-4bit", help = "Load the model in 4-bit."+ ),+ backend: str = typer.Option(+ "unsloth",+ "--backend",+ help = "Model backend: 'unsloth' (fast kernels; needs an NVIDIA/AMD/Intel "+ "GPU) or 'hf' (plain transformers; works on CPU/MPS/Mac). "+ "Auto-falls back to 'hf' on Apple Silicon.",+ ),+ device: Optional[str] = typer.Option(+ None,+ "--device",+ help = "Device for the hf backend (e.g. cpu, mps, cuda). Default: auto.",+ ),+ input_key: str = typer.Option(+ "question", "--input-key", help = "Prompt field for a .jsonl/.csv dataset task."+ ),+ target_key: str = typer.Option(+ "answer", "--target-key", help = "Answer field for a .jsonl/.csv dataset task."+ ),+ output_dir: Path = typer.Option(+ Path("./eval_results"), "--output-dir", "-o", help = "Directory for results.json."+ ),+ confirm_run_unsafe_code: bool = typer.Option(+ False,+ "--confirm-run-unsafe-code",+ help = "Allow tasks that lm-eval marks unsafe (e.g. humaneval): they execute "+ "model-generated code on this machine. Off by default.",+ ),+ hf_token: Optional[str] = typer.Option(+ None, "--hf-token", envvar = "HF_TOKEN", help = "HuggingFace token if needed."+ ),+):+ """Evaluate a checkpoint or LoRA adapter using lm-eval-harness."""+ bs = batch_size+ if batch_size != "auto":+ bs = int(batch_size) if batch_size.isdigit() else 0+ if bs <= 0:+ _fail("--batch-size must be a positive integer or 'auto'.")+ if backend not in ("unsloth", "hf"):+ _fail(f"--backend must be 'unsloth' or 'hf', got '{backend}'.")+ if num_fewshot is not None and num_fewshot < 0:+ _fail("--num-fewshot must be >= 0.")+ if limit is not None:+ # lm-eval reads < 1 as a fraction and int()s counts, so 2.5 would silently run 2+ if limit <= 0 or (limit >= 1 and not limit.is_integer()):+ _fail("--limit must be a whole count or a fraction between 0 and 1.")+ limit = int(limit) if limit >= 1 else limit+ if max_seq_length <= 0:+ _fail("--max-seq-length must be a positive integer.")+ # find_spec, not import: lm_eval imports transformers, which must come after unsloth+ if "lm_eval" not in sys.modules and find_spec("lm_eval") is None:+ _fail("evaluation requires lm-eval. Install it with `pip install unsloth[eval]`.", code = 1)++ if backend == "unsloth":+ with _silence():+ import unsloth+ if getattr(unsloth, "DEVICE_TYPE", None) == "mlx":+ typer.echo("Note: Apple Silicon (MLX) detected, falling back to --backend hf.")+ backend = "hf"+ # a pre-loaded model makes lm-eval single-process, so every rank would run every task+ if backend == "unsloth" and os.environ.get("WORLD_SIZE", "1") not in ("", "1"):+ _fail(+ "multi-process launches (accelerate/torchrun) are not supported with "+ "--backend unsloth. Use --backend hf for multi-GPU evaluation."+ )++ import inspect++ import lm_eval+ from lm_eval.models.huggingface import HFLM+ from lm_eval.tasks import TaskManager++ if hf_token:+ os.environ["HF_TOKEN"] = hf_token+ effective_base = base_model or resolve_base_model(model)++ tmp_dir = Path(tempfile.mkdtemp(prefix = "unsloth_eval_"))+ try:+ try:+ task_names, include_paths = resolve_tasks(tasks, input_key, target_key, tmp_dir)+ task_manager = TaskManager(include_path = include_paths or None)+ task_names = _expand_tasks(task_manager, task_names)+ except (FileNotFoundError, ValueError) as e:+ _fail(str(e))++ typer.echo(f"Running tasks: {', '.join(task_names)} (backend: {backend})")+ eval_kwargs = dict(+ tasks = task_names,+ num_fewshot = num_fewshot,+ limit = limit,+ task_manager = task_manager,+ log_samples = False,+ )+ # lm-eval < 0.4.8 has no unsafe-code gate+ if "confirm_run_unsafe_code" in inspect.signature(lm_eval.simple_evaluate).parameters:+ eval_kwargs["confirm_run_unsafe_code"] = confirm_run_unsafe_code+ elif confirm_run_unsafe_code:+ typer.echo(+ "Note: this lm-eval version has no unsafe-code gate; the flag has no effect."+ )++ if backend == "hf":+ import torch++ if device is None:+ mps = getattr(torch.backends, "mps", None)+ device = (+ "cuda"+ if torch.cuda.is_available()+ else "mps"+ if mps and mps.is_available()+ else "cpu"+ )+ elif (device_error := _hf_device_error(device)) is not None:+ _fail(device_error)+ if bs == "auto" and not device.startswith("cuda"):+ typer.echo(+ "Note: batch size 'auto' is slow on CPU/MPS, using 1 (override with --batch-size)."+ )+ bs = 1+ model_args = {"pretrained": model}+ if effective_base:+ model_args = {"pretrained": effective_base, "peft": model}+ if _has_tokenizer_files(model):+ model_args["tokenizer"] = model+ typer.echo(f"Evaluating adapter '{model}' on base '{effective_base}'.")+ model_args["max_length"] = max_seq_length+ if load_in_4bit and device.startswith("cuda"):+ if find_spec("bitsandbytes") is not None:+ model_args["load_in_4bit"] = True+ else:+ typer.echo("Note: bitsandbytes is not installed, loading in full precision.")+ if model_args.get("load_in_4bit"):+ with _silence():+ eval_kwargs["model"] = _hflm_4bit(HFLM)(+ **model_args, batch_size = bs, device = device+ )+ else:+ eval_kwargs.update(model = "hf", model_args = model_args, batch_size = bs, device = device)+ else:+ from unsloth import FastLanguageModel++ if load_in_4bit and find_spec("bitsandbytes") is None:+ typer.echo("Note: bitsandbytes is not installed, loading in full precision.")+ load_in_4bit = False+ load_kwargs = dict(+ max_seq_length = max_seq_length, load_in_4bit = load_in_4bit, token = hf_token or None+ )+ adapter_vocab = None+ if effective_base and _has_tokenizer_files(model):+ from transformers import AutoTokenizer+ adapter_vocab = len(AutoTokenizer.from_pretrained(model))+ typer.echo(+ f"Loading base model '{effective_base}' with adapter '{model}'..."+ if effective_base+ else f"Loading model: {model}"+ )+ with _silence():+ if effective_base and not base_model:+ # Unsloth's own adapter path applies its PEFT hooks (grouped-linear LoRA, Gemma 4)+ if adapter_vocab and adapter_vocab > _base_vocab_size(effective_base):+ load_kwargs["resize_model_vocab"] = adapter_vocab+ lmodel, tokenizer = FastLanguageModel.from_pretrained(+ model_name = model, **load_kwargs+ )+ else:+ lmodel, tokenizer = FastLanguageModel.from_pretrained(+ model_name = effective_base or model, **load_kwargs+ )+ if effective_base and base_model:+ # --base-model overrides adapter_config.json, which Unsloth's adapter path always reads+ if adapter_vocab:+ from transformers import AutoTokenizer+ tokenizer = AutoTokenizer.from_pretrained(model)+ # grow only: padded vocabularies (Qwen3) have more rows than tokens+ if adapter_vocab > lmodel.get_input_embeddings().weight.shape[0]:+ lmodel.resize_token_embeddings(adapter_vocab)+ from peft import PeftModel+ lmodel = PeftModel.from_pretrained(lmodel, model)+ FastLanguageModel.for_inference(lmodel)+ eval_kwargs["model"] = HFLM(+ pretrained = lmodel, tokenizer = tokenizer, batch_size = bs, max_length = max_seq_length+ )++ with _silence() as ui:+ from rich.status import Status+ with Status(f"Evaluating {', '.join(task_names)}…", console = ui, spinner = "dots"):+ results = lm_eval.simple_evaluate(**eval_kwargs)+ finally:+ shutil.rmtree(tmp_dir, ignore_errors = True)++ if results is None:+ # lm-eval returns None on non-zero ranks+ if os.environ.get("RANK", "0") != "0" or os.environ.get("LOCAL_RANK", "0") != "0":+ return+ _fail("evaluation returned no results.", code = 1)++ _render_results(results)+ output_dir.mkdir(parents = True, exist_ok = True)+ results_path = output_dir / "results.json"+ results_path.write_text(json.dumps(results, indent = 2, default = _json_default), encoding = "utf-8")+ typer.echo(f"Saved results to: {results_path}")diff --git a/unsloth_cli/tests/test_eval.py b/unsloth_cli/tests/test_eval.pynew file mode 100644index 00000000000..91fd75c751e--- /dev/null+++ b/unsloth_cli/tests/test_eval.py@@ -0,0 +1,534 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import json+import os+import sys+import types+from pathlib import Path+from types import SimpleNamespace++_REPO_ROOT = Path(__file__).resolve().parents[2]+if str(_REPO_ROOT) not in sys.path:+ sys.path.insert(0, str(_REPO_ROOT))++import pytest+import typer+import yaml+from typer.testing import CliRunner++import unsloth_cli.commands.eval as evalmod+++def _eval_app():+ cli = typer.Typer()+ cli.command()(evalmod.evaluate)+ return cli+++def _run(*args):+ return CliRunner().invoke(_eval_app(), list(args))+++ "content, expected",+ [+ (json.dumps({"base_model_name_or_path": "unsloth/Llama-3.2-1B"}), "unsloth/Llama-3.2-1B"),+ ("not json", None),+ ("[1, 2]", None),+ ],+)+def test_resolve_base_model_from_local_adapter(tmp_path, content, expected):+ (tmp_path / "adapter_config.json").write_text(content)+ assert evalmod.resolve_base_model(str(tmp_path)) == expected+++def test_resolve_base_model_plain_dir_and_non_hub_name(tmp_path):+ assert evalmod.resolve_base_model(str(tmp_path)) is None+ assert evalmod.resolve_base_model("not a repo id") is None+++ "key, expected",+ [+ ("question", "{{question}}"),+ ("expected answer", "expected answer"),+ ("if", "if"),+ ("none", "none"),+ ],+)+def test_doc_column(key, expected):+ assert evalmod._doc_column(key) == expected+++def test_resolve_tasks_plain_names():+ names, includes = evalmod.resolve_tasks("gsm8k, mmlu ,", "q", "a", Path("/unused"))+ assert names == ["gsm8k", "mmlu"]+ assert includes == []++[email protected]("tasks", ["", " , ", "gsm8k,gsm8k"])+def test_resolve_tasks_rejects_empty_and_duplicates(tasks, tmp_path):+ with pytest.raises(ValueError):+ evalmod.resolve_tasks(tasks, "q", "a", tmp_path)+++def test_resolve_tasks_missing_files(tmp_path):+ for missing in ("nope.jsonl", "nope.yaml"):+ with pytest.raises(FileNotFoundError):+ evalmod.resolve_tasks(str(tmp_path / missing), "q", "a", tmp_path)+++def test_dataset_task_yaml(tmp_path):+ data = tmp_path / "qa.jsonl"+ data.write_text('{"question": "1+1?", "answer": "2"}\n')+ names, includes = evalmod.resolve_tasks(str(data), "question", "answer", tmp_path / "tmp")+ assert names == ["qa"]+ spec = yaml.safe_load(Path(includes[0], "qa.yaml").read_text())+ assert spec["dataset_path"] == "json"+ assert spec["dataset_kwargs"] == {"data_files": str(data.resolve())}+ assert spec["doc_to_text"] == "{{question}}"+ assert spec["fewshot_split"] == "train"+ assert spec["metric_list"][0]["metric"] == "exact_match"+++def test_dataset_task_stays_clear_of_names_requested_alongside(tmp_path):+ (tmp_path / "gsm8k.jsonl").write_text("{}\n")+ (tmp_path / "sub").mkdir()+ (tmp_path / "sub" / "gsm8k.csv").write_text("question,answer\n")+ names, _ = evalmod.resolve_tasks(+ f"gsm8k,{tmp_path / 'gsm8k.jsonl'},{tmp_path / 'sub' / 'gsm8k.csv'}",+ "q",+ "a",+ tmp_path / "tmp",+ )+ assert names == ["gsm8k", "gsm8k_2", "gsm8k_3"]+++def test_standalone_yaml_is_copied_alone(tmp_path):+ (tmp_path / "broken.yaml").write_text("task: [unclosed\n")+ task = tmp_path / "mine.yml"+ task.write_text("task: my_task\ndataset_path: json\n")+ names, includes = evalmod.resolve_tasks(str(task), "q", "a", tmp_path / "tmp")+ assert names == ["my_task"]+ assert sorted(p.name for p in Path(includes[0]).iterdir()) == ["my_task.yaml"]+++def test_group_yaml_puts_its_directory_on_the_include_path(tmp_path):+ group = tmp_path / "suite.yaml"+ group.write_text("group: suite\ntask: [a, b]\n")+ names, includes = evalmod.resolve_tasks(str(group), "q", "a", tmp_path / "tmp")+ assert names == ["suite"]+ assert includes == [str(tmp_path.resolve())]+++def test_function_tag_yaml_uses_its_directory(tmp_path):+ task = tmp_path / "fn.yaml"+ task.write_text("task: fn_task\ndoc_to_text: !function utils.doc_to_text\n")+ names, includes = evalmod.resolve_tasks(str(task), "q", "a", tmp_path / "tmp")+ assert names == ["fn_task"]+ assert includes == [str(tmp_path.resolve())]+++ "filename, content",+ [+ ("bad.yaml", "task: [unclosed\n"),+ ("list.yaml", "- a\n- b\n"),+ ("noname.yaml", "dataset_path: json\n"),+ ("group_no_name.yaml", "task: [a, b]\n"),+ ("sibling.yml", "group: g\ntask: [a]\n"),+ ],+)+def test_invalid_yaml_tasks(tmp_path, filename, content):+ path = tmp_path / filename+ path.write_text(content)+ with pytest.raises(ValueError):+ evalmod.resolve_tasks(str(path), "q", "a", tmp_path / "tmp")+++def _fake_torch(+ cuda_count = 0,+ mps = False,+ xpu_count = 0,+):+ xpu = SimpleNamespace(is_available = lambda: xpu_count > 0, device_count = lambda: xpu_count)+ return SimpleNamespace(+ cuda = SimpleNamespace(is_available = lambda: cuda_count > 0, device_count = lambda: cuda_count),+ backends = SimpleNamespace(mps = SimpleNamespace(is_available = lambda: mps)),+ xpu = xpu,+ )+++ "device, ok",+ [+ ("cpu", True),+ ("cuda", True),+ ("cuda:1", True),+ ("cuda:2", False),+ ("cuda:01", False),+ ("cpu:0", False),+ ("mps", False),+ ("xpu", False),+ ("xpu:0", True),+ ("npu:0", False),+ ("tpu", False),+ ],+)+def test_hf_device_error(monkeypatch, device, ok):+ monkeypatch.setitem(sys.modules, "torch", _fake_torch(cuda_count = 2, xpu_count = 1))+ assert (evalmod._hf_device_error(device) is None) is ok+++def test_hflm_4bit_passes_a_quantization_config_not_load_in_4bit():+ # transformers without torch cannot build BitsAndBytesConfig (macOS runners)+ pytest.importorskip("torch")+ pytest.importorskip("transformers")+ from transformers import BitsAndBytesConfig++ class _Base:+ def _create_model(+ self,+ pretrained,+ quantization_config = None,+ **kwargs,+ ):+ self.seen = dict(kwargs, quantization_config = quantization_config)++ lm = evalmod._hflm_4bit(_Base)()+ lm._create_model(pretrained = "x", load_in_4bit = True, quantization_config = None)+ assert "load_in_4bit" not in lm.seen+ assert isinstance(lm.seen["quantization_config"], BitsAndBytesConfig)+ assert lm.seen["quantization_config"].load_in_4bit is True++ prequantized = object()+ lm._create_model(pretrained = "x", load_in_4bit = True, quantization_config = prequantized)+ assert lm.seen["quantization_config"] is prequantized+++def test_metric_number_and_render(capsys):+ np = pytest.importorskip("numpy")+ assert evalmod._metric_number(np.float32(0.5)) == 0.5+ assert evalmod._metric_number("x") is None+ evalmod._render_results(+ {+ "results": {+ "gsm8k": {"exact_match,strict": np.float32(0.3), "exact_match_stderr,strict": 0.01}+ },+ "groups": {"mmlu": {"acc,none": 0.45, "alias": "mmlu"}},+ }+ )+ out = capsys.readouterr().out+ assert "0.3000" in out and "0.0100" in out and "0.4500" in out+++def test_eval_missing_lm_eval_shows_hint(monkeypatch):+ monkeypatch.delitem(sys.modules, "lm_eval", raising = False)+ monkeypatch.setattr(evalmod, "find_spec", lambda name: None)+ result = _run("fake/model", "--tasks", "gsm8k")+ assert result.exit_code == 1, result.output+ assert "pip install unsloth[eval]" in result.output+++def env(monkeypatch):+ calls = {"task_managers": 0}++ class _FakeFLM:+ @classmethod+ def from_pretrained(+ cls,+ model_name = None,+ **kw,+ ):+ calls["model_name"] = model_name+ calls["load_kwargs"] = kw+ model = SimpleNamespace(+ get_input_embeddings = lambda: SimpleNamespace(weight = SimpleNamespace(shape = (10, 4))),+ resize_token_embeddings = lambda n: calls.__setitem__("resized", n),+ )+ return model, SimpleNamespace(name = "tok")++ @classmethod+ def for_inference(cls, model):+ calls["for_inference"] = True++ class _FakeHFLM:+ def __init__(+ self,+ pretrained = None,+ tokenizer = None,+ batch_size = None,+ max_length = None,+ **kw,+ ):+ calls["hflm"] = dict(+ kw, pretrained = pretrained, batch_size = batch_size, max_length = max_length+ )++ class _FakeTaskManager:+ def __init__(self, include_path = None):+ calls["task_managers"] += 1+ calls["include_path"] = include_path+ self.all_tasks = ["gsm8k", "mmlu", "mmlu_a", "mmlu_b"]+ for directory in include_path or []:+ self.all_tasks += [p.stem for p in Path(directory).glob("*.yaml")]++ def match_tasks(self, patterns):+ import fnmatch+ return [t for p in patterns for t in self.all_tasks if fnmatch.fnmatch(t, p)]++ def _simple_evaluate(+ model = None,+ model_args = None,+ tasks = None,+ confirm_run_unsafe_code = False,+ **kw,+ ):+ calls.update(model = model, model_args = model_args, tasks = tasks, kwargs = kw)+ calls["confirm_run_unsafe_code"] = confirm_run_unsafe_code+ np = pytest.importorskip("numpy")+ return {"results": {"gsm8k": {"exact_match,strict": np.float64(0.42)}}, "configs": {}}++ unsloth_mod = types.ModuleType("unsloth")+ unsloth_mod.FastLanguageModel = _FakeFLM+ lm_eval_mod = types.ModuleType("lm_eval")+ lm_eval_mod.simple_evaluate = _simple_evaluate+ hf_mod = types.ModuleType("lm_eval.models.huggingface")+ hf_mod.HFLM = _FakeHFLM+ tasks_mod = types.ModuleType("lm_eval.tasks")+ tasks_mod.TaskManager = _FakeTaskManager+ peft_mod = types.ModuleType("peft")+ peft_mod.PeftModel = SimpleNamespace(+ from_pretrained = lambda m, path: calls.__setitem__("peft", path) or m+ )+ hub_mod = types.ModuleType("huggingface_hub")+ hub_mod.hf_hub_download = hub_mod.list_repo_files = lambda *a, **k: (_ for _ in ()).throw(+ OSError()+ )+ for name, mod in {+ "unsloth": unsloth_mod,+ "torch": _fake_torch(),+ "huggingface_hub": hub_mod,+ "peft": peft_mod,+ "lm_eval": lm_eval_mod,+ "lm_eval.models": types.ModuleType("lm_eval.models"),+ "lm_eval.models.huggingface": hf_mod,+ "lm_eval.tasks": tasks_mod,+ }.items():+ monkeypatch.setitem(sys.modules, name, mod)+ monkeypatch.setattr(evalmod, "find_spec", lambda name: object())+ monkeypatch.delenv("WORLD_SIZE", raising = False)+ return calls+++def test_unsloth_backend_writes_results(env, tmp_path):+ result = _run("fake/model", "--tasks", "gsm8k", "-o", str(tmp_path / "out"))+ assert result.exit_code == 0, result.output+ assert env["model_name"] == "fake/model"+ assert env["load_kwargs"]["load_in_4bit"] is True+ assert env["for_inference"]+ assert env["hflm"]["max_length"] == 2048 and env["hflm"]["batch_size"] == "auto"+ assert env["tasks"] == ["gsm8k"] and env["task_managers"] == 1+ assert env["kwargs"]["log_samples"] is False+ saved = json.loads((tmp_path / "out" / "results.json").read_text())+ assert saved["results"]["gsm8k"]["exact_match,strict"] == 0.42+++def _adapter(tmp_path, tokenizer = False):+ adapter = tmp_path / "lora"+ adapter.mkdir()+ (adapter / "adapter_config.json").write_text(+ json.dumps({"base_model_name_or_path": "org/base"})+ )+ if tokenizer:+ (adapter / "tokenizer.json").write_text("{}")+ return adapter+++def test_unsloth_backend_loads_adapter_through_unsloth(env, tmp_path):+ adapter = _adapter(tmp_path)+ result = _run(str(adapter), "--tasks", "gsm8k", "-o", str(tmp_path / "out"))+ assert result.exit_code == 0, result.output+ assert env["model_name"] == str(adapter)+ assert "peft" not in env and "resize_model_vocab" not in env["load_kwargs"]+++def test_unsloth_backend_grows_vocab_for_added_tokens(env, tmp_path, monkeypatch):+ adapter = _adapter(tmp_path, tokenizer = True)+ tok = types.ModuleType("transformers")+ tok.AutoTokenizer = SimpleNamespace(from_pretrained = lambda path: [0] * 12)+ monkeypatch.setitem(sys.modules, "transformers", tok)+ monkeypatch.setattr(evalmod, "_base_vocab_size", lambda model: 10)+ assert _run(str(adapter), "--tasks", "gsm8k", "-o", str(tmp_path / "out")).exit_code == 0+ assert env["load_kwargs"]["resize_model_vocab"] == 12+ monkeypatch.setattr(evalmod, "_base_vocab_size", lambda model: 16)+ assert _run(str(adapter), "--tasks", "gsm8k", "-o", str(tmp_path / "out")).exit_code == 0+ assert "resize_model_vocab" not in env["load_kwargs"]+++def test_base_model_override_attaches_adapter_manually(env, tmp_path, monkeypatch):+ adapter = _adapter(tmp_path, tokenizer = True)+ tok = types.ModuleType("transformers")+ tok.AutoTokenizer = SimpleNamespace(from_pretrained = lambda path: [0] * 8)+ monkeypatch.setitem(sys.modules, "transformers", tok)+ args = (+ str(adapter),+ "--tasks",+ "gsm8k",+ "--base-model",+ "org/other",+ "-o",+ str(tmp_path / "o"),+ )+ result = _run(*args)+ assert result.exit_code == 0, result.output+ assert env["model_name"] == "org/other" and env["peft"] == str(adapter)+ assert "resized" not in env+ tok.AutoTokenizer = SimpleNamespace(from_pretrained = lambda path: [0] * 11)+ assert _run(*args).exit_code == 0+ assert env["resized"] == 11+++def test_unsloth_backend_reports_missing_bitsandbytes(env, tmp_path, monkeypatch):+ monkeypatch.setattr(+ evalmod, "find_spec", lambda name: None if name == "bitsandbytes" else object()+ )+ result = _run("fake/model", "--tasks", "gsm8k", "-o", str(tmp_path))+ assert result.exit_code == 0, result.output+ assert "bitsandbytes is not installed" in result.output+ assert env["load_kwargs"]["load_in_4bit"] is False+++def test_custom_dataset_builds_one_task_manager(env, tmp_path):+ data = tmp_path / "qa.jsonl"+ data.write_text('{"question": "q", "answer": "a"}\n')+ result = _run("fake/model", "--tasks", f"gsm8k,{data}", "-o", str(tmp_path / "out"))+ assert result.exit_code == 0, result.output+ assert env["tasks"] == ["gsm8k", "qa"]+ assert env["task_managers"] == 1 and len(env["include_path"]) == 1+++def test_glob_expands_and_unknown_task_errors(env, tmp_path):+ assert _run("fake/model", "--tasks", "mmlu_*", "-o", str(tmp_path)).exit_code == 0+ assert env["tasks"] == ["mmlu_a", "mmlu_b"]+ for tasks in ("nope", "zzz_*"):+ result = _run("fake/model", "--tasks", tasks)+ assert result.exit_code == 2 and "Error:" in result.output+++def test_hf_backend_cpu(env, tmp_path):+ result = _run("fake/model", "--tasks", "gsm8k", "--backend", "hf", "-o", str(tmp_path))+ assert result.exit_code == 0, result.output+ assert env["model"] == "hf"+ assert env["model_args"] == {"pretrained": "fake/model", "max_length": 2048}+ assert env["kwargs"]["device"] == "cpu" and env["kwargs"]["batch_size"] == 1+ assert "model_name" not in env+++def test_hf_backend_cuda_4bit_builds_quantizing_hflm(env, tmp_path):+ sys.modules["torch"].cuda = SimpleNamespace(is_available = lambda: True, device_count = lambda: 1)+ result = _run(+ "fake/model",+ "--tasks",+ "gsm8k",+ "--backend",+ "hf",+ "--device",+ "cuda:0",+ "-o",+ str(tmp_path),+ )+ assert result.exit_code == 0, result.output+ assert type(env["model"]).__name__ == "_HFLM4bit"+ assert env["hflm"]["load_in_4bit"] is True and env["hflm"]["device"] == "cuda:0"+++def test_hf_backend_adapter_uses_peft_arg(env, tmp_path):+ adapter = tmp_path / "lora"+ adapter.mkdir()+ (adapter / "adapter_config.json").write_text(+ json.dumps({"base_model_name_or_path": "org/base"})+ )+ (adapter / "tokenizer.json").write_text("{}")+ result = _run(str(adapter), "--tasks", "gsm8k", "--backend", "hf", "-o", str(tmp_path / "o"))+ assert result.exit_code == 0, result.output+ assert env["model_args"]["pretrained"] == "org/base"+ assert env["model_args"]["peft"] == env["model_args"]["tokenizer"] == str(adapter)+++def test_mlx_falls_back_to_hf(env, tmp_path):+ sys.modules["unsloth"].DEVICE_TYPE = "mlx"+ result = _run("fake/model", "--tasks", "gsm8k", "-o", str(tmp_path))+ assert result.exit_code == 0, result.output+ assert "falling back" in result.output and env["model"] == "hf"+++def test_unsafe_code_flag_is_forwarded(env, tmp_path):+ assert (+ _run(+ "fake/model", "--tasks", "gsm8k", "--confirm-run-unsafe-code", "-o", str(tmp_path)+ ).exit_code+ == 0+ )+ assert env["confirm_run_unsafe_code"] is True+++ "args",+ [+ ["--batch-size", "0"],+ ["--batch-size", "-1"],+ ["--batch-size", "big"],+ ["--backend", "vllm"],+ ["--num-fewshot", "-1"],+ ["--limit", "0"],+ ["--limit", "2.5"],+ ["--max-seq-length", "0"],+ ["--backend", "hf", "--device", "cuda:9"],+ ],+)+def test_invalid_arguments_exit_2(env, args):+ result = _run("fake/model", "--tasks", "gsm8k", *args)+ assert result.exit_code == 2, result.output+ assert "model" not in env+++def test_fractional_and_whole_limits(env, tmp_path):+ assert (+ _run("fake/model", "--tasks", "gsm8k", "--limit", "0.25", "-o", str(tmp_path)).exit_code+ == 0+ )+ assert env["kwargs"]["limit"] == 0.25+ assert (+ _run("fake/model", "--tasks", "gsm8k", "--limit", "5", "-o", str(tmp_path)).exit_code == 0+ )+ assert env["kwargs"]["limit"] == 5 and isinstance(env["kwargs"]["limit"], int)+++def test_unsloth_backend_refuses_multi_process(env, monkeypatch):+ monkeypatch.setenv("WORLD_SIZE", "2")+ result = _run("fake/model", "--tasks", "gsm8k")+ assert result.exit_code == 2 and "--backend hf" in result.output+++def test_silence_keeps_the_reconfigured_stdout_encoding(tmp_path):+ import subprocess++ script = tmp_path / "enc.py"+ script.write_text(+ "import sys\n"+ "sys.stdout.reconfigure(encoding='utf-8')\n"+ "from unsloth_cli.commands.eval import _silence\n"+ "with _silence() as c:\n"+ " c.print('Evaluating gsm8k\\u2026')\n"+ )+ env = dict(os.environ, PYTHONPATH = str(_REPO_ROOT), PYTHONCOERCECLOCALE = "0", LC_ALL = "C")+ proc = subprocess.run(+ [sys.executable, "-X", "utf8=0", str(script)], capture_output = True, env = env+ )+ assert proc.returncode == 0, proc.stderr.decode(errors = "replace")+ assert "Evaluating gsm8k\u2026" in proc.stdout.decode("utf-8")