Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Score a 16-bit fine-tune with unsloth eval in 16-bit, not in 4-bit (#13758)
Nilay2a8def3bdee7
diff --git a/unsloth_cli/commands/eval.py b/unsloth_cli/commands/eval.pyindex 72d3d2000bf..b97b8b91ee1 100644--- a/unsloth_cli/commands/eval.py+++ b/unsloth_cli/commands/eval.py@@ -75,6 +75,49 @@ def resolve_base_model(model: str) -> Optional[str]:return data.get("base_model_name_or_path") if isinstance(data, dict) else None+def _is_studio_output(path: Path) -> bool:+ # same scope as Studio Chat: downloads and merged exports keep the 4-bit default+ try:+ from unsloth_cli._inference import ensure_studio_backend_path++ ensure_studio_backend_path(seed_cache_env = False)+ from utils.paths import outputs_root, studio_root++ accounts = studio_root() / "accounts"+ roots = [outputs_root()]+ if accounts.is_dir():+ roots += [account / "outputs" for account in accounts.iterdir()]+ resolved = path.resolve()+ return any(resolved.is_relative_to(root.resolve()) for root in roots)+ except Exception:+ return False+++def _read_config(file: Path) -> Optional[dict]:+ try:+ config = json.loads(file.read_text(encoding = "utf-8-sig"))+ except (ValueError, OSError):+ return None+ return config if isinstance(config, dict) else None+++def _trained_in_4bit(model: str) -> bool:+ path = Path(model).absolute()+ is_adapter = (path / "adapter_config.json").is_file()+ config = _read_config(path / ("adapter_config.json" if is_adapter else "config.json"))+ if config is None:+ return True+ if not is_adapter:+ return "quantization_config" in config or not _is_studio_output(path)+ if "unsloth_training_method" not in config and path.name.startswith("checkpoint-"):+ # Studio records the precision on the finished run, not on its checkpoint-N folders+ config = _read_config(path.parent / "adapter_config.json") or config+ recorded = config.get("unsloth_load_in_4bit")+ if isinstance(recorded, bool):+ return recorded+ return config.get("unsloth_training_method") != "lora"++def _has_tokenizer_files(model: str) -> bool:path = Path(model)if path.is_dir():@@ -372,8 +415,11 @@ def evaluate(max_seq_length: int = typer.Option(2048, "--max-seq-length", help = "Max sequence length for the model."),- load_in_4bit: bool = typer.Option(- True, "--load-in-4bit/--no-load-in-4bit", help = "Load the model in 4-bit."+ load_in_4bit: Optional[bool] = typer.Option(+ None,+ "--load-in-4bit/--no-load-in-4bit",+ help = "Load the model in 4-bit. Default: the precision a local checkpoint or "+ "adapter was trained in, 4-bit otherwise.",),backend: str = typer.Option("unsloth",@@ -449,6 +495,13 @@ def evaluate(if hf_token:os.environ["HF_TOKEN"] = hf_tokeneffective_base = base_model or resolve_base_model(model)+ if load_in_4bit is None:+ load_in_4bit = _trained_in_4bit(model)+ if not load_in_4bit:+ typer.echo(+ "Note: loading in 16-bit, the precision this checkpoint was trained in "+ "(pass --load-in-4bit to override)."+ )tmp_dir = Path(tempfile.mkdtemp(prefix = "unsloth_eval_"))try:diff --git a/unsloth_cli/tests/test_eval.py b/unsloth_cli/tests/test_eval.pyindex 91fd75c751e..db67268872c 100644--- a/unsloth_cli/tests/test_eval.py+++ b/unsloth_cli/tests/test_eval.py@@ -403,6 +403,100 @@ def test_unsloth_backend_reports_missing_bitsandbytes(env, tmp_path, monkeypatchassert env["load_kwargs"]["load_in_4bit"] is False+ "run_config, expected",+ [+ ({"unsloth_training_method": "lora", "unsloth_load_in_4bit": False}, False),+ ({"unsloth_training_method": "qlora", "unsloth_load_in_4bit": True}, True),+ (None, True),+ ],+ ids = ["lora_16bit_run", "qlora_run", "unfinished_run"],+)+def test_checkpoint_follows_its_run(env, tmp_path, monkeypatch, run_config, expected):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))+ run = tmp_path / "studio" / "outputs" / "run"+ checkpoint = run / "checkpoint-15"+ checkpoint.mkdir(parents = True)+ base = {"base_model_name_or_path": "org/base"}+ (checkpoint / "adapter_config.json").write_text(json.dumps(base))+ if run_config is not None:+ (run / "adapter_config.json").write_text(json.dumps({**base, **run_config}))+ result = _run(str(checkpoint), "--tasks", "gsm8k", "-o", str(tmp_path / "out"))+ assert result.exit_code == 0, result.output+ assert env["load_kwargs"]["load_in_4bit"] is expected+++ "where, expected",+ [+ ("outputs", False),+ ("linked", False),+ ("accounts/acct1/outputs", False),+ ("accounts/acct1/exports", True),+ ],+)+def test_full_finetune_in_any_account_outputs_loads_in_16bit(+ env, tmp_path, monkeypatch, where, expected+):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))+ if where == "linked":+ (tmp_path / "disk").mkdir()+ (tmp_path / "studio").mkdir()+ (tmp_path / "studio" / "outputs").symlink_to(tmp_path / "disk", target_is_directory = True)+ where = "outputs"+ run = tmp_path / "studio" / where / "run"+ run.mkdir(parents = True)+ (run / "config.json").write_text(json.dumps({"model_type": "llama"}))+ result = _run(str(run), "--tasks", "gsm8k", "-o", str(tmp_path / "out"))+ assert result.exit_code == 0, result.output+ assert env["load_kwargs"]["load_in_4bit"] is expected+++ "files, flags, in_outputs, expected",+ [+ ({"config.json": {"model_type": "llama"}}, [], True, False),+ ({"config.json": {"model_type": "llama"}}, [], False, True),+ ({"config.json": {"text_config": {"quantization_config": {}}}}, [], False, True),+ ({"config.json": {"quantization_config": {"load_in_4bit": True}}}, [], True, True),+ ({"adapter_config.json": {"unsloth_load_in_4bit": False}}, [], False, False),+ ({"adapter_config.json": {"unsloth_training_method": "lora"}}, [], False, False),+ ({"adapter_config.json": {"unsloth_load_in_4bit": True}}, [], True, True),+ ({"adapter_config.json": {}}, [], False, True),+ ({"config.json": {"model_type": "llama"}}, ["--load-in-4bit"], True, True),+ (+ {"adapter_config.json": {"unsloth_load_in_4bit": True}},+ ["--no-load-in-4bit"],+ True,+ False,+ ),+ ],+ ids = [+ "full_finetune",+ "downloaded_full_model",+ "downloaded_nested_quantized",+ "prequantized",+ "lora_16bit",+ "lora_method_only",+ "qlora",+ "unknown_adapter",+ "explicit_4bit",+ "explicit_16bit",+ ],+)+def test_checkpoint_loads_in_the_precision_it_was_trained_in(+ env, tmp_path, monkeypatch, files, flags, in_outputs, expected+):+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))+ run = (tmp_path / "studio" / "outputs" if in_outputs else tmp_path) / "run"+ run.mkdir(parents = True)+ for name, data in files.items():+ (run / name).write_text(json.dumps({"base_model_name_or_path": "org/base", **data}))+ result = _run(str(run), "--tasks", "gsm8k", *flags, "-o", str(tmp_path / "out"))+ assert result.exit_code == 0, result.output+ assert env["load_kwargs"]["load_in_4bit"] is expected++def test_custom_dataset_builds_one_task_manager(env, tmp_path):data = tmp_path / "qa.jsonl"data.write_text('{"question": "q", "answer": "a"}\n')