Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Score a 16-bit fine-tune with unsloth eval in 16-bit, not in 4-bit (#13758)

Nilay2a8def3bdee7
diff --git a/unsloth_cli/commands/eval.py b/unsloth_cli/commands/eval.py
index 72d3d2000bf..b97b8b91ee1 100644
--- a/unsloth_cli/commands/eval.py
+++ b/unsloth_cli/commands/eval.py
@@ -75,6 +75,49 @@ def resolve_base_model(model: str) -> Optional[str]:
return data.get("base_model_name_or_path") if isinstance(data, dict) else None
+def _is_studio_output(path: Path) -> bool:
+ # same scope as Studio Chat: downloads and merged exports keep the 4-bit default
+ try:
+ from unsloth_cli._inference import ensure_studio_backend_path
+
+ ensure_studio_backend_path(seed_cache_env = False)
+ from utils.paths import outputs_root, studio_root
+
+ accounts = studio_root() / "accounts"
+ roots = [outputs_root()]
+ if accounts.is_dir():
+ roots += [account / "outputs" for account in accounts.iterdir()]
+ resolved = path.resolve()
+ return any(resolved.is_relative_to(root.resolve()) for root in roots)
+ except Exception:
+ return False
+
+
+def _read_config(file: Path) -> Optional[dict]:
+ try:
+ config = json.loads(file.read_text(encoding = "utf-8-sig"))
+ except (ValueError, OSError):
+ return None
+ return config if isinstance(config, dict) else None
+
+
+def _trained_in_4bit(model: str) -> bool:
+ path = Path(model).absolute()
+ is_adapter = (path / "adapter_config.json").is_file()
+ config = _read_config(path / ("adapter_config.json" if is_adapter else "config.json"))
+ if config is None:
+ return True
+ if not is_adapter:
+ return "quantization_config" in config or not _is_studio_output(path)
+ if "unsloth_training_method" not in config and path.name.startswith("checkpoint-"):
+ # Studio records the precision on the finished run, not on its checkpoint-N folders
+ config = _read_config(path.parent / "adapter_config.json") or config
+ recorded = config.get("unsloth_load_in_4bit")
+ if isinstance(recorded, bool):
+ return recorded
+ return config.get("unsloth_training_method") != "lora"
+
+
def _has_tokenizer_files(model: str) -> bool:
path = Path(model)
if path.is_dir():
@@ -372,8 +415,11 @@ def evaluate(
max_seq_length: int = typer.Option(
2048, "--max-seq-length", help = "Max sequence length for the model."
),
- load_in_4bit: bool = typer.Option(
- True, "--load-in-4bit/--no-load-in-4bit", help = "Load the model in 4-bit."
+ load_in_4bit: Optional[bool] = typer.Option(
+ None,
+ "--load-in-4bit/--no-load-in-4bit",
+ help = "Load the model in 4-bit. Default: the precision a local checkpoint or "
+ "adapter was trained in, 4-bit otherwise.",
),
backend: str = typer.Option(
"unsloth",
@@ -449,6 +495,13 @@ def evaluate(
if hf_token:
os.environ["HF_TOKEN"] = hf_token
effective_base = base_model or resolve_base_model(model)
+ if load_in_4bit is None:
+ load_in_4bit = _trained_in_4bit(model)
+ if not load_in_4bit:
+ typer.echo(
+ "Note: loading in 16-bit, the precision this checkpoint was trained in "
+ "(pass --load-in-4bit to override)."
+ )
tmp_dir = Path(tempfile.mkdtemp(prefix = "unsloth_eval_"))
try:
diff --git a/unsloth_cli/tests/test_eval.py b/unsloth_cli/tests/test_eval.py
index 91fd75c751e..db67268872c 100644
--- a/unsloth_cli/tests/test_eval.py
+++ b/unsloth_cli/tests/test_eval.py
@@ -403,6 +403,100 @@ def test_unsloth_backend_reports_missing_bitsandbytes(env, tmp_path, monkeypatch
assert env["load_kwargs"]["load_in_4bit"] is False
+ "run_config, expected",
+ [
+ ({"unsloth_training_method": "lora", "unsloth_load_in_4bit": False}, False),
+ ({"unsloth_training_method": "qlora", "unsloth_load_in_4bit": True}, True),
+ (None, True),
+ ],
+ ids = ["lora_16bit_run", "qlora_run", "unfinished_run"],
+)
+def test_checkpoint_follows_its_run(env, tmp_path, monkeypatch, run_config, expected):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ run = tmp_path / "studio" / "outputs" / "run"
+ checkpoint = run / "checkpoint-15"
+ checkpoint.mkdir(parents = True)
+ base = {"base_model_name_or_path": "org/base"}
+ (checkpoint / "adapter_config.json").write_text(json.dumps(base))
+ if run_config is not None:
+ (run / "adapter_config.json").write_text(json.dumps({**base, **run_config}))
+ result = _run(str(checkpoint), "--tasks", "gsm8k", "-o", str(tmp_path / "out"))
+ assert result.exit_code == 0, result.output
+ assert env["load_kwargs"]["load_in_4bit"] is expected
+
+
+ "where, expected",
+ [
+ ("outputs", False),
+ ("linked", False),
+ ("accounts/acct1/outputs", False),
+ ("accounts/acct1/exports", True),
+ ],
+)
+def test_full_finetune_in_any_account_outputs_loads_in_16bit(
+ env, tmp_path, monkeypatch, where, expected
+):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ if where == "linked":
+ (tmp_path / "disk").mkdir()
+ (tmp_path / "studio").mkdir()
+ (tmp_path / "studio" / "outputs").symlink_to(tmp_path / "disk", target_is_directory = True)
+ where = "outputs"
+ run = tmp_path / "studio" / where / "run"
+ run.mkdir(parents = True)
+ (run / "config.json").write_text(json.dumps({"model_type": "llama"}))
+ result = _run(str(run), "--tasks", "gsm8k", "-o", str(tmp_path / "out"))
+ assert result.exit_code == 0, result.output
+ assert env["load_kwargs"]["load_in_4bit"] is expected
+
+
+ "files, flags, in_outputs, expected",
+ [
+ ({"config.json": {"model_type": "llama"}}, [], True, False),
+ ({"config.json": {"model_type": "llama"}}, [], False, True),
+ ({"config.json": {"text_config": {"quantization_config": {}}}}, [], False, True),
+ ({"config.json": {"quantization_config": {"load_in_4bit": True}}}, [], True, True),
+ ({"adapter_config.json": {"unsloth_load_in_4bit": False}}, [], False, False),
+ ({"adapter_config.json": {"unsloth_training_method": "lora"}}, [], False, False),
+ ({"adapter_config.json": {"unsloth_load_in_4bit": True}}, [], True, True),
+ ({"adapter_config.json": {}}, [], False, True),
+ ({"config.json": {"model_type": "llama"}}, ["--load-in-4bit"], True, True),
+ (
+ {"adapter_config.json": {"unsloth_load_in_4bit": True}},
+ ["--no-load-in-4bit"],
+ True,
+ False,
+ ),
+ ],
+ ids = [
+ "full_finetune",
+ "downloaded_full_model",
+ "downloaded_nested_quantized",
+ "prequantized",
+ "lora_16bit",
+ "lora_method_only",
+ "qlora",
+ "unknown_adapter",
+ "explicit_4bit",
+ "explicit_16bit",
+ ],
+)
+def test_checkpoint_loads_in_the_precision_it_was_trained_in(
+ env, tmp_path, monkeypatch, files, flags, in_outputs, expected
+):
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "studio"))
+ run = (tmp_path / "studio" / "outputs" if in_outputs else tmp_path) / "run"
+ run.mkdir(parents = True)
+ for name, data in files.items():
+ (run / name).write_text(json.dumps({"base_model_name_or_path": "org/base", **data}))
+ result = _run(str(run), "--tasks", "gsm8k", *flags, "-o", str(tmp_path / "out"))
+ assert result.exit_code == 0, result.output
+ assert env["load_kwargs"]["load_in_4bit"] is expected
+
+
def test_custom_dataset_builds_one_task_manager(env, tmp_path):
data = tmp_path / "qa.jsonl"
data.write_text('{"question": "q", "answer": "a"}\n')