Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Checkpoint scanning utilities for discovering training runs and checkpoints."""
from __future__ import annotations
import json
import os
import re
import threading
import time
import structlog
from collections import OrderedDict
from loggers import get_logger
from pathlib import Path
from typing import Dict, List, Optional, Tuple
from hub.utils.hf_tokens import HfTokenArg
from storage.studio_db import get_connection
from utils.training_runs import (
build_default_output_dir_name,
extract_project_name,
model_segment_from_default_output_dir_name,
)
from utils.paths import outputs_root, resolve_output_dir
from utils.paths.storage_roots import own_entry, within_account
logger = get_logger(__name__)
_CHECKPOINT_STEP_RE = re.compile(r"^checkpoint-(\d+)$")
def _checkpoint_step(checkpoint_name: str) -> Optional[int]:
match = _CHECKPOINT_STEP_RE.fullmatch(checkpoint_name)
if match is None:
return None
return int(match.group(1))
def _checkpoint_sort_key(checkpoint_path: Path) -> tuple[int, int, str]:
step = _checkpoint_step(checkpoint_path.name)
if step is not None:
return (0, -step, checkpoint_path.name)
return (1, 0, str(checkpoint_path))
class _History:
def __init__(self) -> None:
self._conn = None
self._dirs: Optional[list] = None
self._pairs: Optional[list] = None
def _fetch(self, sql: str) -> list:
# A failed open or query is retried by the next run, as when each run opened its own connection.
if self._conn is None:
self._conn = get_connection()
return self._conn.execute(sql).fetchall()
def dir_rows(self) -> list:
if self._dirs is None:
self._dirs = [
(row["model_name"], row["output_dir"])
for row in self._fetch(
"SELECT model_name, output_dir FROM training_runs"
" WHERE output_dir IS NOT NULL ORDER BY started_at DESC"
)
]
return self._dirs
def model_projects(self) -> list:
"""Distinct (model_name, project_name), in order of their newest run."""
if self._pairs is None:
pairs = {}
for row in self._fetch(
"SELECT model_name, config_json FROM training_runs ORDER BY started_at DESC"
):
if not row["model_name"]:
continue
project_name = None
if row["config_json"]:
try:
project_name = extract_project_name(json.loads(row["config_json"]))
except Exception:
project_name = None
pairs.setdefault((row["model_name"], project_name), None)
self._pairs = list(pairs)
return self._pairs
def close(self) -> None:
conn, self._conn = self._conn, None
if conn is not None:
try:
conn.close()
except Exception:
pass
def _infer_base_model_from_history(
checkpoint_dir: Path, history: Optional[_History] = None
) -> Optional[str]:
"""Best-effort base-model lookup using persisted Unsloth run metadata."""
own_history = history is None
if own_history:
history = _History()
try:
return _lookup_base_model(checkpoint_dir, history)
except Exception:
return None
finally:
if own_history:
history.close()
def _lookup_base_model(checkpoint_dir: Path, history: _History) -> Optional[str]:
checkpoint_name = checkpoint_dir.name
exact_dirs = (str(checkpoint_dir.resolve()), str(checkpoint_dir))
dir_rows = history.dir_rows()
for model_name, output_dir in dir_rows:
if model_name and output_dir in exact_dirs:
return model_name
for model_name, output_dir in dir_rows:
output_dir = str(output_dir).rstrip("/\\")
if model_name and (
output_dir.endswith(f"/{checkpoint_name}")
or output_dir.endswith(f"\\{checkpoint_name}")
):
return model_name
parts = checkpoint_name.rsplit("_", 1)
if len(parts) != 2 or not parts[1].isdigit():
return None
timestamp = int(parts[1])
for model_name, project_name in history.model_projects():
if (
build_default_output_dir_name(model_name, project_name, timestamp = timestamp)
== checkpoint_name
):
return model_name
return None
# trainer_state.json reaches MBs on long runs. Files changed within the settle window are not
# cached: a coarse filesystem clock can give a same-size rewrite identical timestamps.
_LOSS_CACHE: "OrderedDict[str, tuple[tuple, Optional[float]]]" = OrderedDict()
_LOSS_CACHE_MAX = 4096
_LOSS_CACHE_SETTLE_NS = 2_000_000_000
_LOSS_CACHE_LOCK = threading.Lock()
def _stat_key(st: os.stat_result) -> tuple:
return (st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns, st.st_ctime_ns)
def _cached_loss(cache_id: str, key: tuple):
with _LOSS_CACHE_LOCK:
hit = _LOSS_CACHE.get(cache_id)
if hit is None or hit[0] != key:
return False, None
_LOSS_CACHE.move_to_end(cache_id)
return True, hit[1]
def _store_loss(cache_id: str, key: tuple, loss) -> None:
if not (loss is None or type(loss) in (int, float)):
return
if time.time_ns() - max(key[3], key[4]) <= _LOSS_CACHE_SETTLE_NS:
return
with _LOSS_CACHE_LOCK:
_LOSS_CACHE[cache_id] = (key, loss)
_LOSS_CACHE.move_to_end(cache_id)
while len(_LOSS_CACHE) > _LOSS_CACHE_MAX:
_LOSS_CACHE.popitem(last = False)
def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]:
"""Read the latest training loss from trainer_state.json's log_history, or None."""
trainer_state = checkpoint_path / "trainer_state.json"
if not own_entry(trainer_state):
return None
cache_id = str(trainer_state)
try:
with open(trainer_state, encoding = "utf-8-sig") as f:
key = _stat_key(os.fstat(f.fileno()))
hit, loss = _cached_loss(cache_id, key)
if hit:
return loss
state = json.load(f)
loss = None
# Not just the last entry: on a step that also evaluates, the Trainer logs the
# eval metrics after the training loss and before it saves the checkpoint.
for entry in reversed(state.get("log_history") or []):
if isinstance(entry, dict) and "loss" in entry:
loss = entry["loss"]
break
# A write during the read changes the stat, so that read is not cached.
if _stat_key(os.fstat(f.fileno())) == key:
_store_loss(cache_id, key, loss)
return loss
except Exception as e:
logger.debug(f"Could not read loss from {trainer_state}: {e}")
return None
def parse_adapter_features(
adapter_path: str, probe_weights: bool = True
) -> Optional[Dict[str, Optional[bool]]]:
"""full_state is tri-state: no config marker proves absence (PEFT stores embedding state only as weight keys), so a weight-header probe is needed; else None."""
cfg_path = os.path.join(adapter_path, "adapter_config.json")
try:
with open(cfg_path, "r", encoding = "utf-8") as f:
cfg = json.load(f)
except (OSError, ValueError):
return None
if not isinstance(cfg, dict):
return None
def _flag(value):
return isinstance(value, bool) and value
def _filled(value):
return isinstance(value, (dict, list)) and len(value) > 0
full_state: Optional[bool] = _filled(cfg.get("full_state_modules")) or _filled(
cfg.get("modules_to_save")
)
if not full_state:
full_state = None
if probe_weights:
for weights, is_adapter_key in (
(
os.path.join(adapter_path, "adapter_model.safetensors"),
lambda key: ".lora_" in key,
),
(
os.path.join(adapter_path, "adapters.safetensors"),
lambda key: key.endswith((".lora_a", ".lora_b", ".m")),
),
):
if not os.path.exists(weights):
continue
try:
from safetensors import safe_open
with safe_open(weights, framework = "numpy") as f:
full_state = any(not is_adapter_key(key) for key in f.keys())
except Exception:
full_state = None
break
return {
"dora": _flag(cfg.get("use_dora")) or str(cfg.get("fine_tune_type", "")).lower() == "dora",
"full_state": full_state,
"moe_target_parameters": _filled(cfg.get("target_parameters")),
"non_uniform": _filled(cfg.get("rank_pattern"))
or _filled(cfg.get("alpha_pattern"))
or _filled(cfg.get("unsloth_mlx_lora_module_ranks"))
or _filled(cfg.get("unsloth_mlx_lora_module_scales")),
}
# Both probe every outputs folder, so an unreadable one is skipped, not fatal to the scan.
def _has_own_model(path: Path) -> bool:
try:
return own_entry(path / "config.json") or own_entry(path / "adapter_config.json")
except OSError:
return False
# unsloth.models.decision.is_decision_checkpoint's Laya rule; Clef folders carry config.json or adapter_config.json.
_LAYA_FILES = ("rl_agent_config.json", "model.safetensors")
_LAYA_DIRS = ("encoder", "tokenizer")
def _is_own_laya_run(path: Path) -> bool:
try:
return all(
own_entry(path / name) and (path / name).is_file() for name in _LAYA_FILES
) and all((path / name).is_dir() for name in _LAYA_DIRS)
except OSError:
return False
def _checkpoint_dirs(run_dir: Path, include_decision: bool = False) -> List[Path]:
try:
return sorted(
(
sub
for sub in run_dir.iterdir()
if sub.is_dir()
and sub.name.startswith("checkpoint-")
and within_account(sub)
and (_has_own_model(sub) or (include_decision and _is_own_laya_run(sub)))
),
key = _checkpoint_sort_key,
)
except OSError:
return []
def scan_checkpoints(
outputs_dir: str | None = None, include_decision: bool = False
) -> List[Tuple[str, List[Tuple[str, str, Optional[float]]], dict]]:
"""Scan outputs folder for training runs and their checkpoints.
``include_decision`` also lists Laya decision runs (GGUF export only; chat cannot load them).
Returns:
[(model_name, [(display_name, checkpoint_path, loss), ...], metadata), ...]
metadata keys (optional): base_model, peft_type, lora_rank.
First entry is the main adapter (loss mirrors the highest-step checkpoint)
only when the run has a final save. Numbered checkpoints are sorted
by numeric step descending; non-numbered checkpoint-* dirs keep the
previous lexicographic directory order.
"""
if outputs_dir is None:
outputs_dir = str(outputs_root())
history = _History()
try:
return _scan_checkpoints(outputs_dir, include_decision, history)
finally:
history.close()
def _scan_checkpoints(
outputs_dir: str, include_decision: bool, history: _History
) -> List[Tuple[str, List[Tuple[str, str, Optional[float]]], dict]]:
models = []
outputs_path = resolve_output_dir(outputs_dir)
if not outputs_path.exists():
logger.warning(f"Outputs directory not found: {outputs_path}")
return models
try:
for item in outputs_path.iterdir():
if not item.is_dir():
continue
if not within_account(item):
continue
has_root_model = _has_own_model(item) or (include_decision and _is_own_laya_run(item))
valid_checkpoints = _checkpoint_dirs(item, include_decision)
# A cancelled or crashed run has no final save but can still have checkpoints.
if not has_root_model and not valid_checkpoints:
continue
meta_dir = item if has_root_model else valid_checkpoints[0]
config_file = meta_dir / "config.json"
adapter_config = meta_dir / "adapter_config.json"
laya_config = meta_dir / "rl_agent_config.json"
# Training metadata from adapter_config.json / config.json
metadata: dict = {}
try:
if own_entry(adapter_config):
cfg = json.loads(adapter_config.read_text(encoding = "utf-8-sig"))
metadata["base_model"] = cfg.get("base_model_name_or_path")
metadata["peft_type"] = cfg.get("peft_type")
metadata["lora_rank"] = cfg.get("r")
metadata["adapter_features"] = parse_adapter_features(
str(meta_dir), probe_weights = False
)
elif own_entry(config_file):
cfg = json.loads(config_file.read_text(encoding = "utf-8-sig"))
metadata["base_model"] = cfg.get("_name_or_path")
elif include_decision and own_entry(laya_config):
laya = json.loads(laya_config.read_text(encoding = "utf-8-sig"))
training = (
laya.get("training") if isinstance(laya.get("training"), dict) else {}
)
metadata["base_model"] = training.get("base") or laya.get("encoder")
# Detect BNB quantization from config.json
if own_entry(config_file):
if "cfg" not in dir():
cfg = json.loads(config_file.read_text(encoding = "utf-8-sig"))
quant_cfg = cfg.get("quantization_config")
if (
isinstance(quant_cfg, dict)
and quant_cfg.get("quant_method") == "bitsandbytes"
):
metadata["is_quantized"] = True
logger.info("Detected BNB-quantized model: %s", item.name)
except Exception:
pass
# Fallback: "unsloth_Llama-3.2-3B-Instruct_1771227800" -> "unsloth/Llama-3.2-3B-Instruct".
if not metadata.get("base_model"):
metadata["base_model"] = _infer_base_model_from_history(item, history)
if not metadata.get("base_model"):
name_part = model_segment_from_default_output_dir_name(item.name)
if name_part:
idx = name_part.find("_")
if idx > 0:
metadata["base_model"] = name_part[:idx] + "/" + name_part[idx + 1 :]
else:
metadata["base_model"] = name_part
checkpoints = [
(sub.name, str(sub), _read_checkpoint_loss(sub)) for sub in valid_checkpoints
]
if has_root_model:
latest_loss = checkpoints[0][2] if checkpoints else None
checkpoints.insert(0, (item.name, str(item), latest_loss))
models.append((item.name, checkpoints, metadata))
logger.debug(f"Found model: {item.name} with {len(checkpoints)} checkpoint(s)")
# Sort by modification time (newest first)
models.sort(key = lambda x: Path(x[1][0][1]).stat().st_mtime, reverse = True)
logger.debug(f"Found {len(models)} training runs in {outputs_dir}")
return models
except Exception as e:
logger.error(f"Error scanning checkpoints: {e}")
return []
def _is_model_dir(path: Path) -> bool:
return (path / "config.json").exists() or (path / "adapter_config.json").exists()
def is_unquantized_full_model_dir(path: str | Path) -> bool:
model_dir = Path(path)
try:
if (model_dir / "adapter_config.json").exists():
return False
config = json.loads((model_dir / "config.json").read_text(encoding = "utf-8-sig"))
except (OSError, ValueError):
return False
return isinstance(config, dict) and "quantization_config" not in config
def _hub_model_config(repo_id: str, hf_token: HfTokenArg) -> Optional[dict]:
"""config.json of a Hub model repo; None for an adapter repo or on any lookup failure."""
try:
from huggingface_hub import file_exists, hf_hub_download
# An adapter repo carries a base config.json too, so a remote LoRA would read as a full model.
if file_exists(repo_id, "adapter_config.json", token = hf_token):
return None
path = hf_hub_download(repo_id, "config.json", token = hf_token)
return json.loads(Path(path).read_text(encoding = "utf-8-sig"))
except Exception:
return None
def is_unquantized_full_finetune(checkpoint_path: str, hf_token: HfTokenArg = None) -> bool:
"""Whether a local or Hub checkpoint is an unquantized full model.
False when unsure, so the caller keeps the 4-bit load that used to fit."""
try:
is_local = Path(checkpoint_path).exists()
except OSError:
return False
if is_local:
return is_unquantized_full_model_dir(checkpoint_path)
config = _hub_model_config(checkpoint_path, hf_token)
return isinstance(config, dict) and "quantization_config" not in config
def is_full_finetune_output(path: Optional[str]) -> bool:
if not path:
return False
try:
# Below 3.13 a symlink loop raises RuntimeError, not OSError.
Path(path).resolve().relative_to(outputs_root().resolve())
except (OSError, RuntimeError, ValueError):
return False
return is_unquantized_full_model_dir(path)
def has_preview_model(output_dir: Optional[str]) -> bool:
"""True when ``output_dir`` holds a previewable root model (what ``/p/{run}``
resolves). A cancelled run keeps ``output_dir`` but saves no root adapter."""
if not output_dir:
return False
path = Path(output_dir)
return path.is_dir() and _is_model_dir(path)
def preview_ref(output_dir: Optional[str]) -> Optional[str]:
"""Posix-joined ref (run or run/checkpoint) under outputs_root, so nested dirs keep a working link; None if unpreviewable, outside the root, or too deep for /p."""
if not has_preview_model(output_dir):
return None
try:
rel = Path(output_dir).resolve().relative_to(outputs_root().resolve())
except (ValueError, OSError):
return None
parts = rel.parts
if not parts or len(parts) > 2:
return None
return "/".join(parts)
def resolve_preview_checkpoint(run: str, checkpoint: Optional[str] = None) -> Path:
relative = run if not checkpoint else f"{run}/{checkpoint}"
path = resolve_output_dir(relative)
if not path.is_dir() or not _is_model_dir(path):
raise FileNotFoundError(
f"No trained checkpoint at '{relative}'. Check the run/checkpoint name (see GET /p)."
)
return path
def list_preview_targets(outputs_dir: str | None = None) -> List[dict]:
if outputs_dir is None:
outputs_dir = str(outputs_root())
targets: List[dict] = []
for run_name, checkpoints, metadata in scan_checkpoints(outputs_dir):
for display_name, path, loss in checkpoints:
is_latest = display_name == run_name
checkpoint = None if is_latest else Path(path).name
targets.append(
{
"run": run_name,
"checkpoint": checkpoint,
"ref": run_name if is_latest else f"{run_name}/{checkpoint}",
"is_latest": is_latest,
"loss": loss,
"base_model": metadata.get("base_model"),
}
)
return targets