Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Path validators and storage roots for the Hub layer."""
from __future__ import annotations
import os
import re
import threading
from collections import OrderedDict
from pathlib import Path
from typing import Optional
from loggers import get_logger
from utils.paths import path_utils as _path_utils
# One policy in utils.paths.storage_roots; a local copy drifted before.
from utils.paths.storage_roots import (
hermes_model_dirs,
studio_root,
cache_root,
assets_root,
datasets_root,
dataset_uploads_root,
recipe_datasets_root,
outputs_root,
exports_root,
tmp_root,
ensure_dir,
legacy_hf_cache_dir,
hf_default_cache_dir,
lmstudio_model_dirs,
ollama_model_dirs,
omlx_model_dirs,
well_known_model_dirs,
)
logger = get_logger(__name__)
# Re-export shim: marks them used so the import-hoist safety net does not flag them.
_REEXPORTED = (
hermes_model_dirs,
lmstudio_model_dirs,
ollama_model_dirs,
omlx_model_dirs,
well_known_model_dirs,
studio_root,
cache_root,
assets_root,
datasets_root,
dataset_uploads_root,
recipe_datasets_root,
outputs_root,
exports_root,
tmp_root,
ensure_dir,
legacy_hf_cache_dir,
hf_default_cache_dir,
)
# normalize_path reads these at call time and tests set them, so they stay attributes of this module.
_IS_WSL = _path_utils._IS_WSL
_WSL_AUTOMOUNT_ROOT = _path_utils._WSL_AUTOMOUNT_ROOT
def normalize_path(path: str) -> str:
if not path:
return path
if len(path) >= 3 and path[1] == ":" and path[2] in ("\\", "/"):
if _IS_WSL:
drive = path[0].lower()
rest = path[3:].replace("\\", "/")
return f"{_WSL_AUTOMOUNT_ROOT}{drive}/{rest}"
return path.replace("\\", "/")
return path.replace("\\", "/")
def is_local_path(path: str) -> bool:
if not path:
return False
normalized = normalize_path(path)
has_local_syntax = (
path.startswith(("/", ".", "~"))
or ":" in path
or "\\" in path
or os.path.isabs(path)
or os.path.isabs(normalized)
)
if path.count("/") == 1 and not has_local_syntax:
return False
try:
if has_local_syntax and Path(normalized).expanduser().exists():
return True
except Exception:
pass
return has_local_syntax
_VALID_REPO_ID_SEGMENT = re.compile(r"^[A-Za-z0-9_](?:[A-Za-z0-9._-]*[A-Za-z0-9_])?$")
_MAX_REPO_ID_LENGTH = 96
def is_valid_repo_id(repo_id: str) -> bool:
"""Validate Hugging Face ``repo_name`` or ``namespace/repo_name`` IDs."""
if not repo_id or repo_id != repo_id.strip():
return False
if repo_id.endswith(".git"):
return False
if "--" in repo_id or ".." in repo_id:
return False
segments = repo_id.split("/")
if len(segments) not in (1, 2):
return False
# Like validate_repo_id: the 96-char limit is per segment.
return all(
segment not in ("", ".", "..")
and len(segment) <= _MAX_REPO_ID_LENGTH
and _VALID_REPO_ID_SEGMENT.fullmatch(segment) is not None
for segment in segments
)
_GGUF_VARIANT_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]")
_MAX_GGUF_VARIANT_LENGTH = 512
def is_valid_gguf_variant(variant: str) -> bool:
"""Unknown GGUF layouts use snapshot-relative keys, which may hold slashes or spaces."""
if not variant or variant != variant.strip():
return False
if len(variant) > _MAX_GGUF_VARIANT_LENGTH:
return False
if _GGUF_VARIANT_CONTROL_CHARS.search(variant) or not variant.isprintable():
return False
normalized = variant.replace("\\", "/")
return all(segment not in ("", ".", "..") for segment in normalized.split("/"))
# Bounded LRU memo for resolve_cached_repo_id_case so long-lived processes cannot grow it unbounded.
_CACHE_CASE_RESOLUTION_MEMO_MAX = 512
_CACHE_CASE_RESOLUTION_MEMO: "OrderedDict[tuple[str, str], str]" = OrderedDict()
_CACHE_CASE_RESOLUTION_LOCK = threading.Lock()
def _memo_get(memo_key: tuple[str, str]) -> Optional[str]:
with _CACHE_CASE_RESOLUTION_LOCK:
value = _CACHE_CASE_RESOLUTION_MEMO.get(memo_key)
if value is not None:
_CACHE_CASE_RESOLUTION_MEMO.move_to_end(memo_key)
return value
def _memo_set(memo_key: tuple[str, str], value: str) -> None:
with _CACHE_CASE_RESOLUTION_LOCK:
_CACHE_CASE_RESOLUTION_MEMO[memo_key] = value
_CACHE_CASE_RESOLUTION_MEMO.move_to_end(memo_key)
while len(_CACHE_CASE_RESOLUTION_MEMO) > _CACHE_CASE_RESOLUTION_MEMO_MAX:
_CACHE_CASE_RESOLUTION_MEMO.popitem(last = False)
def _memo_drop(memo_key: tuple[str, str]) -> None:
with _CACHE_CASE_RESOLUTION_LOCK:
_CACHE_CASE_RESOLUTION_MEMO.pop(memo_key, None)
def _hf_hub_cache_dir() -> Path:
from utils.hf_cache_settings import get_hf_cache_paths
return get_hf_cache_paths().hub_cache
def _hf_hub_cache_dirs() -> list[Path]:
roots: list[Path] = []
seen: set[str] = set()
def _add(path: Path) -> None:
try:
resolved = path.resolve()
except OSError:
return
key = str(resolved)
if key in seen or not resolved.is_dir():
return
seen.add(key)
roots.append(resolved)
from utils.hf_cache_settings import known_hf_hub_caches
for configured in known_hf_hub_caches():
_add(configured)
try:
_add(legacy_hf_cache_dir())
_add(hf_default_cache_dir())
except Exception as exc:
logger.debug("Could not enumerate secondary HF cache roots: %s", exc)
return roots
def _assert_contained(resolved: Path, root: Path) -> None:
try:
resolved_real = Path(os.path.realpath(resolved))
root_real = Path(os.path.realpath(root))
except OSError as exc:
raise ValueError(f"path resolution failed: {exc}") from exc
try:
resolved_real.relative_to(root_real)
except ValueError as exc:
raise ValueError(f"path escapes root: {resolved!s}") from exc
def path_is_same_or_child(path: Path, root: Path) -> bool:
"""Compares normcased realpaths so symlinks and case-only matches count; any error returns False."""
try:
path_real = os.path.normcase(os.path.realpath(str(path)))
root_real = os.path.normcase(os.path.realpath(str(root)))
return os.path.commonpath([path_real, root_real]) == root_real
except (OSError, ValueError):
return False
def resolve_dataset_path(path_value: str) -> Path:
raw = str(path_value or "").strip()
if "\x00" in raw:
raise ValueError("dataset path may not contain null bytes")
# Normalize first so a backslashed ".." is caught by the traversal guard below.
normalized = normalize_path(raw)
path = Path(normalized).expanduser()
if ".." in path.parts:
raise ValueError(f"dataset path may not contain '..' segments: {raw!r}")
if path.is_absolute():
for root in (datasets_root(), dataset_uploads_root(), recipe_datasets_root()):
try:
_assert_contained(path, root)
return path
except ValueError:
continue
raise ValueError(f"dataset path must be relative or under a dataset root: {raw!r}")
parts = [part for part in Path(normalized).parts if part not in ("", ".")]
if parts[:2] == ["assets", "datasets"]:
parts = parts[2:]
if parts and parts[0] == "uploads":
cleaned = Path(*parts[1:]) if len(parts) > 1 else Path()
return dataset_uploads_root() / cleaned
if parts and parts[0] == "recipes":
cleaned = Path(*parts[1:]) if len(parts) > 1 else Path()
return recipe_datasets_root() / cleaned
cleaned = Path(*parts) if parts else Path()
candidates = [
dataset_uploads_root() / cleaned,
recipe_datasets_root() / cleaned,
datasets_root() / cleaned,
dataset_uploads_root() / cleaned.name,
recipe_datasets_root() / cleaned.name,
]
for candidate in candidates:
if candidate.exists():
return candidate
return candidates[0]
def resolve_cached_repo_id_case(
model_name: str,
use_memo: bool = True,
repo_type: str = "model",
) -> str:
"""Reuses the cached spelling when a case-variant exists, to avoid a duplicate download."""
if not model_name or "/" not in model_name:
return model_name
cache_dirs = _hf_hub_cache_dirs()
if not cache_dirs:
return model_name
prefix = f"{repo_type}s--"
expected_dir = f"{prefix}{model_name.replace('/', '--')}"
memo_key = (repo_type, model_name)
for cache_dir in cache_dirs:
exact_path = cache_dir / expected_dir
if exact_path.is_dir():
if use_memo:
_memo_set(memo_key, model_name)
return model_name
if use_memo:
cached = _memo_get(memo_key)
if cached is not None:
if any(
(cache_dir / f"{prefix}{cached.replace('/', '--')}").is_dir()
for cache_dir in cache_dirs
):
return cached
_memo_drop(memo_key)
expected_lower = expected_dir.lower()
try:
candidates: set[str] = set()
for cache_dir in cache_dirs:
for entry in cache_dir.iterdir():
if not entry.is_dir():
continue
if entry.name.lower() != expected_lower:
continue
# Lowercased full-name match proves the prefix; case-sensitive startswith rejects Models--Org--Repo.
repo_part = entry.name[len(prefix) :]
if not repo_part:
continue
candidates.add(repo_part.replace("--", "/"))
if candidates:
resolved = sorted(candidates)[0]
if use_memo:
_memo_set(memo_key, resolved)
return resolved
except Exception as exc:
logger.debug(f"resolve_cached_repo_id_case failed for {model_name!r}: {exc}")
return model_name
__all__ = [
"assets_root",
"cache_root",
"dataset_uploads_root",
"datasets_root",
"ensure_dir",
"exports_root",
"hf_default_cache_dir",
"is_local_path",
"is_valid_gguf_variant",
"is_valid_repo_id",
"legacy_hf_cache_dir",
"lmstudio_model_dirs",
"normalize_path",
"ollama_model_dirs",
"omlx_model_dirs",
"outputs_root",
"path_is_same_or_child",
"recipe_datasets_root",
"resolve_cached_repo_id_case",
"resolve_dataset_path",
"studio_root",
"tmp_root",
"well_known_model_dirs",
]