Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: Keep API keys from training with the server's Hugging Face login (#10809)

Nilay14dcd6c16048
diff --git a/studio/backend/core/training/diffusion_training_service.py b/studio/backend/core/training/diffusion_training_service.py
index 26b98a84482..862f35264f1 100644
--- a/studio/backend/core/training/diffusion_training_service.py
+++ b/studio/backend/core/training/diffusion_training_service.py
@@ -86,7 +86,20 @@ def _run_diffusion_child(*, event_queue: Any, stop_queue: Any, config: dict) ->
def _default_target(*, event_queue: Any, stop_queue: Any, config: dict) -> None:
# First thing in the child (before torch): self-bind to parent death and scrub the native path
- # secret, like the other workers.
+ # secret, like the other workers. Token policy first of all, ahead of the account branch
+ # below, which returns: both children need it applied.
+ if not config.get("allow_ambient", True):
+ # Before any huggingface_hub import, as the LLM worker does: a child env is seeded from
+ # the parent's, so not setting a token is not denying one.
+ import os
+
+ from hub.utils.hf_tokens import apply_token_to_child_env, hf_token_arg, is_anonymous
+
+ hf_token = hf_token_arg(config.get("hf_token"), allow_ambient_token = False)
+ apply_token_to_child_env(os.environ, hf_token)
+ if is_anonymous(hf_token):
+ os.environ["HF_TOKEN_PATH"] = os.devnull
+
account = config.pop("_job_account", None)
if account is not None:
from core.training.account_jobs import run_account_child
diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py
index 9c4d0d71a50..eeb9f6c0488 100644
--- a/studio/backend/core/training/training.py
+++ b/studio/backend/core/training/training.py
@@ -44,6 +44,7 @@ from typing import Optional, Tuple, Any, Callable, Union, TYPE_CHECKING, Literal
if TYPE_CHECKING:
import matplotlib.pyplot as plt
+from hub.utils.hf_tokens import hf_token_arg
from utils.hardware import get_device, prepare_gpu_selection
from utils.native_path_leases import (
native_path_secret_removed_for_child_start,
@@ -273,6 +274,7 @@ def _build_training_worker_config(values: dict[str, Any]) -> dict[str, Any]:
"trust_remote_code": values.get("trust_remote_code", False),
"approved_remote_code_fingerprint": values.get("approved_remote_code_fingerprint"),
"subject": values.get("subject"),
+ "allow_ambient": values.get("allow_ambient", True),
"gpu_ids": values.get("gpu_ids"),
"s3_config": values.get("s3_config"),
"disable_xet": values.get("disable_xet", False),
@@ -1683,6 +1685,9 @@ class TrainingBackend:
self._pump_running = False
config = _build_training_worker_config(kwargs)
+ hf_token = hf_token_arg(
+ config["hf_token"], allow_ambient_token = config.get("allow_ambient", True)
+ )
_apply_cache_pins(config)
from .provenance import initialize_resource_provenance
@@ -1697,7 +1702,7 @@ class TrainingBackend:
gpu_ids = kwargs.get("gpu_ids")
gpu_selection_kwargs = dict(
model_name = config["model_name"],
- hf_token = config["hf_token"] or None,
+ hf_token = hf_token,
training_type = config["training_type"],
load_in_4bit = config["load_in_4bit"],
batch_size = config.get("batch_size", 4),
@@ -1745,7 +1750,7 @@ class TrainingBackend:
effective_training_load_in_4bit(
config,
config.get("model_snapshot_path") or config["model_name"],
- config.get("hf_token") or None,
+ hf_token,
)
with self._lock:
if not self._start_request_allows_spawn_locked(start_request_id, job_id):
diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py
index e803597392e..7aceed273f8 100644
--- a/studio/backend/core/training/worker.py
+++ b/studio/backend/core/training/worker.py
@@ -139,6 +139,21 @@ def _resolve_cached_model_load_name(config: dict) -> str:
return config.get("model_snapshot_path") or config["model_name"]
+def _worker_hf_token(config: dict):
+ """The caller's credential as a three-valued ``HfTokenArg``, not a bare ``str | None``.
+
+ The environment scrub makes the worker's NETWORK traffic anonymous and nothing more: the disk
+ cache stays readable, and the guards over it discriminate on the ``False`` sentinel, so
+ ``or None`` handed an API key the ambient caller class and its cached private weights. It bites
+ on the repos the route never authorized -- a LoRA checkpoint's base, sibling scan targets, a
+ fallback load target. ``core/export/worker.py`` rebuilds it per command for the same reason.
+ """
+ from hub.utils.hf_tokens import hf_token_arg
+ return hf_token_arg(
+ config.get("hf_token"), allow_ambient_token = config.get("allow_ambient", True)
+ )
+
+
def _effective_training_load_in_4bit(
config: dict, model_load_target: str, hf_token: str | None
) -> bool:
@@ -642,8 +657,7 @@ def _load_embedding_hf_dataset(
subset = config.get("subset") or None
train_split = config.get("train_split", "train") or "train"
revision = config.get("dataset_revision")
- token = config.get("hf_token", "")
- token = token if token and token.strip() else None
+ token = _worker_hf_token(config)
dataset = None
config["_dataset_loaded_from_exact_snapshot"] = False
@@ -2319,7 +2333,7 @@ def _run_mlx_training(event_queue, stop_queue, config):
mx.set_wired_limit(wired_cap)
model_name = config["model_name"]
- hf_token = config.get("hf_token") or None
+ hf_token = _worker_hf_token(config)
if hf_token:
os.environ["HF_TOKEN"] = hf_token
model_load_name = _resolve_cached_model_load_name(config)
@@ -3116,7 +3130,7 @@ def run_mlx_training_process(
# Must precede detect_hardware(): its MLX stack check imports mlx_lm, hence transformers.
_activate_transformers_version_or_warn(
model_load_target,
- config.get("hf_token") or None,
+ _worker_hf_token(config),
)
from utils.hardware import hardware as _hw
@@ -3208,6 +3222,14 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
)
os.environ["PYTHONWARNINGS"] = "ignore" # before imports
+ if not config.get("allow_ambient", True):
+ from hub.utils.hf_tokens import apply_token_to_child_env, hf_token_arg, is_anonymous
+
+ hf_token = hf_token_arg(config.get("hf_token"), allow_ambient_token = False)
+ apply_token_to_child_env(os.environ, hf_token)
+ if is_anonymous(hf_token):
+ os.environ["HF_TOKEN_PATH"] = os.devnull
+
# HTTP-fallback respawn: disable Xet before any huggingface_hub import (read at import time).
from utils.hf_xet_fallback import child_should_disable_xet
@@ -3298,7 +3320,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
# Must precede detect_hardware(): its MLX stack check imports mlx_lm, hence transformers.
_activate_transformers_version_or_warn(
model_load_target,
- config.get("hf_token") or None,
+ _worker_hf_token(config),
)
mlx_transformers_activated = True
@@ -3318,7 +3340,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
try:
_activate_transformers_version(
model_load_target,
- config.get("hf_token") or None,
+ _worker_hf_token(config),
)
except Exception as exc:
event_queue.put(
@@ -3341,7 +3363,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
any(sub in _lowered for sub in _NEMOTRON_TRUST_SUBSTRINGS)
and (_lowered.startswith("unsloth/") or _lowered.startswith("nvidia/"))
# Confirm a genuine first-party Hub repo (not a spoofed "unsloth/" name); authenticated.
- and is_trusted_org_repo(model_name, hf_token = config.get("hf_token") or None)
+ and is_trusted_org_repo(model_name, hf_token = _worker_hf_token(config))
and not config.get("trust_remote_code", False)
):
config["trust_remote_code"] = True
@@ -3353,7 +3375,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
security_error = _model_load_security_error(
config,
_resolve_cached_model_load_name(config),
- config.get("hf_token") or None,
+ _worker_hf_token(config),
)
if security_error:
event_queue.put({"type": "error", **security_error, "ts": time.time()})
@@ -3370,7 +3392,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
wants_causal_conv1d = resolved_model_wants_causal_conv1d(
model_name,
model_load_target,
- config.get("hf_token") or None,
+ _worker_hf_token(config),
)
_ensure_causal_conv1d_fast_path(
event_queue,
@@ -3813,8 +3835,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) ->
# Pipeline order: detect -> dataset -> model -> prepare -> train, so both never hold VRAM at once.
try:
- hf_token = config.get("hf_token", "")
- hf_token = hf_token if hf_token and hf_token.strip() else None
+ hf_token = _worker_hf_token(config)
model_load_name = _resolve_cached_model_load_name(config)
model_local_only = _model_local_files_only(config)
model_revision = None if model_local_only else config.get("model_revision")
@@ -4688,8 +4709,7 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) ->
_send_status(event_queue, "Loading embedding model...")
try:
- hf_token = config.get("hf_token", "")
- hf_token = hf_token if hf_token and hf_token.strip() else None
+ hf_token = _worker_hf_token(config)
max_seq_length = config.get("max_seq_length", 512)
training_type = config.get("training_type", "LoRA/QLoRA")
use_lora = training_type == "LoRA/QLoRA"
diff --git a/studio/backend/hub/utils/hf_cache_state.py b/studio/backend/hub/utils/hf_cache_state.py
index a073a6abccc..da97214de88 100644
--- a/studio/backend/hub/utils/hf_cache_state.py
+++ b/studio/backend/hub/utils/hf_cache_state.py
@@ -369,6 +369,124 @@ def validated_repo_cache_path(
return None
+# repo_folder_name builds these: f"{repo_type}s--" + repo_id.split("/") joined by "--". A cached
+# private dataset or space holds readable bytes exactly as a model does.
+_CACHE_REPO_TYPES = ("model", "dataset", "space")
+
+
+def cached_repo_ref_for_path(path: Path | str) -> Optional[tuple[str, str]]:
+ """Map a path inside a Hub cache to the ``(repo_id, repo_type)`` that owns it.
+
+ ``None`` means "not the operator's cache", which every caller reads as "no check needed", so a
+ miss here is fail-OPEN. Hence every repo type rather than models alone, and candidates walked
+ OUTWARD rather than committing to the deepest match: repo file paths are arbitrary, so a
+ private snapshot may itself contain a directory named ``models--foo--bar``.
+ """
+ try:
+ resolved = Path(path).expanduser().resolve(strict = True)
+ except (OSError, RuntimeError, ValueError):
+ return None
+ scan_errors: list = []
+ resolved_roots: Optional[list] = None
+
+ def roots() -> list:
+ # Lazily and once: an ordinary local path matches no prefix below, so the stat calls are
+ # wasted and hf_cache_roots reaches callers that never stub it.
+ nonlocal resolved_roots
+ if resolved_roots is None:
+ resolved_roots = []
+ for root in hf_cache_roots(scan_errors):
+ try:
+ resolved_roots.append(root.resolve(strict = True))
+ except (OSError, RuntimeError):
+ scan_errors.append(root)
+ return resolved_roots
+
+ fallback: Optional[tuple[str, str]] = None
+ for candidate in (resolved, *resolved.parents):
+ name = candidate.name
+ lowered = name.lower()
+ for repo_type in _CACHE_REPO_TYPES:
+ prefix = f"{repo_type}s--"
+ if not lowered.startswith(prefix) or len(name) <= len(prefix):
+ continue
+ # "--" is forbidden in a repo id (validate_repo_id rejects it), so this is unambiguous.
+ ref = (name[len(prefix) :].replace("--", "/"), repo_type)
+ if any(same_existing_path(candidate.parent, root) for root in roots()):
+ return ref
+ # An unreadable root may be this path's: authorize rather than skip. Held, so an outer
+ # candidate genuinely under a readable root still wins.
+ if scan_errors and fallback is None:
+ fallback = ref
+ return fallback
+
+
+def cached_repo_id_for_path(path: Path | str, repo_type: str = "model") -> Optional[str]:
+ """The *repo_type*-restricted view of :func:`cached_repo_ref_for_path`."""
+ ref = cached_repo_ref_for_path(path)
+ if ref is None or ref[1] != repo_type:
+ return None
+ return ref[0]
+
+
+def _is_repo_boilerplate(path: Path) -> bool:
+ """Whether *path* is repository furniture rather than anything a load consumes.
+
+ Deliberately a denylist of things no loader reads, not an allowlist of weight formats: a name
+ this does not recognise still counts as content, so a private snapshot in an unusual format is
+ refused rather than waved through."""
+ name = path.name
+ if name in (".gitattributes", ".gitignore", ".gitmodules"):
+ return True
+ if name.split(".", 1)[0].upper() in ("LICENSE", "LICENCE", "NOTICE"):
+ return True
+ return path.suffix.lower() in (".md", ".txt")
+
+
+def repo_cache_has_usable_snapshot(
+ repo_type: str,
+ repo_id: str,
+ metadata_filenames: tuple[str, ...] = (),
+) -> bool:
+ """Whether the cache holds a snapshot of *repo_id* that a load could actually consume.
+
+ NOT "does a repo directory exist": an interrupted download leaves one with no snapshot under
+ it, and refusing on that costs a caller bytes that were never there, which the anonymous
+ ``/auth-check`` cannot clear on an ``HF_ENDPOINT`` mirror that does not serve it. An unlistable
+ directory counts as usable: the guard's own failure must not open the path it guards.
+ """
+ scan_errors: list = []
+ for repo_dir in iter_repo_cache_dirs(repo_type, repo_id, scan_errors = scan_errors):
+ snapshots = repo_dir / "snapshots"
+ try:
+ revisions = list(snapshots.iterdir()) if snapshots.is_dir() else []
+ except OSError:
+ return True
+ if not metadata_filenames:
+ # Some content, not merely a revision directory: an interrupted download leaves the
+ # latter empty, and a barely started one leaves the model card and .gitattributes that
+ # huggingface_hub fetches first. Any other file at any depth, since a diffusers
+ # snapshot keeps its weights in per-component subdirectories and the metadata name
+ # varies by family: an extension allowlist would skip the check for a private snapshot
+ # in a format not on it, which is the failure that matters. Short-circuits on the first
+ # hit; an unreadable tree counts, as above.
+ for revision in revisions:
+ if revision.is_file():
+ return True
+ try:
+ if any(
+ p.is_file() and not _is_repo_boilerplate(p) for p in revision.rglob("*")
+ ):
+ return True
+ except OSError:
+ return True
+ continue
+ for revision in revisions:
+ if any((revision / name).is_file() for name in metadata_filenames):
+ return True
+ return bool(scan_errors)
+
+
def latest_snapshot_from_cache_path(
local_path: Optional[str],
repo_type: str,
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index a2dfac1d642..c60c4e5fff9 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -86,6 +86,7 @@ except ImportError:
from utils.paths import is_local_path, normalize_path, resolve_dataset_path
from auth.authentication import authenticated_via_api_key, get_current_subject
+from hub.utils.hf_tokens import HfTokenArg, cached_read_refused, hf_token_arg
from utils.utils import (
canonical_model_repo_id,
@@ -460,7 +461,7 @@ def _has_adapter_metadata(path: Path) -> bool:
return path.is_dir() and (path / "adapter_config.json").is_file()
-def _remote_untrainable_model_format(model_name: str, hf_token: Optional[str]) -> Optional[str]:
+def _remote_untrainable_model_format(model_name: str, hf_token: HfTokenArg) -> Optional[str]:
from huggingface_hub import model_info as hf_model_info
from hub.utils.hf_errors import hf_error_status
from utils.security import load_scan_target
@@ -559,6 +560,111 @@ def _remote_untrainable_model_format(model_name: str, hf_token: Optional[str]) -
return None
+def _hf_dataset_is_the_source(request: Any) -> bool:
+ """Whether the run will actually read ``hf_dataset``.
+
+ Mirrors the precedence in UnslothTrainer.load_and_format_dataset: local_datasets wins, then
+ s3_config, and only then dataset_source. A payload carrying a Hub id alongside either never
+ loads the Hub repo, so nothing about it can gate the start."""
+ if getattr(request, "local_datasets", None):
+ return False
+ if getattr(request, "s3_config", None):
+ return False
+ return bool(getattr(request, "hf_dataset", None))
+
+
+def _refuse_unauthorized_cached_dataset(
+ request: TrainingStartRequest, hf_token: HfTokenArg
+) -> None:
+ from hub.utils.dataset_cache import dataset_cache_can_answer, training_dataset_cache_pin
+
+ dataset_id = request.hf_dataset
+
+ def has_cached_dataset():
+ if dataset_cache_can_answer(dataset_id):
+ return True
+ pin, _ = training_dataset_cache_pin(
+ dataset_id,
+ request.dataset_snapshot_path or request.dataset_local_path,
+ )
+ return pin is not None
+
+ # Cached and offline starts skip the Hub check below, and the worker then loads the cache.
+ if cached_read_refused(
+ hf_token,
+ repo_id = dataset_id,
+ repo_type = "dataset",
+ is_cached = has_cached_dataset,
+ offline = hf_env_offline(),
+ ):
+ raise _hf_preflight_error(
+ 422,
+ "hf_dataset_access_denied",
+ "Hugging Face denied access to this cached dataset. Add a token with repository access.",
+ )
+
+
+def _refuse_unauthorized_cached_local_paths(
+ paths: list[str],
+ hf_token: HfTokenArg,
+ label: str = "dataset",
+) -> None:
+ """Refuse a local path that is really a snapshot inside the operator's Hub cache.
+
+ ``local_datasets`` takes any readable path, so pointing it at
+ ``.../datasets--org--private/snapshots/<rev>/data.parquet`` trained on the operator's private
+ dataset with no token at all. Same leak the model leg closes above, on the one input that
+ reaches the trainer without ever naming a repo.
+ """
+ from hub.utils.hf_cache_state import (
+ cached_repo_ref_for_path,
+ repo_cache_has_usable_snapshot,
+ )
+
+ repo_type = "dataset" if label.endswith("dataset") else label
+
+ def cached_under_its_own_id(repo_id: str):
+ # A Hub id rather than a path. The remote probe that would normally cover it can fail open
+ # (a HEAD that cannot be sent is not a denial), and an offline worker then loads the cached
+ # copy with no credential, so ask the disk directly -- but for a SNAPSHOT, not for the repo
+ # directory: an interrupted download leaves the latter behind holding nothing, and refusing
+ # on it costs a public base a download it was entitled to. No metadata filter, because the
+ # name differs by family (model_index.json for diffusers, config.json for a text model) and
+ # guessing wrong here would fail open. Unreadable still counts.
+ def probe() -> bool:
+ return repo_cache_has_usable_snapshot(repo_type, repo_id)
+
+ return probe
+
+ for dataset_path in paths:
+ candidate = (dataset_path or "").strip()
+ if not candidate:
+ continue
+ cached_ref = cached_repo_ref_for_path(candidate)
+ if cached_ref is None:
+ if is_local_path(candidate) or candidate.count("/") != 1:
+ continue
+ cached_ref = (candidate, repo_type)
+ is_cached = cached_under_its_own_id(candidate)
+ else:
+ is_cached = lambda: True # noqa: E731 -- the path itself is the evidence
+ if cached_read_refused(
+ hf_token,
+ repo_id = cached_ref[0],
+ repo_type = cached_ref[1],
+ is_cached = is_cached,
+ offline = hf_env_offline(),
+ ):
+ raise _hf_preflight_error(
+ 422,
+ f"hf_{label}_access_denied",
+ (
+ f"Hugging Face denied access to this cached {label}. "
+ "Add a token with repository access."
+ ),
+ )
+
+
def _preflight_hf_dataset_request(request: TrainingStartRequest) -> None:
dataset_id = request.hf_dataset
if not dataset_id:
@@ -723,7 +829,9 @@ def _authorize_cache_fallback(model_name: str, repo_type: str = "model") -> None
def _reject_untrainable_model_request(
- request: TrainingStartRequest, actual_model_repo_id: Optional[str] = None
+ request: TrainingStartRequest,
+ actual_model_repo_id: Optional[str] = None,
+ hf_token: HfTokenArg = None,
) -> _ModelPreflightResult:
model_format = (request.model_format or "").strip().lower()
if model_format == "gguf":
@@ -759,6 +867,24 @@ def _reject_untrainable_model_request(
if request.model_local_path == request.model_name
else normalize_path(request.model_local_path)
)
+ from hub.utils.hf_cache_state import cached_repo_ref_for_path
+
+ # A snapshot path inside the Hub cache is still that repository's cached weights. Any repo
+ # type counts: a private dataset or space snapshot can hold config.json plus weights, and
+ # is just as much the operator's to grant as a model is.
+ cached_ref = cached_repo_ref_for_path(path)
+ if cached_ref is not None and cached_read_refused(
+ hf_token,
+ repo_id = cached_ref[0],
+ repo_type = cached_ref[1],
+ is_cached = lambda: True,
+ offline = hf_env_offline(),
+ ):
+ raise _hf_preflight_error(
+ 422,
+ "hf_model_access_denied",
+ "Hugging Face denied access to this cached model. Add a token with repository access.",
+ )
else:
model_local_path = (
normalize_path(request.model_local_path) if request.model_local_path else None
@@ -792,6 +918,40 @@ def _reject_untrainable_model_request(
canonical_model_repo_id(request.model_name),
snapshot,
)
+ from hub.utils.hf_cache_state import repo_cache_has_usable_snapshot
+ from utils.security import load_scan_target
+
+ authorization_repo, _ = load_scan_target(
+ canonical_model_repo_id(actual_model_repo_id or request.model_name), ()
+ )
+
+ def has_cached_model():
+ if snapshot:
+ return True
+ # A repo DIRECTORY is not a cached read: an interrupted download leaves one with no
+ # snapshot under it, and refusing on that denies a public model over bytes that were
+ # never there. An unreadable root still counts, so the guard stays fail-closed.
+ return repo_cache_has_usable_snapshot(
+ "model",
+ authorization_repo,
+ with_load_subdirs(request.model_name, ("config.json", "adapter_config.json")),
+ )
+
+ def refuse_unauthorized_cache(is_cached):
+ # HF can reuse cached weights even when remote metadata/HEAD denies access.
+ if cached_read_refused(
+ hf_token,
+ repo_id = authorization_repo,
+ is_cached = is_cached,
+ offline = offline_mode,
+ ):
+ raise _hf_preflight_error(
+ 422,
+ "hf_model_access_denied",
+ "Hugging Face denied access to this cached model. Add a token with repository access.",
+ )
+
+ refuse_unauthorized_cache(has_cached_model)
if path is None and offline_mode:
raise _hf_preflight_error(
409,
@@ -816,10 +976,7 @@ def _reject_untrainable_model_request(
"Retry before starting training."
),
)
- remote_format = _remote_untrainable_model_format(
- request.model_name,
- request.hf_token or None,
- )
+ remote_format = _remote_untrainable_model_format(request.model_name, hf_token)
except HTTPException as error:
metadata_error = error
from core.training.training import _resolve_model_snapshot
@@ -830,6 +987,10 @@ def _reject_untrainable_model_request(
)
if snapshot is None:
raise
+ # The snapshot can land while the metadata probe is in flight.
+ refuse_unauthorized_cache(lambda: True)
+ # Independent axes, so both: the check above asks whether this CALLER's Hugging Face
+ # token reaches the repo, this one whether a managed account was granted the model.
_authorize_cache_fallback(request.model_name)
path = Path(snapshot)
cached_model_pin = (
@@ -1430,10 +1591,29 @@ async def start_training(
"dataset cache; disable streaming to train from the cached copy."
),
)
+ allow_ambient = via_api_key is not True
+ hf_token = hf_token_arg(request.hf_token, allow_ambient_token = allow_ambient)
+ # The local dataset paths were resolved above for existence only; authorize them now that
+ # the caller's token is known, before anything reads them.
+ # Eval paths only while evaluation is on, matching the validation above and the trainer,
+ # which reads them under the same condition: a stale eval path nothing will open must not
+ # refuse an otherwise valid run.
+ for _local_paths, _label in (
+ (request.local_datasets, "dataset"),
+ (
+ request.local_eval_datasets if evaluation_enabled(request.eval_steps) else [],
+ "eval_dataset",
+ ),
+ ):
+ if _local_paths:
+ await asyncio.to_thread(
+ _refuse_unauthorized_cached_local_paths, _local_paths, hf_token, _label
+ )
model_preflight = await asyncio.to_thread(
_reject_untrainable_model_request,
request,
resume_actual_model_repo_id,
+ hf_token,
)
cached_model_pin = model_preflight.cached_model_pin
training_actual_model_repo_id = resume_actual_model_repo_id
@@ -1443,12 +1623,24 @@ async def start_training(
if request.hf_dataset:
await asyncio.to_thread(_preflight_hf_dataset_request, request)
+ # After the preflight: a cache it pinned is still on disk for this scan to refuse.
+ # Not for a streaming start: it reads the Hub (load_dataset(streaming = True)) and never
+ # the materialized cache, the preflight above always verified the repo remotely, and
+ # the route already refuses streaming with a pinned cache. Asking anyway would refuse a
+ # valid start over an unrelated cached copy whenever /auth-check cannot be reached.
+ # And only when the Hub repo is the EFFECTIVE source: load_and_format_dataset takes
+ # local_datasets (and s3_config below it) ahead of dataset_source, so a payload carrying
+ # both never reads the Hub cache, and refusing over it fails a run on a dataset it does
+ # not use. The local paths took their own authorization pass above.
+ if not request.dataset_streaming and _hf_dataset_is_the_source(request):
+ await asyncio.to_thread(_refuse_unauthorized_cached_dataset, request, hf_token)
training_kwargs = {
"model_name": model_preflight.model_name,
"project_name": request.project_name,
"training_type": request.training_type,
- "hf_token": request.hf_token or "",
+ "hf_token": (request.hf_token or "").strip(),
+ "allow_ambient": allow_ambient,
"load_in_4bit": request.load_in_4bit,
"max_seq_length": request.max_seq_length,
"vision_image_size": request.vision_image_size,
@@ -1550,7 +1742,7 @@ async def start_training(
effective_training_load_in_4bit,
training_kwargs,
model_load_target,
- training_kwargs["hf_token"] or None,
+ hf_token,
)
except ExactResumeResourcesUnavailable as exc:
raise HTTPException(status_code = 409, detail = str(exc))
@@ -1569,9 +1761,7 @@ async def start_training(
model_defaults = load_model_defaults(request.model_name)
yaml_trust = model_defaults.get("training", {}).get("trust_remote_code", False)
- if yaml_trust and is_trusted_org_repo(
- request.model_name, hf_token = request.hf_token or None
- ):
+ if yaml_trust and is_trusted_org_repo(request.model_name, hf_token = hf_token):
logger.info(f"YAML config sets trust_remote_code=True for {request.model_name}")
training_kwargs["trust_remote_code"] = True
elif yaml_trust:
@@ -1639,7 +1829,7 @@ async def start_training(
def _can_keep_resident_models():
return can_keep_chat_during_training(
model_name = training_kwargs["model_name"],
- hf_token = training_kwargs["hf_token"],
+ hf_token = hf_token,
training_type = training_kwargs["training_type"],
load_in_4bit = training_kwargs["load_in_4bit"],
batch_size = training_kwargs["batch_size"],
@@ -2782,6 +2972,10 @@ async def start_diffusion_training(
# Resolve + contain the dataset and output paths BEFORE spawning: the trainer subprocess would otherwise resolve
# them relative to its own cwd.
config = body.model_dump()
+ # Same rule the LLM start applies: an API key that sent no token of its own must not reach the
+ # operator's saved Hugging Face login. Diffusion is training too, and its child resolved a
+ # bare `None` token against the server's login for the base-model fetch.
+ config["allow_ambient"] = via_api_key is not True
try:
from utils.paths import outputs_root, resolve_output_dir
@@ -2929,6 +3123,21 @@ async def start_diffusion_training(
normalized_cfg.hf_token,
)
+ # _preflight_gated_base skips a local path, and _assert_trusted_base_model accepts one as long
+ # as it is a real pipeline directory, so a base ALREADY in the operator's cache reached the
+ # trainer unchecked. Scrubbing the child environment does not help there: from_pretrained reads
+ # those files off disk and asks for no credential. Same authorization the LLM start applies.
+ # The EFFECTIVE target only, the same value the gated preflight above is given. The DiT trainer
+ # loads it (diffusion_dit_trainer replaces base_model with fetch_base_model) and for SDXL
+ # normalization makes the two equal, so the original base_model is metadata once a mirror is
+ # chosen: authorizing it as well can refuse a public mirror over a stray upstream snapshot.
+ await asyncio.to_thread(
+ _refuse_unauthorized_cached_local_paths,
+ [normalized_cfg.fetch_base_model or normalized_cfg.base_model or ""],
+ hf_token_arg(normalized_cfg.hf_token, allow_ambient_token = config["allow_ambient"]),
+ "model",
+ )
+
from core.training import diffusion_train_common as _dtc
service = get_diffusion_training_service()
diff --git a/studio/backend/routes/training_vram.py b/studio/backend/routes/training_vram.py
index 9580c96a804..1594e30b558 100644
--- a/studio/backend/routes/training_vram.py
+++ b/studio/backend/routes/training_vram.py
@@ -7,6 +7,7 @@ models when they fit. STT is evicted before chat when training needs memory.
from typing import Any, Callable, Dict, List, Optional, Tuple
+from hub.utils.hf_tokens import HfTokenArg, normalize_token
from loggers import get_logger
logger = get_logger(__name__)
@@ -117,7 +118,7 @@ def summarize_resident_stt() -> Dict[str, Any]:
def can_keep_chat_during_training(
*,
model_name: str,
- hf_token: Optional[str],
+ hf_token: HfTokenArg,
training_type: str,
load_in_4bit: bool,
batch_size: int,
@@ -146,10 +147,9 @@ def can_keep_chat_during_training(
# Full finetuning runs in 16-bit, so ignore the 4-bit request or we under-count.
effective_4bit = False if training_type == "Full Finetuning" else load_in_4bit
- hf_token_arg = hf_token or None
est_kwargs = dict(
- hf_token = hf_token_arg,
+ hf_token = normalize_token(hf_token),
training_type = training_type,
load_in_4bit = effective_4bit,
batch_size = batch_size,
diff --git a/studio/backend/tests/test_training_ambient_hf_token.py b/studio/backend/tests/test_training_ambient_hf_token.py
new file mode 100644
index 00000000000..9bcb5db97b3
--- /dev/null
+++ b/studio/backend/tests/test_training_ambient_hf_token.py
@@ -0,0 +1,515 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import asyncio
+import contextlib
+import json
+import os
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+import pytest
+
+import routes.training as tr
+from models import TrainingStartRequest
+
+_BACKEND = Path(__file__).resolve().parent.parent
+
+_HF_TOKEN_ENV = (
+ "HF_TOKEN",
+ "HF_HUB_TOKEN",
+ "HUGGING_FACE_HUB_TOKEN",
+ "HUGGINGFACE_HUB_TOKEN",
+ "HUGGINGFACEHUB_API_TOKEN",
+ "HF_OIDC_RESOURCE",
+ "HF_TOKEN_PATH",
+ "HF_HUB_DISABLE_IMPLICIT_TOKEN",
+)
+
+
+class _Backend:
+ current_job_id = None
+
+ def __init__(self):
+ self.kwargs = None
+
+ def is_training_active(self):
+ return False
+
+ def start_training(
+ self,
+ job_id,
+ *,
+ before_spawn = None,
+ **kwargs,
+ ):
+ self.kwargs = kwargs
+ self.current_job_id = job_id
+ return True
+
+
+ "via_api_key,request_token,probe_token,allow_ambient",
+ [
+ (True, None, False, False),
+ (True, "hf_caller", "hf_caller", False),
+ (True, " hf_caller\n", "hf_caller", False),
+ (False, None, None, True),
+ (False, "hf_caller", "hf_caller", True),
+ (False, " hf_caller\n", "hf_caller", True),
+ ],
+)
+def test_start_gives_the_model_preflight_only_the_callers_token(
+ monkeypatch, via_api_key, request_token, probe_token, allow_ambient
+):
+ backend = _Backend()
+ probed = []
+
+ def _probe(model_name, hf_token):
+ probed.append(hf_token)
+ return None
+
+ monkeypatch.setattr(tr, "get_training_backend", lambda: backend)
+ monkeypatch.setattr(tr, "_diffusion_training_active", lambda: False)
+ monkeypatch.setattr(tr, "_diffusion_gpu_admission", contextlib.nullcontext)
+ monkeypatch.setattr(tr, "hf_env_offline", lambda: False)
+ monkeypatch.setattr(tr, "_hub_unreachable", lambda: False)
+ monkeypatch.setattr(tr, "_remote_untrainable_model_format", _probe)
+ monkeypatch.setattr("utils.hardware.ensure_hardware_detected", lambda: None)
+
+ request = TrainingStartRequest(
+ model_name = "unsloth/tiny-model",
+ training_type = "LoRA/QLoRA",
+ format_type = "alpaca",
+ hf_token = request_token,
+ load_in_4bit = False,
+ trust_remote_code = True,
+ )
+ response = asyncio.run(
+ tr.start_training(request = request, current_subject = "alice", via_api_key = via_api_key)
+ )
+
+ assert response.status == "queued", response
+ assert probed == [probe_token]
+ assert backend.kwargs["allow_ambient"] is allow_ambient
+ assert backend.kwargs["hf_token"] == (probe_token or "")
+
+
[email protected]("offline", [False, True])
+ "via_api_key,request_token,authorized,refused",
+ [
+ (True, None, False, True),
+ (True, "hf_no_access", False, True),
+ (True, "hf_caller", True, False),
+ (False, None, False, False),
+ ],
+)
+def test_private_cached_dataset_requires_caller_authorization(
+ monkeypatch, tmp_path, offline, via_api_key, request_token, authorized, refused
+):
+ from fastapi import HTTPException
+ from hub.utils import dataset_cache, hf_tokens
+
+ backend = _Backend()
+ monkeypatch.setattr(tr, "get_training_backend", lambda: backend)
+ monkeypatch.setattr(tr, "_diffusion_training_active", lambda: False)
+ monkeypatch.setattr(tr, "_diffusion_gpu_admission", contextlib.nullcontext)
+ monkeypatch.setattr(tr, "hf_env_offline", lambda: offline)
+ monkeypatch.setattr(tr, "_hub_unreachable", lambda: False)
+ monkeypatch.setattr(
+ tr,
+ "_reject_untrainable_model_request",
+ lambda request, *a: tr._ModelPreflightResult(request.model_name, None, None),
+ )
+ monkeypatch.setattr("utils.hardware.ensure_hardware_detected", lambda: None)
+ monkeypatch.setattr(dataset_cache, "dataset_cache_can_answer", lambda repo_id: True)
+ monkeypatch.setattr(
+ dataset_cache, "training_dataset_cache_pin", lambda *a, **k: (str(tmp_path), "rev")
+ )
+ monkeypatch.setattr(hf_tokens, "_explicit_token_reaches_repo", lambda *a, **k: authorized)
+
+ request = TrainingStartRequest(
+ model_name = "unsloth/tiny-model",
+ training_type = "LoRA/QLoRA",
+ format_type = "alpaca",
+ hf_token = request_token,
+ hf_dataset = "org/private-dataset",
+ dataset_known_cached = True,
+ load_in_4bit = False,
+ )
+ start = tr.start_training(request = request, current_subject = "alice", via_api_key = via_api_key)
+ if refused:
+ with pytest.raises(HTTPException) as error:
+ asyncio.run(start)
+ assert error.value.detail["code"] == "hf_dataset_access_denied"
+ assert backend.kwargs is None
+ else:
+ assert asyncio.run(start).status == "queued"
+
+
[email protected]("token,refused", [(False, True), ("hf_no_access", True), (None, False)])
+def test_snapshot_cached_during_the_metadata_probe_is_authorized(
+ monkeypatch, tmp_path, token, refused
+):
+ from fastapi import HTTPException
+ from core.training import training as training_module
+ from hub.utils import hf_tokens
+
+ (tmp_path / "config.json").write_text('{"model_type":"llama"}')
+ (tmp_path / "model.safetensors").write_bytes(b"cached weights")
+ monkeypatch.setattr("hub.utils.hf_cache_state.iter_repo_cache_dirs", lambda *a, **k: iter(()))
+ monkeypatch.setattr(training_module, "_resolve_model_snapshot", lambda *a, **k: str(tmp_path))
+ monkeypatch.setattr(tr, "hf_env_offline", lambda: False)
+ monkeypatch.setattr(tr, "_hub_unreachable", lambda: False)
+ monkeypatch.setattr(hf_tokens, "_explicit_token_reaches_repo", lambda *a, **k: False)
+
+ def denied(*args):
+ raise tr._hf_preflight_error(422, "hf_model_access_denied", "Denied")
+
+ monkeypatch.setattr(tr, "_remote_untrainable_model_format", denied)
+ request = TrainingStartRequest(
+ model_name = "org/private-model",
+ training_type = "LoRA/QLoRA",
+ format_type = "alpaca",
+ )
+ if refused:
+ with pytest.raises(HTTPException) as error:
+ tr._reject_untrainable_model_request(request, hf_token = token)
+ assert error.value.detail["code"] == "hf_model_access_denied"
+ else:
+ result = tr._reject_untrainable_model_request(request, hf_token = token)
+ assert result.cached_model_pin == ("org/private-model", str(tmp_path))
+
+
+def test_dataset_cached_after_the_first_scan_is_not_pinned(monkeypatch, tmp_path):
+ from fastapi import HTTPException
+ from hub.utils import dataset_cache, hf_tokens
+
+ backend = _Backend()
+ scans = []
+
+ def appears_after_first_scan(*args, **kwargs):
+ scans.append(args)
+ return (str(tmp_path), "rev") if len(scans) > 1 else (None, None)
+
+ monkeypatch.setattr(tr, "get_training_backend", lambda: backend)
+ monkeypatch.setattr(tr, "_diffusion_training_active", lambda: False)
+ monkeypatch.setattr(tr, "_diffusion_gpu_admission", contextlib.nullcontext)
+ monkeypatch.setattr(tr, "hf_env_offline", lambda: False)
+ monkeypatch.setattr(tr, "_hub_unreachable", lambda: True)
+ monkeypatch.setattr(
+ tr,
+ "_reject_untrainable_model_request",
+ lambda request, *a: tr._ModelPreflightResult(request.model_name, None, None),
+ )
+ monkeypatch.setattr("utils.hardware.ensure_hardware_detected", lambda: None)
+ monkeypatch.setattr(dataset_cache, "dataset_cache_can_answer", lambda repo_id: bool(scans))
+ monkeypatch.setattr(dataset_cache, "training_dataset_cache_pin", appears_after_first_scan)
+ monkeypatch.setattr(hf_tokens, "_explicit_token_reaches_repo", lambda *a, **k: False)
+
+ request = TrainingStartRequest(
+ model_name = "unsloth/tiny-model",
+ training_type = "LoRA/QLoRA",
+ format_type = "alpaca",
+ hf_dataset = "org/private-dataset",
+ dataset_known_cached = True,
+ load_in_4bit = False,
+ )
+ with pytest.raises(HTTPException):
+ asyncio.run(tr.start_training(request = request, current_subject = "alice", via_api_key = True))
+ assert backend.kwargs is None
+
+
+ "inside_cache,token,refused",
+ [
+ (True, False, True),
+ (True, "hf_no_access", True),
+ (True, None, False),
+ (False, False, False),
+ ],
+)
+def test_local_snapshot_path_in_the_hub_cache_requires_caller_authorization(
+ monkeypatch, tmp_path, inside_cache, token, refused
+):
+ from fastapi import HTTPException
+ from hub.utils import hf_cache_state, hf_tokens
+
+ root = tmp_path / "hub"
+ parent = (
+ root / "models--org--private-model" / "snapshots" if inside_cache else tmp_path / "local"
+ )
+ snapshot = parent / "0123456789abcdef0123456789abcdef01234567"
+ snapshot.mkdir(parents = True)
+ (snapshot / "config.json").write_text('{"model_type":"llama"}')
+ (snapshot / "model.safetensors").write_bytes(b"cached weights")
+ monkeypatch.setattr(hf_cache_state, "hf_cache_roots", lambda scan_errors = None: [root])
+ monkeypatch.setattr(tr, "hf_env_offline", lambda: False)
+ monkeypatch.setattr(hf_tokens, "_explicit_token_reaches_repo", lambda *a, **k: False)
+ root.mkdir(exist_ok = True)
+
+ request = TrainingStartRequest(
+ model_name = str(snapshot),
+ training_type = "LoRA/QLoRA",
+ format_type = "alpaca",
+ )
+ if refused:
+ with pytest.raises(HTTPException) as error:
+ tr._reject_untrainable_model_request(request, hf_token = token)
+ assert error.value.detail["code"] == "hf_model_access_denied"
+ else:
+ result = tr._reject_untrainable_model_request(request, hf_token = token)
+ assert result.model_name == str(snapshot.resolve())
+
+
+def test_cached_repo_id_for_path(monkeypatch, tmp_path):
+ from hub.utils import hf_cache_state
+
+ root = tmp_path / "hub"
+ snapshot = root / "models--Org--Private-Model" / "snapshots" / "abc"
+ snapshot.mkdir(parents = True)
+ outside = tmp_path / "models--org--lookalike" / "snapshots" / "abc"
+ outside.mkdir(parents = True)
+ monkeypatch.setattr(hf_cache_state, "hf_cache_roots", lambda scan_errors = None: [root])
+
+ assert hf_cache_state.cached_repo_id_for_path(snapshot) == "Org/Private-Model"
+ assert (
+ hf_cache_state.cached_repo_id_for_path(root / "models--Org--Private-Model")
+ == "Org/Private-Model"
+ )
+ assert hf_cache_state.cached_repo_id_for_path(outside) is None
+ assert hf_cache_state.cached_repo_id_for_path(root) is None
+
+
+def test_worker_config_carries_the_ambient_policy():
+ from core.training.training import _build_training_worker_config
+
+ assert _build_training_worker_config({"model_name": "org/model"})["allow_ambient"] is True
+ config = _build_training_worker_config({"model_name": "org/model", "allow_ambient": False})
+ assert config["allow_ambient"] is False
+
+
+_WORKER_PROBE = textwrap.dedent(
+ """
+ import json
+ import queue
+ import sys
+
+ from utils.native_path_leases import run_without_native_path_secret
+
+ run_without_native_path_secret(
+ "core.training.worker",
+ "run_training_process",
+ {},
+ event_queue = queue.Queue(),
+ stop_queue = queue.Queue(),
+ config = json.loads(sys.argv[1]),
+ )
+
+ from huggingface_hub import get_token
+ from huggingface_hub.utils import build_hf_headers
+
+ print(json.dumps({"token": get_token(), "header": build_hf_headers().get("authorization")}))
+ """
+)
+
+
+ "allow_ambient,hf_token,expected_token",
+ [
+ (False, "", None),
+ (False, "hf_caller", "hf_caller"),
+ (True, "", "hf_operator_env"),
+ ],
+)
+def test_training_worker_holds_only_the_callers_credential(
+ tmp_path, allow_ambient, hf_token, expected_token
+):
+ (tmp_path / "token").write_text("hf_operator_file")
+ env = {key: value for key, value in os.environ.items() if key not in _HF_TOKEN_ENV}
+ env.update(
+ HF_HOME = str(tmp_path),
+ HF_TOKEN = "hf_operator_env",
+ HUGGING_FACE_HUB_TOKEN = "hf_operator_env",
+ HF_HUB_OFFLINE = "1",
+ PYTHONPATH = os.pathsep.join(filter(None, (str(_BACKEND), env.get("PYTHONPATH")))),
+ )
+ config = {
+ "model_name": "org/model",
+ "model_format": "gguf",
+ "hf_token": hf_token,
+ "allow_ambient": allow_ambient,
+ }
+
+ out = subprocess.run(
+ [sys.executable, "-c", _WORKER_PROBE, json.dumps(config)],
+ cwd = str(_BACKEND),
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 300,
+ )
+
+ assert out.returncode == 0, out.stderr[-2000:]
+ seen = json.loads(out.stdout.strip().splitlines()[-1])
+ assert seen["token"] == expected_token
+ assert seen["header"] == (f"Bearer {expected_token}" if expected_token else None)
+
+
+ "allow_ambient,token,expected",
+ [
+ (False, "", None),
+ (False, "hf_caller", "Bearer hf_caller"),
+ (True, "", "Bearer hf_operator_probe"),
+ ],
+)
+def test_parent_gpu_probe_uses_only_authorized_token(monkeypatch, allow_ambient, token, expected):
+ from core.training import training as training_module
+ from utils.hardware import hardware
+ from huggingface_hub import hf_api
+
+ class Captured(BaseException):
+ pass
+
+ seen = []
+
+ class Session:
+ def get(self, url, **kwargs):
+ seen.append(kwargs.get("headers", {}).get("authorization"))
+ raise Captured
+
+ monkeypatch.setenv("HF_TOKEN", "hf_operator_probe")
+ monkeypatch.setenv("HF_HUB_DISABLE_IMPLICIT_TOKEN", "0")
+ monkeypatch.setattr(training_module, "should_use_mlx_training_backend", lambda **kwargs: False)
+ monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)
+ monkeypatch.setattr(
+ hardware, "_resolve_model_identifier_for_gpu_estimate", lambda name, hf_token = None: name
+ )
+ monkeypatch.setattr(hf_api, "get_session", lambda: Session())
+ backend = training_module.TrainingBackend()
+ with pytest.raises(Captured):
+ backend.start_training(
+ "credential-probe",
+ model_name = "org/public-model",
+ training_type = "LoRA/QLoRA",
+ hf_token = token,
+ allow_ambient = allow_ambient,
+ load_in_4bit = False,
+ )
+ assert seen == [expected]
+
+
+ "cached_path", ["fallback", "known", "offline", "resume", "public_metadata"]
+)
[email protected]("token", [False, "hf_no_access"])
+def test_private_cached_model_requires_caller_authorization(
+ monkeypatch, tmp_path, cached_path, token
+):
+ from fastapi import HTTPException
+ from core.training import training as training_module
+ from hub.utils import hf_tokens
+
+ (tmp_path / "config.json").write_text('{"model_type":"llama"}')
+ (tmp_path / "model.safetensors").write_bytes(b"cached weights")
+ # Also as a real cache repo lays it out. has_cached_model asks for a snapshot the load could
+ # consume rather than for the repo directory alone, so a bare directory is not evidence.
+ _revision = tmp_path / "snapshots" / "abc"
+ _revision.mkdir(parents = True, exist_ok = True)
+ (_revision / "config.json").write_text('{"model_type":"llama"}')
+ (_revision / "model.safetensors").write_bytes(b"cached weights")
+ monkeypatch.setattr(training_module, "_resolve_model_snapshot", lambda *a, **k: str(tmp_path))
+ monkeypatch.setattr(
+ "hub.utils.hf_cache_state.iter_repo_cache_dirs", lambda *a, **k: iter([tmp_path])
+ )
+ monkeypatch.setattr(
+ "hub.utils.hf_cache_state.latest_snapshot_from_cache_path", lambda *a, **k: str(tmp_path)
+ )
+ monkeypatch.setattr(tr, "hf_env_offline", lambda: cached_path == "offline")
+ monkeypatch.setattr(tr, "_hub_unreachable", lambda: False)
+ monkeypatch.setattr(hf_tokens, "_explicit_token_reaches_repo", lambda *a, **k: False)
+
+ def denied(*args):
+ raise tr._hf_preflight_error(422, "hf_model_access_denied", "Denied")
+
+ monkeypatch.setattr(
+ tr,
+ "_remote_untrainable_model_format",
+ (lambda *a: None) if cached_path == "public_metadata" else denied,
+ )
+ request = TrainingStartRequest(
+ model_name = "org/private-model",
+ training_type = "LoRA/QLoRA",
+ format_type = "alpaca",
+ model_known_cached = cached_path == "known",
+ resume_from_checkpoint = str(tmp_path) if cached_path == "resume" else None,
+ model_snapshot_path = str(tmp_path) if cached_path == "resume" else None,
+ )
+ with pytest.raises(HTTPException) as error:
+ tr._reject_untrainable_model_request(request, hf_token = token)
+ assert error.value.detail["code"] == "hf_model_access_denied"
+
+
[email protected]("token,authorized", [(None, False), (False, True), ("hf_caller", True)])
+def test_authorized_cached_models_remain_available(monkeypatch, tmp_path, token, authorized):
+ from core.training import training as training_module
+ from hub.utils import hf_tokens
+
+ (tmp_path / "config.json").write_text('{"model_type":"llama"}')
+ (tmp_path / "model.safetensors").write_bytes(b"cached weights")
+ # Also as a real cache repo lays it out. has_cached_model asks for a snapshot the load could
+ # consume rather than for the repo directory alone, so a bare directory is not evidence.
+ _revision = tmp_path / "snapshots" / "abc"
+ _revision.mkdir(parents = True, exist_ok = True)
+ (_revision / "config.json").write_text('{"model_type":"llama"}')
+ (_revision / "model.safetensors").write_bytes(b"cached weights")
+ monkeypatch.setattr(training_module, "_resolve_model_snapshot", lambda *a, **k: str(tmp_path))
+ monkeypatch.setattr(
+ "hub.utils.hf_cache_state.iter_repo_cache_dirs", lambda *a, **k: iter([tmp_path])
+ )
+ monkeypatch.setattr(tr, "hf_env_offline", lambda: False)
+ monkeypatch.setattr(hf_tokens, "_explicit_token_reaches_repo", lambda *a, **k: authorized)
+ request = TrainingStartRequest(
+ model_name = "org/model",
+ model_known_cached = True,
+ training_type = "LoRA/QLoRA",
+ format_type = "alpaca",
+ )
+ result = tr._reject_untrainable_model_request(request, hf_token = token)
+ assert result.model_name == "org/model"
+
+
+def test_chat_coexistence_preserves_anonymous_token(monkeypatch):
+ from routes.training_vram import can_keep_chat_during_training
+ from utils import hardware
+
+ class Captured(BaseException):
+ pass
+
+ seen = []
+
+ def select(*args, **kwargs):
+ seen.append(kwargs["hf_token"])
+ raise Captured
+
+ monkeypatch.setattr(hardware, "get_device", lambda: hardware.DeviceType.CUDA)
+ monkeypatch.setattr(hardware, "auto_select_gpu_ids", select)
+ with pytest.raises(Captured):
+ can_keep_chat_during_training(
+ model_name = "org/model",
+ hf_token = False,
+ training_type = "LoRA/QLoRA",
+ load_in_4bit = False,
+ batch_size = 1,
+ max_seq_length = 128,
+ lora_rank = 16,
+ target_modules = None,
+ gradient_checkpointing = "unsloth",
+ optimizer = "adamw_8bit",
+ gpu_ids = None,
+ )
+ assert seen == [False]
diff --git a/studio/backend/tests/test_training_cached_repo_authorization.py b/studio/backend/tests/test_training_cached_repo_authorization.py
new file mode 100644
index 00000000000..09e63ac8822
--- /dev/null
+++ b/studio/backend/tests/test_training_cached_repo_authorization.py
@@ -0,0 +1,650 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Cached-path authorization for training starts.
+
+Companion to test_training_ambient_hf_token.py, which covers the token plumbing. These cover the
+path-to-repository mapping the model and dataset legs authorize against, where a miss is fail-OPEN:
+``cached_repo_ref_for_path`` returning ``None`` means the caller skips the access check entirely.
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+BACKEND = Path(__file__).resolve().parent.parent
+if str(BACKEND) not in sys.path:
+ sys.path.insert(0, str(BACKEND))
+
+
+def _make_repo(
+ root: Path,
+ dir_name: str,
+ sha: str = "abc",
+) -> Path:
+ """A cache repo laid out as huggingface_hub writes it, snapshot symlinks included."""
+ repo = root / dir_name
+ snapshot = repo / "snapshots" / sha
+ blobs = repo / "blobs"
+ snapshot.mkdir(parents = True, exist_ok = True)
+ blobs.mkdir(parents = True, exist_ok = True)
+ for index, name in enumerate(("config.json", "model.safetensors")):
+ blob = blobs / f"blob{index}"
+ blob.write_text("{}")
+ link = snapshot / name
+ if not link.exists():
+ try:
+ link.symlink_to(Path("..") / ".." / "blobs" / f"blob{index}")
+ except (OSError, NotImplementedError):
+ # Windows without Developer Mode refuses symlinks; the real cache falls back to
+ # copies there, so a plain file is the faithful layout rather than a skip.
+ link.write_text("{}")
+ return snapshot
+
+
[email protected](autouse = True)
+def restore_environ():
+ """The child-side scrub writes os.environ directly, which monkeypatch cannot undo for keys it
+ never saw. Snapshot the whole mapping so one test's scrub is not the next one's premise."""
+ import os
+
+ saved = dict(os.environ)
+ yield
+ os.environ.clear()
+ os.environ.update(saved)
+
+
+def cache_root(monkeypatch, tmp_path):
+ from hub.utils import hf_cache_state
+
+ root = tmp_path / "hub"
+ root.mkdir()
+ monkeypatch.setattr(hf_cache_state, "hf_cache_roots", lambda scan_errors = None: [root])
+ return root
+
+
+def test_snapshot_symlink_into_blobs_still_names_its_repo(cache_root):
+ """resolve(strict=True) lands in blobs/, which is under the same repo dir."""
+ from hub.utils.hf_cache_state import cached_repo_ref_for_path
+
+ snapshot = _make_repo(cache_root, "models--Org--Private-Model")
+ assert cached_repo_ref_for_path(snapshot / "model.safetensors") == (
+ "Org/Private-Model",
+ "model",
+ )
+
+
+def test_cached_private_dataset_path_is_attributed_to_the_dataset_repo(cache_root):
+ """A dataset snapshot passed where a model is expected must still be authorized: such a repo
+ can hold config.json plus weights, and matching only "models--" skipped the check entirely.
+ """
+ from hub.utils.hf_cache_state import cached_repo_id_for_path, cached_repo_ref_for_path
+
+ snapshot = _make_repo(cache_root, "datasets--org--private-data")
+ assert cached_repo_ref_for_path(snapshot) == ("org/private-data", "dataset")
+ # The repo_type-restricted view still answers only for its own type.
+ assert cached_repo_id_for_path(snapshot) is None
+ assert cached_repo_id_for_path(snapshot, "dataset") == "org/private-data"
+
+
+def test_cached_space_path_is_attributed(cache_root):
+ from hub.utils.hf_cache_state import cached_repo_ref_for_path
+ snapshot = _make_repo(cache_root, "spaces--org--private-space")
+ assert cached_repo_ref_for_path(snapshot) == ("org/private-space", "space")
+
+
+def test_a_decoy_repo_dir_inside_a_snapshot_does_not_hide_the_real_repo(cache_root):
+ """Repo file paths are arbitrary, so a private snapshot may contain "models--foo--bar", and
+ committing to the DEEPEST match returned None for a path plainly inside models--org--private.
+ """
+ from hub.utils.hf_cache_state import cached_repo_ref_for_path
+
+ snapshot = _make_repo(cache_root, "models--org--private")
+ decoy = snapshot / "models--foo--bar"
+ decoy.mkdir()
+ (decoy / "config.json").write_text("{}")
+
+ assert cached_repo_ref_for_path(decoy) == ("org/private", "model")
+ assert cached_repo_ref_for_path(decoy / "config.json") == ("org/private", "model")
+
+
+def test_a_symlink_from_outside_cannot_launder_a_private_snapshot(cache_root, tmp_path):
+ """resolve(strict=True) runs first, so the link's target is what gets authorized."""
+ from hub.utils.hf_cache_state import cached_repo_ref_for_path
+
+ snapshot = _make_repo(cache_root, "models--org--private")
+ link = tmp_path / "looks_harmless"
+ try:
+ link.symlink_to(snapshot, target_is_directory = True)
+ except (OSError, NotImplementedError):
+ pytest.skip("this filesystem does not allow symlinks")
+ assert cached_repo_ref_for_path(link) == ("org/private", "model")
+
+
+def test_a_single_segment_repo_id_round_trips(cache_root):
+ """models--gpt2 has no separator at all."""
+ from hub.utils.hf_cache_state import cached_repo_ref_for_path
+ assert cached_repo_ref_for_path(_make_repo(cache_root, "models--gpt2")) == ("gpt2", "model")
+
+
+def test_a_repo_in_a_second_cache_root_is_attributed(monkeypatch, tmp_path):
+ """A box whose cache has moved keeps the old HF_HOME in the remembered roots."""
+ from hub.utils import hf_cache_state
+
+ first = tmp_path / "hub"
+ second = tmp_path / "hub2"
+ first.mkdir()
+ second.mkdir()
+ monkeypatch.setattr(
+ hf_cache_state,
+ "hf_cache_roots",
+ lambda scan_errors = None: [first, second],
+ )
+ snapshot = _make_repo(second, "models--other--secret")
+ assert hf_cache_state.cached_repo_ref_for_path(snapshot) == ("other/secret", "model")
+
+
+def test_a_prefix_collision_sibling_of_a_cache_root_is_not_attributed(monkeypatch, tmp_path):
+ """ "hub_evil" is not "hub": containment is decided by samefile, not by name."""
+ from hub.utils import hf_cache_state
+
+ root = tmp_path / "hub"
+ root.mkdir()
+ sibling = tmp_path / "hub_evil"
+ sibling.mkdir()
+ monkeypatch.setattr(hf_cache_state, "hf_cache_roots", lambda scan_errors = None: [root])
+ assert hf_cache_state.cached_repo_ref_for_path(_make_repo(sibling, "models--org--p")) is None
+
+
+def test_a_lookalike_outside_every_cache_root_is_not_attributed(cache_root, tmp_path):
+ """Not the operator's cache, so there is nothing of theirs to protect."""
+ from hub.utils.hf_cache_state import cached_repo_ref_for_path
+
+ outside = _make_repo(tmp_path / "elsewhere", "models--org--private")
+ assert cached_repo_ref_for_path(outside) is None
+ assert cached_repo_ref_for_path(cache_root) is None
+
+
+def test_an_unreadable_cache_root_authorizes_rather_than_skips(monkeypatch, tmp_path):
+ """Fail closed: the unscannable root may be the one this path belongs to."""
+ from hub.utils import hf_cache_state
+
+ missing = tmp_path / "gone"
+
+ def roots(scan_errors = None):
+ if scan_errors is not None:
+ scan_errors.append(OSError("unreadable"))
+ return [missing]
+
+ monkeypatch.setattr(hf_cache_state, "hf_cache_roots", roots)
+ snapshot = _make_repo(tmp_path / "elsewhere", "models--org--private")
+ assert hf_cache_state.cached_repo_ref_for_path(snapshot) == ("org/private", "model")
+
+
+def test_repo_ids_round_trip_through_the_cache_dir_name(cache_root):
+ """ "--" is forbidden inside a repo id, so the split is unambiguous.
+
+ Guards the mapping against a "fix" for a name that cannot exist: huggingface_hub's own
+ validate_repo_id rejects "org/my--model", and _scan_cached_repo parses the same way.
+ """
+ from huggingface_hub.utils import HFValidationError, validate_repo_id
+
+ from hub.utils.hf_cache_state import cached_repo_ref_for_path
+
+ for repo_id in ("org/private", "Org/Private-Model", "org/name_with_underscores"):
+ validate_repo_id(repo_id)
+ snapshot = _make_repo(cache_root, "models--" + repo_id.replace("/", "--"))
+ assert cached_repo_ref_for_path(snapshot) == (repo_id, "model")
+
+ with pytest.raises(HFValidationError):
+ validate_repo_id("org/my--model")
+
+
+def test_local_dataset_paths_in_the_cache_require_caller_authorization(monkeypatch, cache_root):
+ """local_datasets takes any readable path, including one inside the operator's cache."""
+ import routes.training as training_routes
+ from fastapi import HTTPException
+
+ snapshot = _make_repo(cache_root, "datasets--org--private-data")
+ target = str(snapshot / "config.json")
+
+ seen = {}
+
+ def refuse(
+ hf_token,
+ *,
+ repo_id,
+ is_cached,
+ repo_type = "model",
+ offline = False,
+ ):
+ seen["repo_id"] = repo_id
+ seen["repo_type"] = repo_type
+ assert is_cached() is True
+ return hf_token is False
+
+ monkeypatch.setattr(training_routes, "cached_read_refused", refuse)
+
+ # An API key with no token of its own: refused, and told which repository to get access to.
+ with pytest.raises(HTTPException) as error:
+ training_routes._refuse_unauthorized_cached_local_paths([target], False)
+ assert error.value.status_code == 422
+ assert seen == {"repo_id": "org/private-data", "repo_type": "dataset"}
+
+ # A UI session entitled to the saved login is unaffected.
+ training_routes._refuse_unauthorized_cached_local_paths([target], None)
+
+
+def test_local_dataset_paths_outside_the_cache_are_left_alone(monkeypatch, cache_root, tmp_path):
+ import routes.training as training_routes
+
+ plain = tmp_path / "my_data.jsonl"
+ plain.write_text("{}")
+
+ def refuse(*args, **kwargs): # pragma: no cover - must not be consulted
+ raise AssertionError("a path outside the Hub cache must not be authorized against a repo")
+
+ monkeypatch.setattr(training_routes, "cached_read_refused", refuse)
+ training_routes._refuse_unauthorized_cached_local_paths([str(plain)], False)
+
+
+def test_diffusion_child_drops_the_saved_login_for_a_tokenless_api_key(monkeypatch):
+ """The diffusion child is spawned, so the scrub must happen inside it, before any Hub import."""
+ from core.training import diffusion_training_service as service
+
+ for key in ("HF_TOKEN", "HF_HUB_TOKEN", "HUGGING_FACE_HUB_TOKEN"):
+ monkeypatch.setenv(key, "operator-saved-login")
+ monkeypatch.delenv("HF_TOKEN_PATH", raising = False)
+ monkeypatch.setattr(service, "_run_diffusion_child", lambda **kwargs: None)
+
+ service._default_target(event_queue = None, stop_queue = None, config = {"allow_ambient": False})
+
+ import os
+
+ assert "HF_TOKEN" not in os.environ
+ assert "HF_HUB_TOKEN" not in os.environ
+ assert "HUGGING_FACE_HUB_TOKEN" not in os.environ
+ assert os.environ["HF_TOKEN_PATH"] == os.devnull
+ assert os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] == "1"
+
+
+def test_diffusion_child_forwards_the_callers_own_token(monkeypatch):
+ from core.training import diffusion_training_service as service
+
+ monkeypatch.setenv("HF_TOKEN", "operator-saved-login")
+ monkeypatch.setattr(service, "_run_diffusion_child", lambda **kwargs: None)
+
+ service._default_target(
+ event_queue = None,
+ stop_queue = None,
+ config = {"allow_ambient": False, "hf_token": " caller-own-token "},
+ )
+
+ import os
+
+ # Trimmed, and the operator's credential is gone rather than sitting beside it.
+ assert os.environ["HF_TOKEN"] == "caller-own-token"
+ assert os.environ["HF_HUB_DISABLE_IMPLICIT_TOKEN"] == "0"
+
+
+def test_diffusion_child_leaves_a_studio_session_alone(monkeypatch):
+ """No allow_ambient, or True: the saved login is exactly what a UI start should use."""
+ from core.training import diffusion_training_service as service
+
+ monkeypatch.setenv("HF_TOKEN", "operator-saved-login")
+ monkeypatch.setattr(service, "_run_diffusion_child", lambda **kwargs: None)
+
+ import os
+
+ # conftest points HF_TOKEN_PATH at the isolated cache for every test, so the claim is that it
+ # is left where it was rather than redirected to devnull.
+ token_path_before = os.environ.get("HF_TOKEN_PATH")
+ # Both read as "unchanged", not as a literal value: another test in this worker may have left
+ # HF_HUB_DISABLE_IMPLICIT_TOKEN set, and the claim here is that this call does not set it.
+ implicit_before = os.environ.get("HF_HUB_DISABLE_IMPLICIT_TOKEN")
+ for config in ({}, {"allow_ambient": True}):
+ service._default_target(event_queue = None, stop_queue = None, config = config)
+ assert os.environ["HF_TOKEN"] == "operator-saved-login"
+ assert os.environ.get("HF_TOKEN_PATH") == token_path_before
+ assert os.environ.get("HF_HUB_DISABLE_IMPLICIT_TOKEN") == implicit_before
+
+
+def test_a_cached_diffusion_base_requires_caller_authorization(monkeypatch, cache_root):
+ """Scrubbing the child environment does not protect a base that is ALREADY cached.
+
+ _preflight_gated_base returns early for a local path and _assert_trusted_base_model accepts any
+ real pipeline directory, so from_pretrained read it off disk with no credential.
+ """
+ import routes.training as training_routes
+ from fastapi import HTTPException
+
+ snapshot = _make_repo(cache_root, "models--org--private-sdxl")
+ seen = {}
+
+ def refuse(
+ hf_token,
+ *,
+ repo_id,
+ is_cached,
+ repo_type = "model",
+ offline = False,
+ ):
+ seen["repo_id"] = repo_id
+ return hf_token is False and is_cached()
+
+ monkeypatch.setattr(training_routes, "cached_read_refused", refuse)
+
+ with pytest.raises(HTTPException) as error:
+ training_routes._refuse_unauthorized_cached_local_paths(
+ [str(snapshot), ""],
+ False,
+ "model",
+ )
+ assert error.value.status_code == 422
+ assert error.value.detail["code"] == "hf_model_access_denied"
+ assert seen["repo_id"] == "org/private-sdxl"
+
+ # A Hub id for a base this disk does not hold has nothing to disclose.
+ training_routes._refuse_unauthorized_cached_local_paths(
+ ["black-forest-labs/FLUX.2-klein-4B", ""],
+ False,
+ "model",
+ )
+
+
+def test_a_remote_named_cached_base_is_authorized_when_the_hub_probe_fails_open(
+ monkeypatch, cache_root
+):
+ """`org/private` names no path, and the HEAD that would cover it is best-effort.
+
+ `_preflight_gated_base` treats an unreachable Hub as "not a denial", so on an offline box the
+ scrubbed child still loaded the cached private copy. The disk is asked directly instead.
+ """
+ import routes.training as training_routes
+ from fastapi import HTTPException
+
+ _make_repo(cache_root, "models--org--private-sdxl")
+ seen = {}
+
+ def refuse(
+ hf_token,
+ *,
+ repo_id,
+ is_cached,
+ repo_type = "model",
+ offline = False,
+ ):
+ seen["repo_id"] = repo_id
+ seen["repo_type"] = repo_type
+ seen["is_cached"] = is_cached()
+ return hf_token is False and seen["is_cached"]
+
+ monkeypatch.setattr(training_routes, "cached_read_refused", refuse)
+
+ with pytest.raises(HTTPException) as error:
+ training_routes._refuse_unauthorized_cached_local_paths(
+ ["org/private-sdxl"],
+ False,
+ "model",
+ )
+ assert error.value.status_code == 422
+ assert seen == {"repo_id": "org/private-sdxl", "repo_type": "model", "is_cached": True}
+
+ # A repo that is NOT on this disk has nothing to leak, so it is not refused.
+ training_routes._refuse_unauthorized_cached_local_paths(
+ ["org/never-downloaded"],
+ False,
+ "model",
+ )
+ assert seen["is_cached"] is False
+
+ # Nor does an interrupted download that left the repo directory and nothing under it: a
+ # public base would otherwise be refused the download it was entitled to.
+ (cache_root / "models--org--interrupted-base").mkdir()
+ training_routes._refuse_unauthorized_cached_local_paths(
+ ["org/interrupted-base"],
+ False,
+ "model",
+ )
+ assert seen["is_cached"] is False
+
+ # A bare model name (no owner) is not a repo id; it must not be probed as one.
+ seen.clear()
+ training_routes._refuse_unauthorized_cached_local_paths(["gpt2", ""], False, "model")
+ assert seen == {}
+
+
+def test_the_worker_rebuilds_the_anonymous_sentinel_rather_than_none():
+ """`or None` handed an API key the AMBIENT caller class inside the worker.
+
+ The scrub covers network traffic only, and cache_reads_authorized(None) is True, so cached
+ private weights stayed readable for the repos the route never authorized: a LoRA checkpoint's
+ base, sibling scan targets, a fallback load target.
+ """
+ from core.training.worker import _worker_hf_token
+ from hub.utils.hf_tokens import AmbientAuthorizedToken, cache_reads_authorized, is_anonymous
+
+ # A tokenless API key: the sentinel, and it authorizes no cache read.
+ token = _worker_hf_token({"allow_ambient": False})
+ assert is_anonymous(token)
+ assert cache_reads_authorized(token, repo_id = "org/private") is False
+
+ # A tokenless UI session keeps the ambient login, exactly as before.
+ assert _worker_hf_token({"allow_ambient": True}) is None
+ assert _worker_hf_token({}) is None
+
+ # An API key's own token stays a plain str, so it is probed rather than trusted outright.
+ own = _worker_hf_token({"allow_ambient": False, "hf_token": " hf_caller "})
+ assert own == "hf_caller"
+ assert not isinstance(own, AmbientAuthorizedToken)
+
+ # A UI session's own token is entitled to ambient and must not be demoted by the trim.
+ ui = _worker_hf_token({"allow_ambient": True, "hf_token": " hf_ui "})
+ assert isinstance(ui, AmbientAuthorizedToken) and ui == "hf_ui"
+ assert cache_reads_authorized(ui, repo_id = "org/private") is True
+
+
+def test_the_worker_no_longer_launders_the_sentinel_through_or_none():
+ """A guard against the tempting `config.get("hf_token") or None` simplification coming back."""
+ import inspect
+
+ from core.training import worker
+
+ source = inspect.getsource(worker)
+ assert 'config.get("hf_token") or None' not in source
+
+
+def test_an_interrupted_download_is_not_evidence_of_a_cached_read(monkeypatch, cache_root):
+ """A repo DIRECTORY with no usable snapshot has disclosed nothing, so refusing it protects
+ nothing. Reachable because the anonymous rescue needs /auth-check, which an HF_ENDPOINT mirror
+ need not serve: there a PUBLIC model is rejected for bytes that were never on disk.
+ """
+ from hub.utils.hf_cache_state import repo_cache_has_usable_snapshot
+
+ metadata = ("config.json", "adapter_config.json")
+
+ def usable(repo_id: str) -> bool:
+ return repo_cache_has_usable_snapshot("model", repo_id, metadata)
+
+ # Interrupted: the repo dir exists, nothing under it does.
+ (cache_root / "models--org--interrupted").mkdir()
+ assert usable("org/interrupted") is False
+
+ # Present but empty: snapshots/<rev> with no metadata for the load to consume.
+ (cache_root / "models--org--partial" / "snapshots" / "abc").mkdir(parents = True)
+ assert usable("org/partial") is False
+
+ # A real one still counts, so the guard has not been blunted.
+ snapshot = _make_repo(cache_root, "models--org--usable")
+ assert (snapshot / "config.json").exists()
+ assert usable("org/usable") is True
+
+ # Never downloaded at all: nothing to disclose and nothing to refuse.
+ assert usable("org/absent") is False
+
+
+def test_an_unreadable_repo_directory_still_counts_as_cached(monkeypatch, cache_root):
+ """The predicate answers a guard, so its own failure must not open what it guards."""
+ import os
+
+ from hub.utils.hf_cache_state import repo_cache_has_usable_snapshot
+
+ repo = cache_root / "models--org--locked"
+ (repo / "snapshots").mkdir(parents = True)
+ os.chmod(repo / "snapshots", 0o000)
+ try:
+ if os.access(repo / "snapshots", os.R_OK):
+ pytest.skip("running as a user that ignores directory permissions")
+ assert repo_cache_has_usable_snapshot("model", "org/locked", ("config.json",)) is True
+ finally:
+ os.chmod(repo / "snapshots", 0o755)
+
+
+def test_a_disabled_eval_path_is_not_authorized(monkeypatch, cache_root):
+ """The route validates local_eval_datasets only while evaluation is on, and so does the
+ trainer, so a stale eval path nothing will ever open must not refuse the run."""
+ import inspect
+
+ import routes.training as training_routes
+
+ source = inspect.getsource(training_routes.start_training)
+ marker = "request.local_eval_datasets if evaluation_enabled(request.eval_steps) else []"
+ assert marker in source
+
+ # And the condition itself agrees with the validation above it.
+ assert training_routes.evaluation_enabled(0) is False
+ assert training_routes.evaluation_enabled(None) is False
+ assert training_routes.evaluation_enabled(10) is True
+
+
+def test_only_the_effective_diffusion_fetch_target_is_authorized():
+ """The DiT trainer loads fetch_base_model, and for SDXL the two are equal by construction.
+
+ Authorizing the original base_model as well refused a public mirror whenever a stray or partial
+ snapshot of the gated upstream happened to sit in the cache.
+ """
+ import inspect
+
+ import routes.training as training_routes
+
+ source = inspect.getsource(training_routes.start_diffusion_training)
+ assert '[normalized_cfg.fetch_base_model or normalized_cfg.base_model or ""]' in source
+ # The gated preflight above resolves the same target, so the two cannot drift apart.
+ assert "normalized_cfg.fetch_base_model or normalized_cfg.base_model" in source
+
+
+def test_a_metadata_less_probe_needs_content_not_just_a_revision_dir(cache_root):
+ """The diffusion preflight probes without a metadata name, since it varies by family, so an
+ empty snapshots/<rev>/ left by an interrupted download must not read as usable."""
+ from hub.utils.hf_cache_state import repo_cache_has_usable_snapshot
+
+ (cache_root / "models--org--empty-rev" / "snapshots" / "abc").mkdir(parents = True)
+ assert repo_cache_has_usable_snapshot("model", "org/empty-rev") is False
+
+ # Weights in a per-component subdirectory, as diffusers lays a pipeline out, do count.
+ unet = cache_root / "models--org--pipeline" / "snapshots" / "abc" / "unet"
+ unet.mkdir(parents = True)
+ (unet / "diffusion_pytorch_model.safetensors").write_text("w")
+ assert repo_cache_has_usable_snapshot("model", "org/pipeline") is True
+
+
+def test_a_streaming_start_skips_the_cached_dataset_check():
+ """Streaming reads the Hub (load_dataset(streaming = True)) and never the materialized cache,
+ and the preflight already verified the repo remotely, so asking again only adds a refusal."""
+ import inspect
+
+ import routes.training as training_routes
+
+ source = inspect.getsource(training_routes.start_training)
+ assert "not request.dataset_streaming" in source
+ guarded = source.split("not request.dataset_streaming", 1)[1]
+ assert "_refuse_unauthorized_cached_dataset" in guarded.split("\n\n", 1)[0]
+
+
+def test_a_metadata_less_probe_ignores_repo_boilerplate(cache_root):
+ """huggingface_hub fetches the model card and .gitattributes first, so a barely started
+ download leaves a revision holding nothing a loader reads. That must not read as cached."""
+ from hub.utils.hf_cache_state import repo_cache_has_usable_snapshot
+
+ revision = cache_root / "models--org--card-only" / "snapshots" / "abc"
+ revision.mkdir(parents = True)
+ (revision / "README.md").write_text("# model card")
+ (revision / ".gitattributes").write_text("*.safetensors filter=lfs")
+ (revision / "LICENSE").write_text("apache-2.0")
+ assert repo_cache_has_usable_snapshot("model", "org/card-only") is False
+
+ # Anything else counts, including a format the loaders here do not know: skipping the check is
+ # the failure that matters, so the denylist never grows into an allowlist of weight formats.
+ (revision / "weights.unknown-format").write_bytes(b"w")
+ assert repo_cache_has_usable_snapshot("model", "org/card-only") is True
+
+
+def test_a_shadowed_hub_dataset_is_not_the_source():
+ """load_and_format_dataset takes local_datasets, then s3_config, ahead of dataset_source, so a
+ payload carrying both never reads the Hub cache and must not be refused over it."""
+ from types import SimpleNamespace
+
+ from routes.training import _hf_dataset_is_the_source
+
+ def request(**kwargs):
+ base = dict(hf_dataset = "org/private", local_datasets = [], s3_config = None)
+ base.update(kwargs)
+ return SimpleNamespace(**base)
+
+ assert _hf_dataset_is_the_source(request()) is True
+ assert _hf_dataset_is_the_source(request(local_datasets = ["/data/train.jsonl"])) is False
+ assert _hf_dataset_is_the_source(request(s3_config = {"bucket": "b"})) is False
+ assert _hf_dataset_is_the_source(request(hf_dataset = "")) is False
+
+
+def test_the_shadowed_dataset_gate_is_wired_into_the_start():
+ import inspect
+
+ import routes.training as training_routes
+
+ source = inspect.getsource(training_routes.start_training)
+ assert "_hf_dataset_is_the_source(request)" in source
+ guarded = source.split("_hf_dataset_is_the_source(request)", 1)[1]
+ assert "_refuse_unauthorized_cached_dataset" in guarded.split("\n\n", 1)[0]
+
+
+def test_a_diffusion_data_dir_cannot_name_a_cached_repo(tmp_path, monkeypatch):
+ """The diffusion data_dir is contained to the dataset roots, so it cannot name a path in the
+ operator's Hugging Face cache: absolute, traversing and symlinked forms are all refused."""
+ import routes.training as training_routes
+ from utils.paths import datasets_root
+
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
+ cache = tmp_path / "hfcache" / "datasets--org--private" / "snapshots" / "abc"
+ cache.mkdir(parents = True)
+ (cache / "0001.png").write_bytes(b"x")
+
+ with pytest.raises(ValueError):
+ training_routes._resolve_diffusion_data_dir(str(cache))
+ with pytest.raises(ValueError):
+ training_routes._resolve_diffusion_data_dir(
+ "../hfcache/datasets--org--private/snapshots/abc"
+ )
+
+ datasets_root().mkdir(parents = True, exist_ok = True)
+ (datasets_root() / "shadow").symlink_to(cache)
+ with pytest.raises(Exception) as caught:
+ training_routes._resolve_diffusion_data_dir("shadow")
+ assert "symbolic link" in str(caught.value)
+
+
+def test_the_diffusion_config_tolerates_the_new_policy_key():
+ """allow_ambient rides the raw config dict; the trainer dataclass must ignore it."""
+ from core.training.diffusion_train_common import _config_from_dict
+
+ cfg = _config_from_dict(
+ {
+ "base_model": "org/sdxl",
+ "data_dir": "/tmp/data",
+ "output_dir": "/tmp/out",
+ "allow_ambient": False,
+ }
+ )
+ assert not hasattr(cfg, "allow_ambient")