Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Restore facts the comment trim cut from the Studio backend (#10329)

Daniel Hanf2b105e57b94
diff --git a/studio/backend/core/data_recipe/service.py b/studio/backend/core/data_recipe/service.py
index 553c102d86b..fc96427757d 100644
--- a/studio/backend/core/data_recipe/service.py
+++ b/studio/backend/core/data_recipe/service.py
@@ -181,7 +181,8 @@ def recipe_has_stdio_mcp(recipe: dict[str, Any]) -> bool:
def build_mcp_providers(recipe: dict[str, Any]) -> list:
from data_designer.config.mcp import LocalStdioMCPProvider, MCPProvider # pyright: ignore[reportMissingImports]
- # Stdio providers spawn a local subprocess, so build them only when this host allows it.
+ # Same gate as the chat MCP path: stdio providers spawn a local subprocess, so build
+ # them only when this host allows it (desktop loopback default / explicit opt-in).
from core.inference.mcp_client import stdio_mcp_enabled
stdio_allowed = stdio_mcp_enabled()
diff --git a/studio/backend/core/export/orchestrator.py b/studio/backend/core/export/orchestrator.py
index a6030f58136..14ecdceee33 100644
--- a/studio/backend/core/export/orchestrator.py
+++ b/studio/backend/core/export/orchestrator.py
@@ -493,8 +493,8 @@ class ExportOrchestrator:
try:
self._spawn_subprocess(sub_config)
except Exception:
- # A stale current_checkpoint would make the Export page claim a loaded checkpoint the next op then
- # fails on.
+ # The old worker is already gone; a stale current_checkpoint would make the Export page claim a
+ # loaded checkpoint that the next op then fails on with "No export subprocess running".
self.current_checkpoint = None
self.is_vision = False
self.is_peft = False
diff --git a/studio/backend/core/rag/ingestion.py b/studio/backend/core/rag/ingestion.py
index cd846ff88d9..3dc31bc397e 100644
--- a/studio/backend/core/rag/ingestion.py
+++ b/studio/backend/core/rag/ingestion.py
@@ -635,8 +635,10 @@ def job_events(job_id: str):
break
yield event
finally:
- # Keep the queue while the worker runs so an early disconnect can reconnect and resume; terminal is
- # still False at disconnect, since _run writes the DB status before emitting it.
+ # Drop the queue once nothing more will be emitted into it: a terminal exit, or a disconnect after the
+ # job already finished. The UI stops on the terminal event, before [DONE], so terminal is still False
+ # here; the re-read below catches it because _run writes the terminal DB status before emitting it.
+ # Keep the queue only while the worker is still running, so an early disconnect can reconnect and resume.
if not terminal:
try:
row = get_job_status(job_id)
diff --git a/studio/backend/core/training/diffusion_checkpoint.py b/studio/backend/core/training/diffusion_checkpoint.py
index 146a41b0011..1a2f85777bf 100644
--- a/studio/backend/core/training/diffusion_checkpoint.py
+++ b/studio/backend/core/training/diffusion_checkpoint.py
@@ -937,8 +937,11 @@ def _write_text(path: Path, text: str) -> None:
_fsync_file(path)
-# Windows' _commit needs write access, and network/container filesystems return EINVAL/ENOTSUP for
-# fsync; neither says the data did not make it.
+# errno values treated as "this platform or filesystem will not flush that handle": Windows' _commit
+# needs write access, and network/container filesystems return EINVAL/ENOTSUP for fsync. EBADF is the
+# exception and is NOT such a signal -- Windows collapses genuine FlushFileBuffers failures into it
+# too, so it is here as a deliberate tradeoff, not because the data is known to have made it. See
+# _fsync_file for what is left guarding that case.
_FSYNC_UNSUPPORTED = frozenset(
code
for code in (
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.py
index 7f95bd26848..4baf6ea478a 100644
--- a/studio/backend/core/training/diffusion_dit_trainer.py
+++ b/studio/backend/core/training/diffusion_dit_trainer.py
@@ -2372,7 +2372,8 @@ def _train_dit(
retire_own_checkpoints(out_dir, preexisting_checkpoints, resumed_here = resumed_here)
elif not resumed_here:
# A stop-with-save on a fresh retrain is a LOWER step than the earlier run's leftovers, and resume-
- # by-directory picks the newest by step, so those would outrank the partial just saved.
+ # by-directory picks the newest by step, so those would outrank the partial just saved and continue
+ # the wrong training. A run that RESUMED here instead keeps what it found.
discard_preexisting_checkpoints(out_dir, preexisting_checkpoints)
else:
# save_steps writes bundles as the run goes, so without this a discard leaves up to
diff --git a/studio/backend/core/training/diffusion_training_service.py b/studio/backend/core/training/diffusion_training_service.py
index afdef557d60..874f740398c 100644
--- a/studio/backend/core/training/diffusion_training_service.py
+++ b/studio/backend/core/training/diffusion_training_service.py
@@ -389,7 +389,7 @@ def _append_metric(
steps.append(istep)
for key in _METRIC_SERIES:
# setdefault, not [key]: a run resumed from a record written before a series existed carries a state
- # dict without it.
+ # dict without it, and a short tail on the new series beats a KeyError that drops the whole update.
state.setdefault(key, []).append(values[key])
diff --git a/studio/backend/hub/services/datasets/cache_inventory.py b/studio/backend/hub/services/datasets/cache_inventory.py
index f627563d2e6..6049586a246 100644
--- a/studio/backend/hub/services/datasets/cache_inventory.py
+++ b/studio/backend/hub/services/datasets/cache_inventory.py
@@ -231,8 +231,8 @@ def _is_payload_file(name: str) -> bool:
def _is_present_payload_file(path: Path) -> bool:
- # existence is not enough: a zero-byte file, and a blobs/ link whose blob was pruned, both look like
- # payload and are not.
+ # existence is not enough: _empty_payload covers both shapes that look like payload and are not,
+ # a zero-byte file and a blobs/ link whose blob was pruned.
if not _is_payload_file(path.name):
return False
if local_options._empty_payload(path):
diff --git a/studio/backend/hub/services/datasets/downloads.py b/studio/backend/hub/services/datasets/downloads.py
index 75299a25c28..a483b0d9363 100644
--- a/studio/backend/hub/services/datasets/downloads.py
+++ b/studio/backend/hub/services/datasets/downloads.py
@@ -70,7 +70,8 @@ def get_dataset_snapshot_metadata_cached(
size, hashes, restricted, cached_fp, ts = cached
if (time.monotonic() - ts) >= _DATASET_SIZE_POS_TTL:
del _dataset_size_cache[repo_id]
- # A gated or private repo's metadata is only served back to the token that fetched it.
+ # A gated or private repo's metadata is only served back to the token that fetched it;
+ # another token may have no access at all.
elif not restricted or cached_fp == token_fp:
_dataset_size_cache.move_to_end(repo_id)
return size, hashes
diff --git a/studio/backend/hub/services/download_lifecycle.py b/studio/backend/hub/services/download_lifecycle.py
index 50997ca75da..acc44fca23b 100644
--- a/studio/backend/hub/services/download_lifecycle.py
+++ b/studio/backend/hub/services/download_lifecycle.py
@@ -366,8 +366,10 @@ def finalize_worker_exit(
if state == "complete":
hf_cache_scan.invalidate_hf_cache_scans()
registry.set_job(key, "complete")
- # Where /v1 learns a new model exists: its resolver answers from a cached scan with no watcher.
- # Models only, since noting a dataset id as a local model would refuse a bare request naming it.
+ # Where /v1 learns a new model exists: its resolver answers from a cached scan with no watcher,
+ # so it would report the model absent and serve whatever is resident. Models only, since noting
+ # a dataset id as a local model would refuse a bare request naming it instead of letting a
+ # foreign id fall through.
if repo_type == "model":
try:
from core.inference.local_model_resolver import (
diff --git a/studio/backend/hub/services/models/cache_inventory.py b/studio/backend/hub/services/models/cache_inventory.py
index 67607cbe618..342e56a98c9 100644
--- a/studio/backend/hub/services/models/cache_inventory.py
+++ b/studio/backend/hub/services/models/cache_inventory.py
@@ -431,7 +431,10 @@ def _cache_inventory_fields(
):
capabilities["supports_vision"] = True
# Qwen3-ASR's required mmproj is an audio projector, not a vision one, and stt_only covers any
- # repo whose config sniffs as Whisper: can_chat is what auto-load and the chat picker filter on.
+ # repo whose config sniffs as Whisper, curated or not: a third-party checkpoint or a user's own
+ # fine-tune is just as unchattable. can_chat is what auto-load and the chat picker filter on,
+ # neither of which looks at the task, so task-scoping alone would leave curated STT rows
+ # eligible for chat auto-load.
if stt_only or is_curated_stt_repo_id(repo_id):
capabilities["supports_vision"] = False
capabilities["can_chat"] = False
@@ -708,7 +711,8 @@ def _scan_cached_inventory_snapshot(scanner, expected_epoch: int) -> list[dict]:
raise _CacheSourceChanged
rows = scanner(cache_scans = cache_scans, active_hub_cache = active_hub_cache)
# The walk itself takes seconds, so a delete or finished download landing during it supersedes these
- # rows as surely as one landing before it.
+ # rows as surely as one landing before it: without this a repo deleted mid-scan is still listed in
+ # the response.
if hf_cache_scan.hf_cache_scans_epoch() != expected_epoch:
raise _CacheSourceChanged
return rows
diff --git a/studio/backend/hub/services/models/deletion.py b/studio/backend/hub/services/models/deletion.py
index eead89e4d5c..43c7b7d713a 100644
--- a/studio/backend/hub/services/models/deletion.py
+++ b/studio/backend/hub/services/models/deletion.py
@@ -912,14 +912,18 @@ def _delete_cached_model_blocking(
)
# A companion base repo carries the text encoders, VAE and tokenizer for every quant of its
- # family, so removing it while one is installed leaves that quant unloadable. Derived from what is
- # installed right now, and only for a WHOLE-repo delete.
+ # family, so removing it while one is installed leaves that quant unloadable with nothing on screen
+ # to say why. Derived from what is installed right now, never from a stored count, and only for a
+ # WHOLE-repo delete. Deleting the dependants first makes the base an orphan, which Free up space
+ # then offers.
# Here rather than in the async caller so it shares this function's cache walk and stubs: the check
# IS part of the destructive stage.
- # The exception is a companion whose asset IS a named GGUF variant: native Qwen-Image opens one
- # fixed filename inside a chat GGUF repo, so removing that quant strands the image checkpoint. A
- # FLAG, never a rewrite of `variant`: widening the scope would delete revisions the user did not
- # ask for.
+ # The exception is a companion whose asset IS a named GGUF variant: native Qwen-Image opens exactly
+ # Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf inside a chat GGUF repo, so removing that one quant strands the
+ # image checkpoint however many siblings remain, and none of them is a substitute for a fixed
+ # filename. A FLAG, never a rewrite of `variant`: that name is the destructive scope, and widening
+ # it here would delete every revision and manifest the user did not ask for, and purge a sibling
+ # quant out from under an in-flight download.
guard_this_delete = variant is None or _variant_is_a_required_companion_asset(repo_id, variant)
if guard_this_delete and _is_companion_base_repo(repo_id):
# Fails CLOSED, and only here: the lookup above already established this repo IS a companion base,
diff --git a/studio/backend/hub/services/snapshot_progress.py b/studio/backend/hub/services/snapshot_progress.py
index 4fa51ebdbb3..a22e61a19f9 100644
--- a/studio/backend/hub/services/snapshot_progress.py
+++ b/studio/backend/hub/services/snapshot_progress.py
@@ -494,7 +494,8 @@ def compute_snapshot_progress(
readings: list[tuple[int, int, Optional[str], bool, Optional[bool]]] = []
# The enumeration suppresses OSError per root, so an unreadable cache root came back as "no dirs",
- # indistinguishable from a wiped cache, which hydration retires the job on.
+ # indistinguishable from a wiped cache, which hydration retires the job on. Collected so the empty
+ # answer below can say unknown instead of absent.
scan_errors: list = []
cache_dirs = (
preferred_repo_cache_dirs(
@@ -515,8 +516,9 @@ def compute_snapshot_progress(
# downloading the WHOLE file, so summing them overshoots.
partial_bytes: dict[str, int] = {}
completed_hashes: set[str] = set()
- # A partial attributable to no target is not evidence for this variant, nor against it while the
- # hashes are unresolved, and the by-name scan cannot see it.
+ # A partial attributable to no target is not evidence for this variant, since it may be a
+ # sibling quant's, nor against it while the hashes are unresolved, and the by-name scan cannot
+ # see it because a partial is not linked into a snapshot yet.
unattributable_partial = False
cache_path = hf_cache_scan.resolve_hf_cache_realpath(entry)
blobs_dir = entry / "blobs"
diff --git a/studio/backend/hub/tests/test_dataset_local_options.py b/studio/backend/hub/tests/test_dataset_local_options.py
index c650eac8b8b..b2c6c3788a2 100644
--- a/studio/backend/hub/tests/test_dataset_local_options.py
+++ b/studio/backend/hub/tests/test_dataset_local_options.py
@@ -233,7 +233,6 @@ def test_snapshot_options_infer_one_train_split_from_unlabelled_files(tmp_path):
snapshot = _snapshot(tmp_path)
_rows(snapshot, "records.jsonl")
- # A card with no bytes declares nothing, so it blocks nothing.
assert local_options._snapshot_options(snapshot) == {("default", "train")}
@@ -256,11 +255,6 @@ def test_snapshot_options_reject_a_shard_name_the_loader_refuses(tmp_path):
_rows(snapshot, "data/train-clean-00000-of-00001.parquet")
# datasets raises on the name rather than falling through to the keyword stages.
- # datasets calls .get on each entry, so a list of scalars raises.
- # DatasetCard.load reads the card as utf-8 and raises before any data file.
- # The inner .jsonl still votes, so datasets picks json and dies on the .zst.
- # DatasetCardData updates a dict from it and raises on a scalar.
- # imagefolder wins the vote and drops the jsonl, so there is nothing to train on.
assert local_options._snapshot_options(snapshot) == set()
@@ -296,9 +290,6 @@ def test_snapshot_options_ignore_a_metadata_file_when_it_steers_a_split(tmp_path
(snapshot / "test").mkdir()
(snapshot / "test" / "dataset_info.json").write_text("{}", encoding = "utf-8")
- # The header alone yields no row, and datasets still builds train around it.
- # resolve_pattern keeps a link only when its target is a file, so the loader never sees this one and the
- # surviving split still loads.
assert local_options._snapshot_options(snapshot) == {("default", "train")}
@@ -307,7 +298,6 @@ def test_snapshot_options_match_extensions_case_sensitively(tmp_path):
_rows(snapshot, "TRAIN.JSONL")
# A POSIX glob never matches .JSONL, so datasets finds no data files here.
- # datasets treats the empty declaration as authoritative and exposes no config.
assert local_options._snapshot_options(snapshot) == set()
@@ -676,6 +666,7 @@ def test_snapshot_options_reject_a_split_whose_module_needs_a_missing_codec(tmp_
(snapshot / "train.csv").write_text("text\nrow\n", encoding = "utf-8")
(snapshot / "train2.jsonl.zst").write_bytes(b"\x28\xb5\x2f\xfd\x00\x00")
+ # The inner .jsonl still votes, so datasets picks json and dies on the .zst.
assert local_options._snapshot_options(snapshot) == set()
@@ -692,6 +683,7 @@ def test_snapshot_options_stand_down_beside_a_card_that_cannot_be_decoded(tmp_pa
(snapshot / "README.md").write_bytes(b"\xff\xfe---\nconfigs: x\n---\n")
_rows(snapshot, "train.jsonl")
+ # DatasetCard.load reads the card as utf-8 and raises before any data file.
assert local_options._snapshot_options(snapshot) == set()
@@ -700,6 +692,7 @@ def test_snapshot_options_stand_down_beside_a_malformed_dataset_info_list(tmp_pa
_card(snapshot, "dataset_info: [foo]\n")
_rows(snapshot, "train.jsonl")
+ # datasets calls .get on each entry, so a list of scalars raises.
assert local_options._snapshot_options(snapshot) == set()
@@ -708,6 +701,8 @@ def test_snapshot_options_ignore_a_dangling_link_beside_a_good_file(tmp_path):
_rows(snapshot, "train.jsonl")
(snapshot / "train-missing.jsonl").symlink_to("../../blobs/gone")
+ # resolve_pattern keeps a link only when its target is a file, so the loader never sees this one and the
+ # surviving split still loads.
assert local_options._snapshot_options(snapshot) == {("default", "train")}
@@ -716,6 +711,7 @@ def test_snapshot_options_drop_a_header_only_csv_split(tmp_path):
(snapshot / "train.csv").write_text("text\nrow\n", encoding = "utf-8")
(snapshot / "test.csv").write_text("text\n", encoding = "utf-8")
+ # The header alone yields no row, and datasets still builds train around it.
assert local_options._snapshot_options(snapshot) == {("default", "train")}
@@ -731,6 +727,7 @@ def test_snapshot_options_stand_down_beside_an_empty_split_declaration(tmp_path)
_card(snapshot, "dataset_info:\n splits: {}\n")
_rows(snapshot, "train.jsonl")
+ # datasets treats the empty declaration as authoritative and exposes no config.
assert local_options._snapshot_options(snapshot) == set()
@@ -754,7 +751,6 @@ def test_snapshot_options_keep_a_compressed_csv_split(tmp_path):
snapshot = _snapshot(tmp_path)
(snapshot / "train.csv.gz").write_bytes(gzip.compress(b"text\nrow\n"))
- # `[]` parses fine and yields no row, so datasets builds train around it.
# Compressed bytes say nothing about the rows inside, so the split still stands.
assert local_options._snapshot_options(snapshot) == {("default", "train")}
@@ -764,6 +760,7 @@ def test_snapshot_options_drop_a_split_holding_an_empty_json_container(tmp_path)
_rows(snapshot, "train.jsonl")
(snapshot / "test.json").write_text("[]", encoding = "utf-8")
+ # `[]` parses fine and yields no row, so datasets builds train around it.
assert local_options._snapshot_options(snapshot) == {("default", "train")}
diff --git a/studio/backend/hub/tests/test_model_services.py b/studio/backend/hub/tests/test_model_services.py
index 32146b25899..574c8eb08d8 100644
--- a/studio/backend/hub/tests/test_model_services.py
+++ b/studio/backend/hub/tests/test_model_services.py
@@ -4011,7 +4011,7 @@ def test_gguf_progress_recovers_the_windows_shaped_stale_download_card(monkeypat
lambda: SimpleNamespace(hub_cache = str(hub_cache)),
)
# A manifest as an older build filed it: hashed from the unresolved spelling, and with no sha256
- # because HF metadata was already unreachable when the worker recorded it.
+ # because HF metadata was already unreachable when the worker recorded it from the finished snapshot.
legacy = state_dir.manifest_path(
"model",
"Org/Model-GGUF",
diff --git a/studio/backend/hub/utils/companion_assets.py b/studio/backend/hub/utils/companion_assets.py
index c96968bb6f3..0e67986227b 100644
--- a/studio/backend/hub/utils/companion_assets.py
+++ b/studio/backend/hub/utils/companion_assets.py
@@ -333,8 +333,9 @@ def _family_bases(repo_id: str, gguf_filename: Optional[str] = None) -> set[str]
bases = {resolve_base_repo(fam, None)}
bases |= _curated_variant_bases(fam, repo_id, gguf_filename)
# The NATIVE engine never reads the diffusers base: it fetches a single-file VAE and text encoder
- # from their own repos, so a pre-existing native GGUF with no recorded link would have had its
- # encoder listed as unused and removed underneath it.
+ # from their own repos, and those repos are offerable for deletion, so a pre-existing native GGUF
+ # with no recorded link would have had its encoder listed as unused and removed underneath it. The
+ # recorded links cover a load that has happened since; this covers the install that predates them.
bases |= {repo for repo, _file in _sd_cpp_component_specs(fam, repo_id, gguf_filename)}
return _with_mirrors(bases)
diff --git a/studio/backend/hub/utils/download_manifest.py b/studio/backend/hub/utils/download_manifest.py
index 5ac8fe3cdd0..6c7d9188e56 100644
--- a/studio/backend/hub/utils/download_manifest.py
+++ b/studio/backend/hub/utils/download_manifest.py
@@ -1198,7 +1198,8 @@ def purge_all_state_for_repo(
# resurrect as state for a cache that is gone.
search = []
# _scope_spellings, not cache_scope_names: every production caller resolves its target root first,
- # so the raw-spelling probe would be inert here while the read path still has it.
+ # so the raw-spelling probe would be inert here while the read path still has it - a purged
+ # variant that the next read brings back.
scopes = _scope_spellings(hub_cache)
for path_factory, base, cancel_markers in (
(manifest_path, manifests_dir(create = False), False),
diff --git a/studio/backend/hub/utils/download_registry.py b/studio/backend/hub/utils/download_registry.py
index c1bcd39faad..7624a1f9741 100644
--- a/studio/backend/hub/utils/download_registry.py
+++ b/studio/backend/hub/utils/download_registry.py
@@ -831,7 +831,11 @@ def prepare_cache_for_transport(
if mode == TRANSPORT_XET:
main_purge = _purge_incomplete_blobs(entry, only_blob_hashes, protected)
else:
- # A matching marker only matters while something can still append to the partial it vouches for.
+ # A matching marker vouches for provenance, which is only worth something while something can
+ # still append to the partial it vouches for. When nothing can, what survives is dead weight
+ # that holds the disk the refetch needs and, carrying the etag of the blob being refetched,
+ # pins the bar to its own stale high-water mark until the new attempt overtakes it. Sweep it,
+ # but only once abandoned.
if _read_marker(entry, variant) != mode:
main_purge = _purge_incomplete_blobs(entry, only_blob_hashes, protected)
else:
@@ -1194,10 +1198,12 @@ def existing_blob_bytes(
and not partial_is_resumable(blob.name, entry.parent)
and not blob_download_lock_held(entry, blob_hash)
):
- # An unresumable partial is refetched in full into a new path, so counting it would clear a
- # download for a disk that cannot hold it. A LOCKED one is different: a live peer is finishing it
- # and snapshot_download blocks on that lock and reuses the result, so those bytes are not ours to
- # find room for.
+ # Callers spend this on "bytes we will not have to fetch again", and _preflight_disk_space
+ # subtracts it from the space a download needs. An unresumable partial is refetched in full into
+ # a new path, so counting it would clear a download for a disk that cannot hold it. A LOCKED one
+ # is different: a live peer is finishing it and snapshot_download blocks on that lock and reuses
+ # the result, so those bytes are not ours to find room for. Two GGUF variants sharing an mmproj
+ # hit this every time.
continue
# Measured by the bytes actually ON DISK: hf_transfer's parallel Range writer leaves a sparse file
# whose st_size runs ahead of what was written, observed at 1.2 GB reported against 112 MB. A
@@ -1637,8 +1643,9 @@ class DownloadRegistry:
return False, conflict_state
current = self._jobs.get(key, DownloadState("idle")).state
if current in _ACTIVE_STATES and not replace_active:
- # A scope slot is shared by every file set that rides it, so adopting the live job would let the
- # caller wait on files it never asked for.
+ # A scope slot is shared by every file set that rides it (the images and video pages both
+ # key as "@diffusion"), so adopting the live job would let the caller wait on files it
+ # never asked for. Reject instead, under the lock.
live = self._metadata.get(key)
if (
scoped_files is not None
diff --git a/studio/backend/hub/utils/resumable_partials.py b/studio/backend/hub/utils/resumable_partials.py
index 52a790bd3bf..77a2d226995 100644
--- a/studio/backend/hub/utils/resumable_partials.py
+++ b/studio/backend/hub/utils/resumable_partials.py
@@ -351,7 +351,8 @@ def _objection_to(descriptor: int) -> Optional[str]:
return "hard linked from elsewhere"
euid = getattr(os, "geteuid", None)
if euid is None:
- # No owner to compare against, so nothing here can be vouched for.
+ # No owner to compare against, so nothing here can be vouched for. Reachable only if the writer
+ # is ever enabled without geteuid; _exclusion_is_provable refuses that today.
return "on a platform where ownership cannot be established"
if info.st_uid != euid():
return "owned by another user"
diff --git a/studio/backend/lan_access.py b/studio/backend/lan_access.py
index 7d2200f195b..8a0b0e6a5e5 100644
--- a/studio/backend/lan_access.py
+++ b/studio/backend/lan_access.py
@@ -451,7 +451,8 @@ def stop_lan_listener() -> bool:
server.should_exit = True
if _running_on_event_loop():
# /api/shutdown tears down from a task on this very loop, so waiting would deadlock; ownership is
- # kept because uvicorn cannot close the sockets until the loop is free again
+ # kept because uvicorn cannot close the sockets until the loop is free again, and
+ # _graceful_shutdown blocks it for seconds stopping subprocesses
logger.info("LAN access stopping")
return True
if loop is None or loop.is_closed() or not loop.is_running():
diff --git a/studio/backend/loggers/handlers.py b/studio/backend/loggers/handlers.py
index 488f1e774b7..12d52b1c37d 100644
--- a/studio/backend/loggers/handlers.py
+++ b/studio/backend/loggers/handlers.py
@@ -289,7 +289,7 @@ class LoggingMiddleware:
if window_ms <= 0:
return False
# The liveness group shares one bucket, so a burst logs once rather than once per path; only the
- # query-less form joins it.
+ # query-less form joins it, so a parameterized call still gets its own status and latency line.
key = _LIVENESS_DEDUP_KEY if is_liveness else (method, norm, query, status_code)
last = self._last_log.get(key)
if last is not None and (now - last) * 1000.0 < window_ms:
diff --git a/studio/backend/models/training.py b/studio/backend/models/training.py
index 6ec7d81f818..b5a9e6ca670 100644
--- a/studio/backend/models/training.py
+++ b/studio/backend/models/training.py
@@ -1113,8 +1113,9 @@ class DiffusionTrainableFamily(BaseModel):
precision_modes: List[str] = Field(default_factory = list)
recommended_precision: str = "nf4"
supports_compile: bool = False
- # save_steps is refused, not ignored, for a checkpointless family, so offering the control means
- # offering a value that rejects Start; defaults True so an older backend's payload keeps it.
+ # Whether this family's loop writes checkpoint bundles. False makes the panel drop the "Checkpoint
+ # every" control: save_steps is refused, not ignored, for a checkpointless family, so offering the
+ # control means offering a value that rejects Start; defaults True so an older backend's payload keeps it.
supports_checkpoints: bool = True
# 1 for a family whose forward covers one packed sequence: a value above the cap is refused rather
# than clamped, and declaring it here is what stops Pydantic dropping it from the response.
diff --git a/studio/backend/routes/chat_generation_runs.py b/studio/backend/routes/chat_generation_runs.py
index e8d32422047..f5846399550 100644
--- a/studio/backend/routes/chat_generation_runs.py
+++ b/studio/backend/routes/chat_generation_runs.py
@@ -353,6 +353,7 @@ async def chat_generation_events(
# A bare keep-alive proves only that the CONNECTION is healthy, so a follower rearming its no-progress
# deadline on one could never settle a wedged run while the socket stayed up, the one case that fallback
# exists for.
+ # Comment framing, so _SSEDecoder still drops it and no client parsing it as an event is affected.
yield f": keep-alive {int(snapshot['updatedAt'])}\n\n"
return StreamingResponse(
diff --git a/studio/backend/routes/data_recipe/jobs.py b/studio/backend/routes/data_recipe/jobs.py
index 998720c124b..4049218d9bc 100644
--- a/studio/backend/routes/data_recipe/jobs.py
+++ b/studio/backend/routes/data_recipe/jobs.py
@@ -353,7 +353,7 @@ def _inject_local_providers(
# Disable thinking for local recipe inference: the <think> preamble roughly doubles tokens per row and
# pushes answers past data_designer's json-fence regex.
# Forwarded as chat_template_kwargs={enable_thinking: False} via extra_body so llama-server renders the
- # template without it.
+ # template without it: llm-text columns get the latency cut, structured columns stop leaking think tags.
params = mc.get("inference_parameters")
if not isinstance(params, dict):
params = {}
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index e109c2fc854..aae36f59eae 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -1714,7 +1714,7 @@ def update_openai_auto_switch_override(
# Load order, not collection order: a lookup reads the concrete load path before the advertised repo id, so
# reading the repo row first adopts tuning no load has used.
_alias_ids.sort(key = lambda _key: not is_cache_load_path_key(_key))
- # Taken as a unit from the first row that exists.
+ # Taken as a unit from the first row that exists, not field by field down the list.
# A load stops at the first non-empty row (resolve_override_for_load) rather than merging, so filling a gap
# in the winner from a loser would switch dormant tuning on.
for _alias_id in _alias_ids:
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index a2c90a3f846..464b146acf9 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -1172,7 +1172,8 @@ async def start_training(
job_id = f"job_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{_uuid.uuid4().hex[:8]}"
if request.start_request_id:
# A retry of an already-resolved id replays its stored outcome even when a NEW start would be refused;
- # peeking also refreshes a cancellation tombstone so a retry cannot outlive it.
+ # peeking also refreshes a cancellation tombstone so a retry cannot outlive it and go on to
+ # spawn the run the user cancelled.
existing_record = backend.peek_start_request(request.start_request_id)
if existing_record is not None:
return _start_request_response(existing_record)
diff --git a/studio/backend/storage/api_usage_db.py b/studio/backend/storage/api_usage_db.py
index 5f8f7cbac2e..4d8777ba8f1 100644
--- a/studio/backend/storage/api_usage_db.py
+++ b/studio/backend/storage/api_usage_db.py
@@ -241,7 +241,8 @@ class ApiUsageWriter:
break
# record_api_usage already made its bounded fast retries: retain this accepted item at
# the head of
- # the single writer until a long transaction releases SQLite, with the stop sentinel behind it.
+ # the single writer until a long transaction releases SQLite, with the stop sentinel behind it
+ # so final shutdown drains rather than silently dropping usage.
busy_failures += 1
if busy_failures == 1 or busy_failures % 20 == 0:
logger.warning(
diff --git a/studio/backend/storage/research_runs_db.py b/studio/backend/storage/research_runs_db.py
index 6c4f1740191..6f029541e89 100644
--- a/studio/backend/storage/research_runs_db.py
+++ b/studio/backend/storage/research_runs_db.py
@@ -786,7 +786,8 @@ def retry(run_id: str, max_retries: int = 3) -> str:
if row["status"] not in {"failed", "cancelled"}:
raise ResearchConflictError("Only failed or cancelled runs can be retried")
# Counted per question, not per run row: a thread re-points one run at each new question, so a raw
- # retry_count would hand a fresh question whatever the stopped one left over.
+ # retry_count would hand a fresh question whatever the stopped one left over, and nothing at all
+ # once the budget was spent.
spent = conn.execute(
"SELECT COUNT(*) FROM research_events WHERE run_id=? AND event_type='run.retried' "
"AND seq > COALESCE((SELECT MAX(seq) FROM research_events "
diff --git a/studio/backend/utils/mlx_repair.py b/studio/backend/utils/mlx_repair.py
index 81e3826d6d9..227315b7b11 100644
--- a/studio/backend/utils/mlx_repair.py
+++ b/studio/backend/utils/mlx_repair.py
@@ -428,8 +428,9 @@ def attempt_mlx_repair(*, timeout: int = _REPAIR_TIMEOUT_S) -> bool:
)
return False
logger.info("MLX self-heal: installing %s", ", ".join(MLX_PACKAGES))
- # Before the wait, not after: every package is passed with --reinstall-package, so a timeout part way through
- # leaves a stack neither the one detection measured nor the one asked for.
+ # Before the wait, not after: every package is passed with --reinstall-package, so uv removes and
+ # replaces them as it goes and a timeout or a non-zero exit part way through leaves a stack neither
+ # the one detection measured nor the one asked for. Nothing before this line touches the environment.
_environment_mutated = True
result = subprocess.run(
cmd,
diff --git a/studio/backend/utils/models/model_identity.py b/studio/backend/utils/models/model_identity.py
index f5fc507fa22..3f6699b857f 100644
--- a/studio/backend/utils/models/model_identity.py
+++ b/studio/backend/utils/models/model_identity.py
@@ -81,9 +81,9 @@ def restore_hf_cache_repo_identity(
"_name_or_path",
repo_id,
)
- # PreTrainedModel.__init__ copies config.name_or_path onto the instance, and PEFT reads exactly this slot, which is
- # how a pinned snapshot path reaches adapter_config.json and every export.
- # PEFT reads exactly this slot (mapping_func.py) and overwrites base_model_name_or_path with it.
+ # PreTrainedModel.__init__ copies config.name_or_path onto the instance, so updating the config alone leaves
+ # this stale. PEFT reads exactly this slot (mapping_func.py) and overwrites base_model_name_or_path with it,
+ # which is how a pinned snapshot path reaches adapter_config.json and every export.
changed = _set_standard_identity(model, "name_or_path", repo_id) or changed
changed = _set_standard_identity(model, "_hf_repo", repo_id) or changed
diff --git a/studio/backend/utils/process_lifetime.py b/studio/backend/utils/process_lifetime.py
index 330736781a4..f0fcb58af74 100644
--- a/studio/backend/utils/process_lifetime.py
+++ b/studio/backend/utils/process_lifetime.py
@@ -78,7 +78,8 @@ _tracked_pgids: "dict[int, int]" = {}
_record_lock = threading.Lock()
-# Whether cleanup-on-abnormal-exit is in force, and why not.
+# Whether cleanup-on-abnormal-exit is in force, and why not. A silent failure here leaks every
+# child on a crash, so record it and log it.
_win_job_status: "tuple[bool, str]" = (False, "not attempted")
@@ -1022,7 +1023,8 @@ def terminate_all(timeout: float = 5.0) -> "list[int]":
if current is not None and identity is not None and not _same_identity(identity, current):
continue
if identity is None or current is None:
- # Cannot prove this is still our child, so do not signal it.
+ # Cannot prove this is still our child, so do not signal it. Keep it recorded while it
+ # is alive: the startup sweep runs the same test.
if _pid_alive(pid) and not _pid_is_zombie(pid):
with _record_lock:
_tracked_pids[pid] = identity
diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py
index ffede8b3bce..b2a2c46e555 100644
--- a/studio/install_python_stack.py
+++ b/studio/install_python_stack.py
@@ -6343,7 +6343,8 @@ _UV_INDEX_ENV_VARS = (
"UV_FIND_LINKS",
"PIP_EXTRA_INDEX_URL",
"PIP_FIND_LINKS",
- # PIP_NO_INDEX would defeat --index-url; PIP_INDEX_URL dropped so a mirror cannot outrank it.
+ # PIP_NO_INDEX=1 makes the pip fallback ignore ALL indexes, defeating --index-url; PIP_INDEX_URL is
+ # dropped too so a stale mirror env cannot outrank the pin.
"PIP_NO_INDEX",
"PIP_INDEX_URL",
)
diff --git a/studio/scripts/unsloth_freeze_report.py b/studio/scripts/unsloth_freeze_report.py
index 52fd9ac3586..15c9fc6b9f8 100644
--- a/studio/scripts/unsloth_freeze_report.py
+++ b/studio/scripts/unsloth_freeze_report.py
@@ -529,7 +529,13 @@ def classify(
"this started"
)
- # Did the interface stop polling partway through while the watchdog carried on?
+ # Did the interface stop polling partway through while the watchdog carried on? That is the reported
+ # symptom, and a total that looks healthy can still hide it.
+ # Only after the warmup boundary: on a cold launch the native watchdog is answering before the webview
+ # has finished loading, so the very first samples always show a still interface count and a rising
+ # watchdog count, and comparing them reported healthy runs as FROZE at the moment they finished starting.
+ # A freeze does not recover, so a gap the interface polls its way out of is a delayed request rather
+ # than the symptom.
post = [s for s in samples if s[0] >= warmup]
resumed_at = _last_rise(post, 1)
watchdog_last = _last_rise(post, 2)