Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: overlap the quant scheme probe with the planner import on a cold image load (#16882)
Daniel Han9c084751abcc
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 32e182d9845..4d901baf443 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -527,8 +527,10 @@ from .diffusion_transformer_quant import (NATIVE_QUANT_SCHEMES,normalize_transformer_quant,int8_text_stream_scope,+ join_probe_table_prewarm,quantize_transformer,select_transformer_quant_scheme,+ start_probe_table_prewarm,stored_denoiser_precision,transformer_is_quantised,)@@ -6273,51 +6275,60 @@ class DiffusionBackend:is not None):return None- if auto and _auto_keeps_bf16_reason(fam, base = base or repo_id) is not None:- bf16_memory = snapshot_device_memory(target)- bf16_plan = self._bf16_table_plan(- target,- fam,- base or repo_id or "",- memory_mode,- cpu_offload,- kind = kind,- repo_id = repo_id,- fetch_base = fetch_base,- text_encoder_quant = None if local_files_only else text_encoder_quant,- device_memory_override = replace(bf16_memory, free_mib = bf16_memory.total_mib),- )- # Offline without the released shards, bf16 cannot assemble: a cached seed is the only way.- if (- bf16_plan is not None- and _auto_quant_eager_reason(- fam, bf16_plan, transformer_prequant_path, base = base or repo_id+ # The scheme probe in _planned_quant_scheme waits ~4-5 s on a child (cold card): start that child now so it+ # overlaps the diffusers import _auto_keeps_bf16_reason pays, and join it before free VRAM is read.+ probe = start_probe_table_prewarm(str(getattr(target, "device", "cuda")))+ try:+ if auto and _auto_keeps_bf16_reason(fam, base = base or repo_id) is not None:+ join_probe_table_prewarm(probe)+ bf16_memory = snapshot_device_memory(target)+ bf16_plan = self._bf16_table_plan(+ target,+ fam,+ base or repo_id or "",+ memory_mode,+ cpu_offload,+ kind = kind,+ repo_id = repo_id,+ fetch_base = fetch_base,+ text_encoder_quant = None if local_files_only else text_encoder_quant,+ device_memory_override = replace(+ bf16_memory, free_mib = bf16_memory.total_mib+ ),)- is not None- and not (- local_files_only- and not any(- self._released_transformer_cached(candidate)- for candidate in dict.fromkeys(- (- fetch_base,- base,- repo_id,- prefer_ungated_mirror(base or repo_id or ""),+ # Offline without the released shards, bf16 cannot assemble: a cached seed is the only way.+ if (+ bf16_plan is not None+ and _auto_quant_eager_reason(+ fam, bf16_plan, transformer_prequant_path, base = base or repo_id+ )+ is not None+ and not (+ local_files_only+ and not any(+ self._released_transformer_cached(candidate)+ for candidate in dict.fromkeys(+ (+ fetch_base,+ base,+ repo_id,+ prefer_ungated_mirror(base or repo_id or ""),+ ))+ if candidate)- if candidate)- )- ):- return None- scheme = _planned_quant_scheme(- fam,- target,- mode,- base_repo = base,- prequant_path = transformer_prequant_path,- )+ ):+ return None+ scheme = _planned_quant_scheme(+ fam,+ target,+ mode,+ base_repo = base,+ prequant_path = transformer_prequant_path,+ )+ finally:+ join_probe_table_prewarm(probe, discard = True)if scheme is None or scheme == TQ_AUTO:return None# Under auto, a rung whose plan still offloads yields to the next resident rung; an explicit scheme is never swapped, only honored or refused.diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex 4be72f681ad..fab7e835e30 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -1410,8 +1410,10 @@ def _scheme_supported(# Re-checked under the lock: concurrent plans must not each spawn a child.if (scheme, card) not in _SMOKE_CACHE:table = _persisted_probe_table(card)+ # The load's early kick (start_probe_table_prewarm) already ran this child: same table, earlier.+ prewarmed = _PREWARMED_TABLES.pop(card, None)if table is None or scheme not in table:- table = _child_probe_table(card)+ table = prewarmed if prewarmed is not None else _child_probe_table(card)if table is not None:for name, child_verdict in table.items():# None is the child's out-of-memory: not a verdict, so not cached.@@ -1564,6 +1566,77 @@ def prewarm_probe_table(device: str = "cuda") -> bool:return True+PROBE_OVERLAP_ENV = "UNSLOTH_DIFFUSION_PROBE_OVERLAP"+# card -> the child table an early kick resolved; consumed by _scheme_supported, dropped by the kicking load's join.+_PREWARMED_TABLES: dict[str, dict[str, Optional[bool]]] = {}+++def start_probe_table_prewarm(device: str) -> Optional[tuple]:+ """Start the scheme probe's child on a daemon thread so it overlaps the load's imports (~4-5 s cold), only when+ ``_scheme_supported`` would spawn it anyway (no cached or persisted verdict). Returns a handle for+ ``join_probe_table_prewarm``, or None. ``UNSLOTH_DIFFUSION_PROBE_OVERLAP=0`` keeps the serial probe."""+ if (_os.environ.get(PROBE_OVERLAP_ENV) or "").strip().lower() in ("0", "false", "no", "off"):+ return None+ try:+ if not str(device).startswith("cuda"):+ return None+ import torch++ if not torch.cuda.is_available():+ return None+ # Keyed in the caller's thread, under its device scope, exactly as _scheme_supported keys it.+ card = _smoke_cache_device_key(str(device))+ except Exception: # noqa: BLE001 - no early probe; the load probes as before+ return None+ schemes = without_nvfp4(TQ_SCHEMES)+ if any((scheme, card) in _SMOKE_CACHE for scheme in schemes):+ return None++ def _run() -> None:+ # Never queue behind another probe: the join must bound this thread by one child's own deadline, so a held+ # lock means the load's serial probe runs instead.+ if not _CHILD_PROBE_LOCK.acquire(blocking = False):+ return+ try:+ if any((scheme, card) in _SMOKE_CACHE for scheme in schemes):+ return+ if _persisted_probe_table(card) is not None:+ return+ import logging+ import time++ started = time.monotonic()+ table = _child_probe_table(card)+ if table is not None:+ _PREWARMED_TABLES[card] = table+ logging.getLogger(__name__).info(+ "diffusion.transformer_quant: scheme probe ran beside the load (%.1f s, %s)",+ time.monotonic() - started,+ card,+ )+ except Exception: # noqa: BLE001 - the load's own probe runs instead+ pass+ finally:+ _CHILD_PROBE_LOCK.release()++ thread = _threading.Thread(target = _run, name = "quant-probe-overlap", daemon = True)+ thread.start()+ return (thread, card)+++def join_probe_table_prewarm(handle: Optional[tuple], *, discard: bool = False) -> None:+ """Wait for an early probe (its child, and so its CUDA context, is gone after this); ``discard`` drops a table+ the load never consumed so it cannot outlive the load that started it. Untimed: the thread never waits on the+ probe lock and ``_child_probe_table`` closes its child at ``_CHILD_PROBE_TIMEOUT``, so it always ends."""+ if not handle:+ return+ thread, card = handle+ thread.join()+ if discard:+ with _CHILD_PROBE_LOCK:+ _PREWARMED_TABLES.pop(card, None)++def _persisted_probe_table(card: str) -> Optional[dict[str, bool]]:"""A previous process's clean child table for this card and stack, or None."""try:diff --git a/studio/backend/tests/test_diffusion_probe_overlap.py b/studio/backend/tests/test_diffusion_probe_overlap.pynew file mode 100644index 00000000000..f2d8f535338--- /dev/null+++ b/studio/backend/tests/test_diffusion_probe_overlap.py@@ -0,0 +1,159 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A cold image load's quant scheme probe (a spawned child, ~4.5 s on an L4) starts before the planner's diffusers+import instead of after it, and hands the load the same table the serial probe would have produced."""++from __future__ import annotations++import threading+import time+import types++import pytest++pytest.importorskip("torch")++_TABLE = {"int8": True, "fp8": False}+++def _fam(name):+ from core.inference.diffusion_families import _FAMILIES+ return next(f for f in _FAMILIES if f.name == name)+++def probe_env(monkeypatch):+ import torch++ from core.inference import diffusion as D+ from core.inference import diffusion_transformer_quant as tq++ ev = {+ "child": [],+ "import": [],+ "snapshot": [],+ "table": dict(_TABLE),+ "keeps": None,+ "persisted": None,+ }++ def child(card):+ ev["child"].append(("start", time.monotonic(), threading.current_thread().name, card))+ time.sleep(0.3)+ ev["child"].append(("end", time.monotonic()))+ return dict(ev["table"])++ def keeps_bf16(*_a, **_k):+ ev["import"].append(("start", time.monotonic()))+ time.sleep(0.3) # stands in for the diffusers import this call pays on a cold process+ ev["import"].append(("end", time.monotonic()))+ return ev["keeps"]++ def snapshot(_target):+ ev["snapshot"].append(any(e[0] == "end" for e in ev["child"]) or not ev["child"])+ raise RuntimeError("stop after the snapshot") # swallowed by the planner++ def planned(_fam, target, _mode, **_k):+ ok = tq._scheme_supported("int8", str(target.device))+ ev["verdict"] = ok+ return None++ monkeypatch.delenv("UNSLOTH_DIFFUSION_PROBE_OVERLAP", raising = False)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(tq, "_smoke_cache_device_key", lambda device, ordinal = None: "cuda:0")+ monkeypatch.setattr(tq, "_persisted_probe_table", lambda card: ev["persisted"])+ monkeypatch.setattr(tq, "_child_probe_table", child)+ monkeypatch.setattr(tq, "_smoke_probe", lambda *a, **k: pytest.fail("in-process probe reached"))+ monkeypatch.setattr(tq, "_SMOKE_CACHE", {})+ if hasattr(tq, "_PREWARMED_TABLES"):+ monkeypatch.setattr(tq, "_PREWARMED_TABLES", {})+ monkeypatch.setattr(D, "dense_transformer_supported", lambda *a, **k: True)+ monkeypatch.setattr(D, "_pipeline_quant_uncompilable_reason", lambda *a, **k: None)+ monkeypatch.setattr(D, "_auto_keeps_bf16_reason", keeps_bf16)+ monkeypatch.setattr(D, "snapshot_device_memory", snapshot)+ monkeypatch.setattr(D, "_planned_quant_scheme", planned)+ backend = D.DiffusionBackend.__new__(D.DiffusionBackend)+ backend._target_for_ordinal = lambda *a, **k: types.SimpleNamespace(device = "cuda", ordinal = 0)+ fam = _fam("flux.2-klein")++ def plan():+ return backend._pipeline_planned_denoiser_scheme(+ fam,+ base = fam.base_repo,+ kind = "pipeline",+ transformer_quant = None,+ speed_mode = None,+ repo_id = fam.base_repo,+ )++ ev["plan"] = plan+ ev["tq"] = tq+ return ev+++def test_the_probe_child_runs_during_the_planner_import(probe_env):+ ev = probe_env+ assert ev["plan"]() is None+ starts = [e for e in ev["child"] if e[0] == "start"]+ assert len(starts) == 1, "exactly one probe child per cold load"+ import_end = next(e[1] for e in ev["import"] if e[0] == "end")+ # Base starts the child only once the import has returned (inside _planned_quant_scheme).+ assert starts[0][1] < import_end, "the probe child must overlap the planner's import"+ assert ev["verdict"] is True+ tq = ev["tq"]+ assert tq._SMOKE_CACHE == {("int8", "cuda:0"): True, ("fp8", "cuda:0"): False}+ assert tq._PREWARMED_TABLES == {}+++def test_the_kill_switch_keeps_the_serial_probe(probe_env, monkeypatch):+ ev = probe_env+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PROBE_OVERLAP", "0")+ ev["plan"]()+ starts = [e for e in ev["child"] if e[0] == "start"]+ assert len(starts) == 1+ assert starts[0][1] >= next(e[1] for e in ev["import"] if e[0] == "end")+ assert ev["verdict"] is True+++def test_a_persisted_table_starts_no_child(probe_env):+ ev = probe_env+ ev["persisted"] = dict(_TABLE)+ ev["plan"]()+ assert ev["child"] == []+ assert ev["verdict"] is True+++def test_an_out_of_memory_verdict_is_honoured_without_a_second_child(probe_env):+ # None (the child's OOM) is not persisted, so the load must take the early table, not spawn again.+ ev = probe_env+ ev["table"] = {"int8": None, "fp8": False}+ ev["plan"]()+ assert len([e for e in ev["child"] if e[0] == "start"]) == 1+ assert ev["verdict"] is False # unproven_ok defaults to False, as with the serial probe+ assert ("int8", "cuda:0") not in ev["tq"]._SMOKE_CACHE+++def test_free_vram_is_read_only_after_the_child_exits(probe_env):+ ev = probe_env+ ev["keeps"] = "small DiT keeps bf16"+ ev["plan"]()+ assert ev["snapshot"] == [True], "snapshot_device_memory ran while the probe child was alive"+ assert ev["tq"]._PREWARMED_TABLES == {}+++def test_a_held_probe_lock_skips_the_early_child_and_join_returns(probe_env):+ # Another card's probe holds the lock: the early thread must not queue behind it (its join would then outlast+ # one child's deadline), so it gives up at once and the join does not wait on the lock holder.+ tq = probe_env["tq"]+ assert tq._CHILD_PROBE_LOCK.acquire(timeout = 1.0)+ try:+ started = time.monotonic()+ handle = tq.start_probe_table_prewarm("cuda")+ assert handle is not None+ tq.join_probe_table_prewarm(handle)+ assert time.monotonic() - started < 5.0+ assert not handle[0].is_alive()+ finally:+ tq._CHILD_PROBE_LOCK.release()+ assert probe_env["child"] == []