Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: overlap the quant scheme probe with the planner import on a cold image load (#16882)

Daniel Han9c084751abcc
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 32e182d9845..4d901baf443 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -527,8 +527,10 @@ from .diffusion_transformer_quant import (
NATIVE_QUANT_SCHEMES,
normalize_transformer_quant,
int8_text_stream_scope,
+ join_probe_table_prewarm,
quantize_transformer,
select_transformer_quant_scheme,
+ start_probe_table_prewarm,
stored_denoiser_precision,
transformer_is_quantised,
)
@@ -6273,51 +6275,60 @@ class DiffusionBackend:
is not None
):
return None
- if auto and _auto_keeps_bf16_reason(fam, base = base or repo_id) is not None:
- bf16_memory = snapshot_device_memory(target)
- bf16_plan = self._bf16_table_plan(
- target,
- fam,
- base or repo_id or "",
- memory_mode,
- cpu_offload,
- kind = kind,
- repo_id = repo_id,
- fetch_base = fetch_base,
- text_encoder_quant = None if local_files_only else text_encoder_quant,
- device_memory_override = replace(bf16_memory, free_mib = bf16_memory.total_mib),
- )
- # Offline without the released shards, bf16 cannot assemble: a cached seed is the only way.
- if (
- bf16_plan is not None
- and _auto_quant_eager_reason(
- fam, bf16_plan, transformer_prequant_path, base = base or repo_id
+ # The scheme probe in _planned_quant_scheme waits ~4-5 s on a child (cold card): start that child now so it
+ # overlaps the diffusers import _auto_keeps_bf16_reason pays, and join it before free VRAM is read.
+ probe = start_probe_table_prewarm(str(getattr(target, "device", "cuda")))
+ try:
+ if auto and _auto_keeps_bf16_reason(fam, base = base or repo_id) is not None:
+ join_probe_table_prewarm(probe)
+ bf16_memory = snapshot_device_memory(target)
+ bf16_plan = self._bf16_table_plan(
+ target,
+ fam,
+ base or repo_id or "",
+ memory_mode,
+ cpu_offload,
+ kind = kind,
+ repo_id = repo_id,
+ fetch_base = fetch_base,
+ text_encoder_quant = None if local_files_only else text_encoder_quant,
+ device_memory_override = replace(
+ bf16_memory, free_mib = bf16_memory.total_mib
+ ),
)
- is not None
- and not (
- local_files_only
- and not any(
- self._released_transformer_cached(candidate)
- for candidate in dict.fromkeys(
- (
- fetch_base,
- base,
- repo_id,
- prefer_ungated_mirror(base or repo_id or ""),
+ # Offline without the released shards, bf16 cannot assemble: a cached seed is the only way.
+ if (
+ bf16_plan is not None
+ and _auto_quant_eager_reason(
+ fam, bf16_plan, transformer_prequant_path, base = base or repo_id
+ )
+ is not None
+ and not (
+ local_files_only
+ and not any(
+ self._released_transformer_cached(candidate)
+ for candidate in dict.fromkeys(
+ (
+ fetch_base,
+ base,
+ repo_id,
+ prefer_ungated_mirror(base or repo_id or ""),
+ )
)
+ if candidate
)
- if candidate
)
- )
- ):
- return None
- scheme = _planned_quant_scheme(
- fam,
- target,
- mode,
- base_repo = base,
- prequant_path = transformer_prequant_path,
- )
+ ):
+ return None
+ scheme = _planned_quant_scheme(
+ fam,
+ target,
+ mode,
+ base_repo = base,
+ prequant_path = transformer_prequant_path,
+ )
+ finally:
+ join_probe_table_prewarm(probe, discard = True)
if scheme is None or scheme == TQ_AUTO:
return None
# Under auto, a rung whose plan still offloads yields to the next resident rung; an explicit scheme is never swapped, only honored or refused.
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py
index 4be72f681ad..fab7e835e30 100644
--- a/studio/backend/core/inference/diffusion_transformer_quant.py
+++ b/studio/backend/core/inference/diffusion_transformer_quant.py
@@ -1410,8 +1410,10 @@ def _scheme_supported(
# Re-checked under the lock: concurrent plans must not each spawn a child.
if (scheme, card) not in _SMOKE_CACHE:
table = _persisted_probe_table(card)
+ # The load's early kick (start_probe_table_prewarm) already ran this child: same table, earlier.
+ prewarmed = _PREWARMED_TABLES.pop(card, None)
if table is None or scheme not in table:
- table = _child_probe_table(card)
+ table = prewarmed if prewarmed is not None else _child_probe_table(card)
if table is not None:
for name, child_verdict in table.items():
# None is the child's out-of-memory: not a verdict, so not cached.
@@ -1564,6 +1566,77 @@ def prewarm_probe_table(device: str = "cuda") -> bool:
return True
+PROBE_OVERLAP_ENV = "UNSLOTH_DIFFUSION_PROBE_OVERLAP"
+# card -> the child table an early kick resolved; consumed by _scheme_supported, dropped by the kicking load's join.
+_PREWARMED_TABLES: dict[str, dict[str, Optional[bool]]] = {}
+
+
+def start_probe_table_prewarm(device: str) -> Optional[tuple]:
+ """Start the scheme probe's child on a daemon thread so it overlaps the load's imports (~4-5 s cold), only when
+ ``_scheme_supported`` would spawn it anyway (no cached or persisted verdict). Returns a handle for
+ ``join_probe_table_prewarm``, or None. ``UNSLOTH_DIFFUSION_PROBE_OVERLAP=0`` keeps the serial probe."""
+ if (_os.environ.get(PROBE_OVERLAP_ENV) or "").strip().lower() in ("0", "false", "no", "off"):
+ return None
+ try:
+ if not str(device).startswith("cuda"):
+ return None
+ import torch
+
+ if not torch.cuda.is_available():
+ return None
+ # Keyed in the caller's thread, under its device scope, exactly as _scheme_supported keys it.
+ card = _smoke_cache_device_key(str(device))
+ except Exception: # noqa: BLE001 - no early probe; the load probes as before
+ return None
+ schemes = without_nvfp4(TQ_SCHEMES)
+ if any((scheme, card) in _SMOKE_CACHE for scheme in schemes):
+ return None
+
+ def _run() -> None:
+ # Never queue behind another probe: the join must bound this thread by one child's own deadline, so a held
+ # lock means the load's serial probe runs instead.
+ if not _CHILD_PROBE_LOCK.acquire(blocking = False):
+ return
+ try:
+ if any((scheme, card) in _SMOKE_CACHE for scheme in schemes):
+ return
+ if _persisted_probe_table(card) is not None:
+ return
+ import logging
+ import time
+
+ started = time.monotonic()
+ table = _child_probe_table(card)
+ if table is not None:
+ _PREWARMED_TABLES[card] = table
+ logging.getLogger(__name__).info(
+ "diffusion.transformer_quant: scheme probe ran beside the load (%.1f s, %s)",
+ time.monotonic() - started,
+ card,
+ )
+ except Exception: # noqa: BLE001 - the load's own probe runs instead
+ pass
+ finally:
+ _CHILD_PROBE_LOCK.release()
+
+ thread = _threading.Thread(target = _run, name = "quant-probe-overlap", daemon = True)
+ thread.start()
+ return (thread, card)
+
+
+def join_probe_table_prewarm(handle: Optional[tuple], *, discard: bool = False) -> None:
+ """Wait for an early probe (its child, and so its CUDA context, is gone after this); ``discard`` drops a table
+ the load never consumed so it cannot outlive the load that started it. Untimed: the thread never waits on the
+ probe lock and ``_child_probe_table`` closes its child at ``_CHILD_PROBE_TIMEOUT``, so it always ends."""
+ if not handle:
+ return
+ thread, card = handle
+ thread.join()
+ if discard:
+ with _CHILD_PROBE_LOCK:
+ _PREWARMED_TABLES.pop(card, None)
+
+
def _persisted_probe_table(card: str) -> Optional[dict[str, bool]]:
"""A previous process's clean child table for this card and stack, or None."""
try:
diff --git a/studio/backend/tests/test_diffusion_probe_overlap.py b/studio/backend/tests/test_diffusion_probe_overlap.py
new file mode 100644
index 00000000000..f2d8f535338
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_probe_overlap.py
@@ -0,0 +1,159 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A cold image load's quant scheme probe (a spawned child, ~4.5 s on an L4) starts before the planner's diffusers
+import instead of after it, and hands the load the same table the serial probe would have produced."""
+
+from __future__ import annotations
+
+import threading
+import time
+import types
+
+import pytest
+
+pytest.importorskip("torch")
+
+_TABLE = {"int8": True, "fp8": False}
+
+
+def _fam(name):
+ from core.inference.diffusion_families import _FAMILIES
+ return next(f for f in _FAMILIES if f.name == name)
+
+
+def probe_env(monkeypatch):
+ import torch
+
+ from core.inference import diffusion as D
+ from core.inference import diffusion_transformer_quant as tq
+
+ ev = {
+ "child": [],
+ "import": [],
+ "snapshot": [],
+ "table": dict(_TABLE),
+ "keeps": None,
+ "persisted": None,
+ }
+
+ def child(card):
+ ev["child"].append(("start", time.monotonic(), threading.current_thread().name, card))
+ time.sleep(0.3)
+ ev["child"].append(("end", time.monotonic()))
+ return dict(ev["table"])
+
+ def keeps_bf16(*_a, **_k):
+ ev["import"].append(("start", time.monotonic()))
+ time.sleep(0.3) # stands in for the diffusers import this call pays on a cold process
+ ev["import"].append(("end", time.monotonic()))
+ return ev["keeps"]
+
+ def snapshot(_target):
+ ev["snapshot"].append(any(e[0] == "end" for e in ev["child"]) or not ev["child"])
+ raise RuntimeError("stop after the snapshot") # swallowed by the planner
+
+ def planned(_fam, target, _mode, **_k):
+ ok = tq._scheme_supported("int8", str(target.device))
+ ev["verdict"] = ok
+ return None
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PROBE_OVERLAP", raising = False)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(tq, "_smoke_cache_device_key", lambda device, ordinal = None: "cuda:0")
+ monkeypatch.setattr(tq, "_persisted_probe_table", lambda card: ev["persisted"])
+ monkeypatch.setattr(tq, "_child_probe_table", child)
+ monkeypatch.setattr(tq, "_smoke_probe", lambda *a, **k: pytest.fail("in-process probe reached"))
+ monkeypatch.setattr(tq, "_SMOKE_CACHE", {})
+ if hasattr(tq, "_PREWARMED_TABLES"):
+ monkeypatch.setattr(tq, "_PREWARMED_TABLES", {})
+ monkeypatch.setattr(D, "dense_transformer_supported", lambda *a, **k: True)
+ monkeypatch.setattr(D, "_pipeline_quant_uncompilable_reason", lambda *a, **k: None)
+ monkeypatch.setattr(D, "_auto_keeps_bf16_reason", keeps_bf16)
+ monkeypatch.setattr(D, "snapshot_device_memory", snapshot)
+ monkeypatch.setattr(D, "_planned_quant_scheme", planned)
+ backend = D.DiffusionBackend.__new__(D.DiffusionBackend)
+ backend._target_for_ordinal = lambda *a, **k: types.SimpleNamespace(device = "cuda", ordinal = 0)
+ fam = _fam("flux.2-klein")
+
+ def plan():
+ return backend._pipeline_planned_denoiser_scheme(
+ fam,
+ base = fam.base_repo,
+ kind = "pipeline",
+ transformer_quant = None,
+ speed_mode = None,
+ repo_id = fam.base_repo,
+ )
+
+ ev["plan"] = plan
+ ev["tq"] = tq
+ return ev
+
+
+def test_the_probe_child_runs_during_the_planner_import(probe_env):
+ ev = probe_env
+ assert ev["plan"]() is None
+ starts = [e for e in ev["child"] if e[0] == "start"]
+ assert len(starts) == 1, "exactly one probe child per cold load"
+ import_end = next(e[1] for e in ev["import"] if e[0] == "end")
+ # Base starts the child only once the import has returned (inside _planned_quant_scheme).
+ assert starts[0][1] < import_end, "the probe child must overlap the planner's import"
+ assert ev["verdict"] is True
+ tq = ev["tq"]
+ assert tq._SMOKE_CACHE == {("int8", "cuda:0"): True, ("fp8", "cuda:0"): False}
+ assert tq._PREWARMED_TABLES == {}
+
+
+def test_the_kill_switch_keeps_the_serial_probe(probe_env, monkeypatch):
+ ev = probe_env
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PROBE_OVERLAP", "0")
+ ev["plan"]()
+ starts = [e for e in ev["child"] if e[0] == "start"]
+ assert len(starts) == 1
+ assert starts[0][1] >= next(e[1] for e in ev["import"] if e[0] == "end")
+ assert ev["verdict"] is True
+
+
+def test_a_persisted_table_starts_no_child(probe_env):
+ ev = probe_env
+ ev["persisted"] = dict(_TABLE)
+ ev["plan"]()
+ assert ev["child"] == []
+ assert ev["verdict"] is True
+
+
+def test_an_out_of_memory_verdict_is_honoured_without_a_second_child(probe_env):
+ # None (the child's OOM) is not persisted, so the load must take the early table, not spawn again.
+ ev = probe_env
+ ev["table"] = {"int8": None, "fp8": False}
+ ev["plan"]()
+ assert len([e for e in ev["child"] if e[0] == "start"]) == 1
+ assert ev["verdict"] is False # unproven_ok defaults to False, as with the serial probe
+ assert ("int8", "cuda:0") not in ev["tq"]._SMOKE_CACHE
+
+
+def test_free_vram_is_read_only_after_the_child_exits(probe_env):
+ ev = probe_env
+ ev["keeps"] = "small DiT keeps bf16"
+ ev["plan"]()
+ assert ev["snapshot"] == [True], "snapshot_device_memory ran while the probe child was alive"
+ assert ev["tq"]._PREWARMED_TABLES == {}
+
+
+def test_a_held_probe_lock_skips_the_early_child_and_join_returns(probe_env):
+ # Another card's probe holds the lock: the early thread must not queue behind it (its join would then outlast
+ # one child's deadline), so it gives up at once and the join does not wait on the lock holder.
+ tq = probe_env["tq"]
+ assert tq._CHILD_PROBE_LOCK.acquire(timeout = 1.0)
+ try:
+ started = time.monotonic()
+ handle = tq.start_probe_table_prewarm("cuda")
+ assert handle is not None
+ tq.join_probe_table_prewarm(handle)
+ assert time.monotonic() - started < 5.0
+ assert not handle[0].is_alive()
+ finally:
+ tq._CHILD_PROBE_LOCK.release()
+ assert probe_env["child"] == []