Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: latch the CFG-batch verdict per load and across processes so a seed keeps one path (#14849)

Daniel Hanf15d723ff9c8
diff --git a/studio/backend/core/inference/diffusion_cfg_batch.py b/studio/backend/core/inference/diffusion_cfg_batch.py
index f79f57c68c1..bd467675906 100644
--- a/studio/backend/core/inference/diffusion_cfg_batch.py
+++ b/studio/backend/core/inference/diffusion_cfg_batch.py
@@ -19,7 +19,11 @@ Priced once per request shape, before any batched result is used: steps 1-2 run
pair >= ``_RESIDENT_MAX_PAIR_S`` stays plain untried. Otherwise a discarded trial (plain pairs interleaved with doubled
calls) batches steps 3+ only if the fastest batched step (predict, merge, snapshot, doubled call, split, verify) beats the
fastest plain pair by ``_MEASURED_FRACTION``, so every render of a shape takes one path (same seed, same image). The
-doubled call needs room for twice a plain call's activations; an OOM turns the shape plain for that render.
+verdict is latched for the load (memory pressure never re-times it) and stored per (model, speed tier, offload policy,
+card, compiled or eager, request shape, runtimes) next to the learned peaks, first writer wins: a borderline shape
+cannot land on a different path in another process (``UNSLOTH_DIFFUSION_CFG_BATCH_VERDICTS=0`` keeps it per load;
+``UNSLOTH_DIFFUSION_CFG_BATCH_VERDICTS_FILE`` names the file). The doubled call needs room for twice a plain call's
+activations; no room, or an OOM, turns the shape plain for the rest of that render.
Not always bit-exact (other GEMM / attention tiles): B200 bf16 resident FLUX.2-klein-base-4B is bitwise equal,
Wan2.1-T2V-1.3B LPIPS 0.019 mean; offloaded FLUX.2-klein-base LPIPS ~0.01. ``UNSLOTH_DIFFUSION_CFG_BATCH=0`` keeps the exact
@@ -254,10 +258,146 @@ def _usable_free_mib(device: Any) -> Optional[int]:
return None
+VERDICTS_ENV = "UNSLOTH_DIFFUSION_CFG_BATCH_VERDICTS"
+VERDICTS_FILE_ENV = "UNSLOTH_DIFFUSION_CFG_BATCH_VERDICTS_FILE"
+_VERDICTS_FILE = "diffusion_cfg_batch_verdicts.json"
+_VERDICTS_MAX = 512
+_BATCH = "batch"
+_PLAIN = "plain"
+
+
+def _verdicts_path() -> Optional[Any]:
+ """The timing verdict store (next to the learned peaks), None when turned off or nowhere safe to write."""
+ if (os.environ.get(VERDICTS_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
+ return None
+ from pathlib import Path
+
+ explicit = (os.environ.get(VERDICTS_FILE_ENV) or "").strip()
+ if explicit:
+ return Path(explicit)
+ from . import diffusion_measured_peaks as peaks
+
+ base = peaks.store_path()
+ return None if base is None else base.parent / _VERDICTS_FILE
+
+
+def _read_verdicts(path: Any) -> dict:
+ import json
+
+ from . import diffusion_measured_peaks as peaks
+
+ try:
+ data = json.loads(path.read_text(encoding = "utf-8"))
+ except Exception: # noqa: BLE001 - absent or corrupt: an empty store
+ return {}
+ if not isinstance(data, dict) or data.get("runtime") != peaks.runtime_stamp():
+ return {} # written under other kernels: their timings do not apply
+ entries = data.get("entries")
+ return entries if isinstance(entries, dict) else {}
+
+
+def _stored(entries: dict, key: str) -> Optional[dict]:
+ entry = entries.get(key)
+ if isinstance(entry, dict) and entry.get("verdict") in (_BATCH, _PLAIN):
+ return entry
+ return None
+
+
+def stored_verdict(key: str) -> Optional[dict]:
+ path = _verdicts_path()
+ if path is None:
+ return None
+ try:
+ return _stored(_read_verdicts(path), key)
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _settle_verdict(key: str, verdict: dict) -> dict:
+ """First writer wins: the stored verdict for ``key`` if any process measured one, else ``verdict`` (stored).
+ A store that cannot be read or written leaves this process with its own measurement."""
+ path = _verdicts_path()
+ if path is None:
+ return verdict
+ from . import diffusion_measured_peaks as peaks
+
+ try:
+ with peaks._lock, peaks._file_lock(path):
+ entries = _read_verdicts(path)
+ stored = _stored(entries, key)
+ if stored is not None:
+ return stored
+ entries[key] = verdict
+ for stale in list(entries)[: max(0, len(entries) - _VERDICTS_MAX)]:
+ entries.pop(stale, None)
+ import json
+ import tempfile
+
+ path.parent.mkdir(parents = True, exist_ok = True)
+ fd, tmp = tempfile.mkstemp(prefix = ".cfg_batch.", dir = str(path.parent))
+ try:
+ with os.fdopen(fd, "w", encoding = "utf-8") as handle:
+ json.dump(
+ {"version": 1, "runtime": peaks.runtime_stamp(), "entries": entries},
+ handle,
+ indent = 1,
+ )
+ os.replace(tmp, path)
+ except BaseException:
+ try:
+ os.unlink(tmp)
+ except OSError:
+ pass
+ raise
+ except Exception: # noqa: BLE001 - persistence is best-effort
+ pass
+ return verdict
+
+
+def _card_name(latent: Any) -> Optional[str]:
+ """The CUDA card a verdict was timed on (None off CUDA: nothing timed, nothing stored)."""
+ if getattr(latent.device, "type", None) != "cuda":
+ return None
+ from .diffusion_measured_peaks import device_name
+
+ return device_name(latent.device)
+
+
+def _load_identity(denoiser: Any, offload_policy: str, speed_mode: Optional[str]) -> Optional[str]:
+ """What a doubled call's timing depends on in the load: class, config, weight dtypes / layer classes (quant),
+ offload policy and speed tier. Hashed: it is a store key, never read back."""
+ import hashlib
+ import json
+
+ try:
+ config = getattr(denoiser, "config", None)
+ layers = sorted({f"{type(m).__module__}.{type(m).__name__}" for m in denoiser.modules()})
+ dtypes = sorted({str(p.dtype) for p in denoiser.parameters()})
+ blob = json.dumps(
+ [
+ f"{type(denoiser).__module__}.{type(denoiser).__name__}",
+ dict(config) if config is not None else None,
+ layers,
+ dtypes,
+ offload_policy,
+ speed_mode,
+ ],
+ sort_keys = True,
+ default = str,
+ )
+ except Exception: # noqa: BLE001 - unidentifiable: the verdict stays per load
+ return None
+ return hashlib.sha1(blob.encode("utf-8")).hexdigest()
+
+
class _Shape:
"""Per request shape: warm-up, timing, verdict and the run of misses."""
- def __init__(self) -> None:
+ def __init__(self, key: Any = None) -> None:
+ self.key = key
+ # the timing verdict (_BATCH / _PLAIN), latched for the load and persisted: memory pressure never re-times it
+ self.verdict: Optional[str] = None
+ self.doubled_ran = False
self.steps = 0
self.plain_pair_s: Optional[float] = None
self.batched_s: list[float] = []
@@ -306,6 +446,9 @@ class CfgBatchRouter:
self.pair_t0: Optional[float] = None
self.renders = 0 # completed renders: a doubled-batch compile is warm-up only in a shape's trial render
self.render_compiled: Optional[str] = None
+ self.identity: Optional[str] = (
+ None # the load's verdict-store identity (None: verdicts stay per load)
+ )
def _reset_step(self) -> None:
self.branch = 0
@@ -319,6 +462,7 @@ class CfgBatchRouter:
"armed_for": self.reason,
"disabled_reason": self.disabled,
"shape_off": None if shape is None else shape.off,
+ "verdict": None if shape is None else shape.verdict,
"plain_pair_s": None
if shape is None or shape.plain_pair_s is None
else round(shape.plain_pair_s, 4),
@@ -346,11 +490,10 @@ class CfgBatchRouter:
self._reset_step()
for shape in self.shapes.values():
shape.render_steps = 0
- if (
- shape.off_kind == _OFF_TRANSIENT
- ): # memory pressure is per render: retry (and re-time) on the next one
+ if shape.off_kind == _OFF_TRANSIENT:
+ # memory pressure is per render: retry on the next one, keeping a timing verdict already reached
shape.off = shape.off_kind = None
- shape.judged = False
+ shape.judged = shape.verdict == _BATCH
def disable(self, reason: str) -> None:
if self.disabled is None:
@@ -429,15 +572,31 @@ class CfgBatchRouter:
out = self._plain(args, kwargs)
if not shape.judged and self.disabled is None and shape.render_steps == _PLAIN_STEPS:
t0, self.pair_t0 = self.pair_t0, None
+ if shape.verdict is None:
+ stored = self._stored_verdict(shape, latent)
+ if stored is not None:
+ self._apply_verdict(shape, stored, stored = True)
+ if shape.verdict == _BATCH:
+ # already priced (this load, or any process on this model, shape and card): never re-timed, so
+ # a borderline shape cannot flip; only the memory gate below can keep this render plain
+ if self._memory_ok(latent, shape):
+ self._judge(shape)
+ else:
+ self._shape_off(shape, "no room for a doubled call", _OFF_TRANSIENT)
+ return out
+ if shape.off is not None:
+ return out
if t0 is not None and not self.offloaded and self._sync(latent):
live = time.perf_counter() - t0
if live >= _RESIDENT_MAX_PAIR_S:
- self._shape_off(
+ shape.plain_pair_s = live
+ self._decide(
shape,
+ latent,
+ _PLAIN,
f"compute-bound: a plain pair takes {live:.3f}s, where one doubled call cannot win "
f"{round((1 - _MEASURED_FRACTION) * 100)}%",
)
- shape.plain_pair_s = live
return out
self._trial((args, kwargs), latent, shape)
return out
@@ -458,7 +617,7 @@ class CfgBatchRouter:
key = tuple(tuple(t.shape) for t in mg._iter_tensors((args, kwargs)))
shape = self.shapes.get(key)
if shape is None:
- shape = self.shapes[key] = _Shape()
+ shape = self.shapes[key] = _Shape(key)
if shape is not self.last_shape:
self.template = (
None # a template predicts its own shape's second call: one plain pair first
@@ -508,6 +667,7 @@ class CfgBatchRouter:
f"({CFG_BATCH_ENV}=force batches anyway)",
)
return
+ shape.verdict = _BATCH # untimed (offloaded off CUDA): the same verdict every time, nothing to store
self._judge(shape)
return
if not self._memory_ok(latent, shape):
@@ -531,6 +691,7 @@ class CfgBatchRouter:
_matches(snapshots, second)
self._sync(latent)
shape.batched_s.append(time.perf_counter() - t0)
+ shape.doubled_ran = True
del b_args, b_kwargs, snapshots, predicted
shape.plain_pair_s = min(plain_s)
if _graph_count() != graphs and self.render_compiled is None and not self.offloaded:
@@ -553,12 +714,50 @@ class CfgBatchRouter:
return
batched = min(shape.batched_s)
if shape.plain_pair_s and batched > _MEASURED_FRACTION * shape.plain_pair_s:
- self._shape_off(
+ self._decide(
shape,
+ latent,
+ _PLAIN,
f"measured no gain: batched pair {batched:.4f}s vs plain pair {shape.plain_pair_s:.4f}s",
)
return
- self._judge(shape)
+ self._decide(
+ shape,
+ latent,
+ _BATCH,
+ f"measured gain: batched pair {batched:.4f}s vs plain pair {shape.plain_pair_s or 0:.4f}s",
+ )
+
+ def _verdict_key(self, shape: _Shape, latent: Any) -> Optional[str]:
+ """Store key of a timing verdict: the load's identity, the card, compiled or eager, the request shape.
+ None off CUDA or for an unidentified load (the verdict then holds for this load only)."""
+ card = None if self.identity is None else _card_name(latent)
+ if not card:
+ return None
+ compiled = "compiled" if self.render_compiled is not None else "eager"
+ return f"{self.identity}|{card}|{compiled}|{shape.key}"
+
+ def _stored_verdict(self, shape: _Shape, latent: Any) -> Optional[dict]:
+ key = self._verdict_key(shape, latent)
+ return None if key is None else stored_verdict(key)
+
+ def _apply_verdict(self, shape: _Shape, entry: dict, *, stored: bool) -> None:
+ shape.verdict = entry["verdict"]
+ if shape.verdict == _PLAIN:
+ reason = str(entry.get("reason") or "measured no gain")
+ self._shape_off(shape, f"stored verdict: {reason}" if stored else reason)
+
+ def _decide(self, shape: _Shape, latent: Any, verdict: str, reason: str) -> None:
+ """Latch a measured verdict; the first one stored for this model, shape and card wins across processes."""
+ entry = {"verdict": verdict, "reason": reason}
+ key = self._verdict_key(shape, latent)
+ settled = entry if key is None else _settle_verdict(key, entry)
+ self._apply_verdict(shape, settled, stored = settled is not entry)
+ if shape.verdict == _BATCH:
+ if self._memory_ok(latent, shape):
+ self._judge(shape)
+ else:
+ self._shape_off(shape, "no room for a doubled call", _OFF_TRANSIENT)
def _judge(self, shape: _Shape) -> None:
shape.judged = True
@@ -619,6 +818,8 @@ class CfgBatchRouter:
def _batched(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:
torch = _torch()
if not self._memory_ok(latent, shape):
+ # plain for the rest of this render (not one step here and there), batched again on the next one
+ self._shape_off(shape, "no room for a doubled call", _OFF_TRANSIENT)
return None
try:
second = self._predict(latent)
@@ -654,7 +855,15 @@ class CfgBatchRouter:
return None
self.disable(f"the doubled call failed ({type(exc).__name__}: {exc})")
return None
- if _graph_count() != graphs and self.renders > shape.trial_render:
+ if not shape.doubled_ran:
+ # a stored verdict skipped the trial: this render's first doubled call is the shape's warm-up
+ shape.doubled_ran = True
+ shape.trial_render = self.renders
+ if _graph_count() != graphs and self.render_compiled is None and not self.offloaded:
+ self.pending_disable = (
+ "the doubled batch compiled a new graph (the denoiser runs compiled code)"
+ )
+ elif _graph_count() != graphs and self.renders > shape.trial_render:
self.pending_disable = "the doubled batch compiled a graph after its warm-up render"
self.kept = {
"out": other,
@@ -877,6 +1086,7 @@ def _install(
logger = logger,
)
router.offloaded = _offloaded(denoiser, offload_policy)
+ router.identity = _load_identity(denoiser, offload_policy, speed_mode)
try:
_ROUTERS[pipe] = router
except TypeError:
diff --git a/studio/backend/tests/test_diffusion_cfg_batch.py b/studio/backend/tests/test_diffusion_cfg_batch.py
index f4e2d5f2f78..3976b232689 100644
--- a/studio/backend/tests/test_diffusion_cfg_batch.py
+++ b/studio/backend/tests/test_diffusion_cfg_batch.py
@@ -777,3 +777,118 @@ def test_both_backends_pass_the_requested_speed_mode():
assert calls and all(
"speed_mode = speed_mode" in c.split(")")[0] for c in calls
), module.__name__
+
+
+_TRIAL = [
+ 1,
+ 1,
+ 2,
+ 1,
+ 1,
+ 2,
+] # the discarded trial after steps 1-2 (two plain pairs, two doubled calls)
+
+
+def _switchable(
+ monkeypatch,
+ tmp_path,
+ card = "FakeGPU",
+):
+ """A timed fake whose doubled-call cost the test moves (timing noise around the gate), on a named card, with
+ the verdict store in ``tmp_path``. ``raising = False``: main has no store, so the behaviour is what fails there."""
+ cost = {"batched": 1.2}
+ model = _timed(monkeypatch, lambda b: 1.0 if b == 1 else cost["batched"])
+ monkeypatch.setenv(cb.CFG_BATCH_ENV, "auto")
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_CFG_BATCH_VERDICTS_FILE", str(tmp_path / "verdicts.json"))
+ monkeypatch.setattr(cb, "_card_name", lambda latent: card, raising = False)
+ return model, cost
+
+
+def _render(pipe, model):
+ start = len(model.calls)
+ _flux_loop(model)
+ cb.end_cfg_batch_render(pipe)
+ return model.calls[start:]
+
+
+def _fresh_load(model_cls):
+ """A new load of the same model (another process: only the verdict store survives)."""
+ model = model_cls()
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None
+ return pipe, model
+
+
[email protected]("first_wins", [True, False])
+def test_a_borderline_shape_takes_one_path_in_every_process(monkeypatch, tmp_path, first_wins):
+ model, cost = _switchable(monkeypatch, tmp_path)
+ cost["batched"] = (
+ 1.8 if first_wins else 1.9
+ ) # 0.90 vs 0.95 of the plain pair: either side of the 0.92 gate
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None
+ first = _render(pipe, model)
+ assert first[4:10] == _TRIAL
+ first_path = first[:4] + first[10:]
+ # the next process times the same shape on the other side of the gate (noise): it must not re-decide
+ cost["batched"] = 1.9 if first_wins else 1.8
+ for _ in range(2):
+ pipe2, model2 = _fresh_load(type(model))
+ later = _render(pipe2, model2)
+ assert later == first_path # no trial, the stored path
+ assert cb.router_for(pipe2).last_shape.verdict == ("batch" if first_wins else "plain")
+ assert first_path == ([1] * 4 + [2] * 4 if first_wins else [1] * 12)
+
+
+def test_memory_pressure_never_re_times_a_judged_shape(monkeypatch, tmp_path):
+ model, cost = _switchable(monkeypatch, tmp_path)
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_CFG_BATCH_VERDICTS", "0") # the in-load latch alone
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None
+ router = cb.router_for(pipe)
+ assert _render(pipe, model)[10:] == [2] * 4
+ real = router.orig_forward
+ state = {"oom": True}
+
+ def oom_once(*a, **k):
+ if state["oom"] and int(k["hidden_states"].shape[0]) == 2:
+ state["oom"] = False
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")
+ return real(*a, **k)
+
+ router.orig_forward = oom_once
+ assert _render(pipe, model) == [1] * 12 # the OOM render fell back to plain
+ router.orig_forward = real
+ cost["batched"] = 1.9 # a re-timing now would turn the shape plain for good
+ assert _render(pipe, model) == [1] * 4 + [2] * 4
+ assert router.last_shape.off is None and router.last_shape.verdict == "batch"
+
+
+def test_no_room_mid_render_keeps_the_rest_of_that_render_plain(monkeypatch, tmp_path):
+ model, _ = _switchable(monkeypatch, tmp_path)
+ free = iter([True, True, True, False, True, True] + [True] * 20)
+ monkeypatch.setattr(cb.CfgBatchRouter, "_memory_ok", lambda self, latent, shape: next(free))
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None
+ calls = _render(pipe, model)
+ # trial gate, verdict gate, step 3 batched, step 4 no room: steps 4-6 plain (not batched again at step 5)
+ assert calls[10:] == [2] + [1] * 6
+ assert _render(pipe, model) == [1] * 4 + [2] * 4 # the next render batches again, untimed
+
+
+def test_the_verdict_store_is_keyed_on_the_card_and_skipped_off_cuda(monkeypatch, tmp_path):
+ model, cost = _switchable(monkeypatch, tmp_path)
+ pipe = _Pipe(model)
+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None
+ _render(pipe, model)
+ assert "verdict" in (tmp_path / "verdicts.json").read_text()
+ cost["batched"] = 1.9
+ monkeypatch.setattr(cb, "_card_name", lambda latent: "OtherGPU")
+ pipe2, model2 = _fresh_load(type(model))
+ assert _render(pipe2, model2)[4:10] == _TRIAL # another card is priced on its own
+ assert "measured no gain" in cb.router_for(pipe2).last_shape.off
+ monkeypatch.setattr(cb, "_card_name", lambda latent: None) # off CUDA: nothing timed is stored
+ before = (tmp_path / "verdicts.json").read_text()
+ pipe3, model3 = _fresh_load(type(model))
+ assert _render(pipe3, model3)[4:10] == _TRIAL
+ assert (tmp_path / "verdicts.json").read_text() == before