Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: latch the CFG-batch verdict per load and across processes so a seed keeps one path (#14849)
Daniel Hanf15d723ff9c8
diff --git a/studio/backend/core/inference/diffusion_cfg_batch.py b/studio/backend/core/inference/diffusion_cfg_batch.pyindex f79f57c68c1..bd467675906 100644--- a/studio/backend/core/inference/diffusion_cfg_batch.py+++ b/studio/backend/core/inference/diffusion_cfg_batch.py@@ -19,7 +19,11 @@ Priced once per request shape, before any batched result is used: steps 1-2 runpair >= ``_RESIDENT_MAX_PAIR_S`` stays plain untried. Otherwise a discarded trial (plain pairs interleaved with doubledcalls) batches steps 3+ only if the fastest batched step (predict, merge, snapshot, doubled call, split, verify) beats thefastest plain pair by ``_MEASURED_FRACTION``, so every render of a shape takes one path (same seed, same image). The-doubled call needs room for twice a plain call's activations; an OOM turns the shape plain for that render.+verdict is latched for the load (memory pressure never re-times it) and stored per (model, speed tier, offload policy,+card, compiled or eager, request shape, runtimes) next to the learned peaks, first writer wins: a borderline shape+cannot land on a different path in another process (``UNSLOTH_DIFFUSION_CFG_BATCH_VERDICTS=0`` keeps it per load;+``UNSLOTH_DIFFUSION_CFG_BATCH_VERDICTS_FILE`` names the file). The doubled call needs room for twice a plain call's+activations; no room, or an OOM, turns the shape plain for the rest of that render.Not always bit-exact (other GEMM / attention tiles): B200 bf16 resident FLUX.2-klein-base-4B is bitwise equal,Wan2.1-T2V-1.3B LPIPS 0.019 mean; offloaded FLUX.2-klein-base LPIPS ~0.01. ``UNSLOTH_DIFFUSION_CFG_BATCH=0`` keeps the exact@@ -254,10 +258,146 @@ def _usable_free_mib(device: Any) -> Optional[int]:return None+VERDICTS_ENV = "UNSLOTH_DIFFUSION_CFG_BATCH_VERDICTS"+VERDICTS_FILE_ENV = "UNSLOTH_DIFFUSION_CFG_BATCH_VERDICTS_FILE"+_VERDICTS_FILE = "diffusion_cfg_batch_verdicts.json"+_VERDICTS_MAX = 512+_BATCH = "batch"+_PLAIN = "plain"+++def _verdicts_path() -> Optional[Any]:+ """The timing verdict store (next to the learned peaks), None when turned off or nowhere safe to write."""+ if (os.environ.get(VERDICTS_ENV) or "").strip().lower() in ("0", "off", "false", "no"):+ return None+ from pathlib import Path++ explicit = (os.environ.get(VERDICTS_FILE_ENV) or "").strip()+ if explicit:+ return Path(explicit)+ from . import diffusion_measured_peaks as peaks++ base = peaks.store_path()+ return None if base is None else base.parent / _VERDICTS_FILE+++def _read_verdicts(path: Any) -> dict:+ import json++ from . import diffusion_measured_peaks as peaks++ try:+ data = json.loads(path.read_text(encoding = "utf-8"))+ except Exception: # noqa: BLE001 - absent or corrupt: an empty store+ return {}+ if not isinstance(data, dict) or data.get("runtime") != peaks.runtime_stamp():+ return {} # written under other kernels: their timings do not apply+ entries = data.get("entries")+ return entries if isinstance(entries, dict) else {}+++def _stored(entries: dict, key: str) -> Optional[dict]:+ entry = entries.get(key)+ if isinstance(entry, dict) and entry.get("verdict") in (_BATCH, _PLAIN):+ return entry+ return None+++def stored_verdict(key: str) -> Optional[dict]:+ path = _verdicts_path()+ if path is None:+ return None+ try:+ return _stored(_read_verdicts(path), key)+ except Exception: # noqa: BLE001+ return None+++def _settle_verdict(key: str, verdict: dict) -> dict:+ """First writer wins: the stored verdict for ``key`` if any process measured one, else ``verdict`` (stored).+ A store that cannot be read or written leaves this process with its own measurement."""+ path = _verdicts_path()+ if path is None:+ return verdict+ from . import diffusion_measured_peaks as peaks++ try:+ with peaks._lock, peaks._file_lock(path):+ entries = _read_verdicts(path)+ stored = _stored(entries, key)+ if stored is not None:+ return stored+ entries[key] = verdict+ for stale in list(entries)[: max(0, len(entries) - _VERDICTS_MAX)]:+ entries.pop(stale, None)+ import json+ import tempfile++ path.parent.mkdir(parents = True, exist_ok = True)+ fd, tmp = tempfile.mkstemp(prefix = ".cfg_batch.", dir = str(path.parent))+ try:+ with os.fdopen(fd, "w", encoding = "utf-8") as handle:+ json.dump(+ {"version": 1, "runtime": peaks.runtime_stamp(), "entries": entries},+ handle,+ indent = 1,+ )+ os.replace(tmp, path)+ except BaseException:+ try:+ os.unlink(tmp)+ except OSError:+ pass+ raise+ except Exception: # noqa: BLE001 - persistence is best-effort+ pass+ return verdict+++def _card_name(latent: Any) -> Optional[str]:+ """The CUDA card a verdict was timed on (None off CUDA: nothing timed, nothing stored)."""+ if getattr(latent.device, "type", None) != "cuda":+ return None+ from .diffusion_measured_peaks import device_name++ return device_name(latent.device)+++def _load_identity(denoiser: Any, offload_policy: str, speed_mode: Optional[str]) -> Optional[str]:+ """What a doubled call's timing depends on in the load: class, config, weight dtypes / layer classes (quant),+ offload policy and speed tier. Hashed: it is a store key, never read back."""+ import hashlib+ import json++ try:+ config = getattr(denoiser, "config", None)+ layers = sorted({f"{type(m).__module__}.{type(m).__name__}" for m in denoiser.modules()})+ dtypes = sorted({str(p.dtype) for p in denoiser.parameters()})+ blob = json.dumps(+ [+ f"{type(denoiser).__module__}.{type(denoiser).__name__}",+ dict(config) if config is not None else None,+ layers,+ dtypes,+ offload_policy,+ speed_mode,+ ],+ sort_keys = True,+ default = str,+ )+ except Exception: # noqa: BLE001 - unidentifiable: the verdict stays per load+ return None+ return hashlib.sha1(blob.encode("utf-8")).hexdigest()++class _Shape:"""Per request shape: warm-up, timing, verdict and the run of misses."""- def __init__(self) -> None:+ def __init__(self, key: Any = None) -> None:+ self.key = key+ # the timing verdict (_BATCH / _PLAIN), latched for the load and persisted: memory pressure never re-times it+ self.verdict: Optional[str] = None+ self.doubled_ran = Falseself.steps = 0self.plain_pair_s: Optional[float] = Noneself.batched_s: list[float] = []@@ -306,6 +446,9 @@ class CfgBatchRouter:self.pair_t0: Optional[float] = Noneself.renders = 0 # completed renders: a doubled-batch compile is warm-up only in a shape's trial renderself.render_compiled: Optional[str] = None+ self.identity: Optional[str] = (+ None # the load's verdict-store identity (None: verdicts stay per load)+ )def _reset_step(self) -> None:self.branch = 0@@ -319,6 +462,7 @@ class CfgBatchRouter:"armed_for": self.reason,"disabled_reason": self.disabled,"shape_off": None if shape is None else shape.off,+ "verdict": None if shape is None else shape.verdict,"plain_pair_s": Noneif shape is None or shape.plain_pair_s is Noneelse round(shape.plain_pair_s, 4),@@ -346,11 +490,10 @@ class CfgBatchRouter:self._reset_step()for shape in self.shapes.values():shape.render_steps = 0- if (- shape.off_kind == _OFF_TRANSIENT- ): # memory pressure is per render: retry (and re-time) on the next one+ if shape.off_kind == _OFF_TRANSIENT:+ # memory pressure is per render: retry on the next one, keeping a timing verdict already reachedshape.off = shape.off_kind = None- shape.judged = False+ shape.judged = shape.verdict == _BATCHdef disable(self, reason: str) -> None:if self.disabled is None:@@ -429,15 +572,31 @@ class CfgBatchRouter:out = self._plain(args, kwargs)if not shape.judged and self.disabled is None and shape.render_steps == _PLAIN_STEPS:t0, self.pair_t0 = self.pair_t0, None+ if shape.verdict is None:+ stored = self._stored_verdict(shape, latent)+ if stored is not None:+ self._apply_verdict(shape, stored, stored = True)+ if shape.verdict == _BATCH:+ # already priced (this load, or any process on this model, shape and card): never re-timed, so+ # a borderline shape cannot flip; only the memory gate below can keep this render plain+ if self._memory_ok(latent, shape):+ self._judge(shape)+ else:+ self._shape_off(shape, "no room for a doubled call", _OFF_TRANSIENT)+ return out+ if shape.off is not None:+ return outif t0 is not None and not self.offloaded and self._sync(latent):live = time.perf_counter() - t0if live >= _RESIDENT_MAX_PAIR_S:- self._shape_off(+ shape.plain_pair_s = live+ self._decide(shape,+ latent,+ _PLAIN,f"compute-bound: a plain pair takes {live:.3f}s, where one doubled call cannot win "f"{round((1 - _MEASURED_FRACTION) * 100)}%",)- shape.plain_pair_s = livereturn outself._trial((args, kwargs), latent, shape)return out@@ -458,7 +617,7 @@ class CfgBatchRouter:key = tuple(tuple(t.shape) for t in mg._iter_tensors((args, kwargs)))shape = self.shapes.get(key)if shape is None:- shape = self.shapes[key] = _Shape()+ shape = self.shapes[key] = _Shape(key)if shape is not self.last_shape:self.template = (None # a template predicts its own shape's second call: one plain pair first@@ -508,6 +667,7 @@ class CfgBatchRouter:f"({CFG_BATCH_ENV}=force batches anyway)",)return+ shape.verdict = _BATCH # untimed (offloaded off CUDA): the same verdict every time, nothing to storeself._judge(shape)returnif not self._memory_ok(latent, shape):@@ -531,6 +691,7 @@ class CfgBatchRouter:_matches(snapshots, second)self._sync(latent)shape.batched_s.append(time.perf_counter() - t0)+ shape.doubled_ran = Truedel b_args, b_kwargs, snapshots, predictedshape.plain_pair_s = min(plain_s)if _graph_count() != graphs and self.render_compiled is None and not self.offloaded:@@ -553,12 +714,50 @@ class CfgBatchRouter:returnbatched = min(shape.batched_s)if shape.plain_pair_s and batched > _MEASURED_FRACTION * shape.plain_pair_s:- self._shape_off(+ self._decide(shape,+ latent,+ _PLAIN,f"measured no gain: batched pair {batched:.4f}s vs plain pair {shape.plain_pair_s:.4f}s",)return- self._judge(shape)+ self._decide(+ shape,+ latent,+ _BATCH,+ f"measured gain: batched pair {batched:.4f}s vs plain pair {shape.plain_pair_s or 0:.4f}s",+ )++ def _verdict_key(self, shape: _Shape, latent: Any) -> Optional[str]:+ """Store key of a timing verdict: the load's identity, the card, compiled or eager, the request shape.+ None off CUDA or for an unidentified load (the verdict then holds for this load only)."""+ card = None if self.identity is None else _card_name(latent)+ if not card:+ return None+ compiled = "compiled" if self.render_compiled is not None else "eager"+ return f"{self.identity}|{card}|{compiled}|{shape.key}"++ def _stored_verdict(self, shape: _Shape, latent: Any) -> Optional[dict]:+ key = self._verdict_key(shape, latent)+ return None if key is None else stored_verdict(key)++ def _apply_verdict(self, shape: _Shape, entry: dict, *, stored: bool) -> None:+ shape.verdict = entry["verdict"]+ if shape.verdict == _PLAIN:+ reason = str(entry.get("reason") or "measured no gain")+ self._shape_off(shape, f"stored verdict: {reason}" if stored else reason)++ def _decide(self, shape: _Shape, latent: Any, verdict: str, reason: str) -> None:+ """Latch a measured verdict; the first one stored for this model, shape and card wins across processes."""+ entry = {"verdict": verdict, "reason": reason}+ key = self._verdict_key(shape, latent)+ settled = entry if key is None else _settle_verdict(key, entry)+ self._apply_verdict(shape, settled, stored = settled is not entry)+ if shape.verdict == _BATCH:+ if self._memory_ok(latent, shape):+ self._judge(shape)+ else:+ self._shape_off(shape, "no room for a doubled call", _OFF_TRANSIENT)def _judge(self, shape: _Shape) -> None:shape.judged = True@@ -619,6 +818,8 @@ class CfgBatchRouter:def _batched(self, args: tuple, kwargs: dict, latent: Any, shape: _Shape) -> Any:torch = _torch()if not self._memory_ok(latent, shape):+ # plain for the rest of this render (not one step here and there), batched again on the next one+ self._shape_off(shape, "no room for a doubled call", _OFF_TRANSIENT)return Nonetry:second = self._predict(latent)@@ -654,7 +855,15 @@ class CfgBatchRouter:return Noneself.disable(f"the doubled call failed ({type(exc).__name__}: {exc})")return None- if _graph_count() != graphs and self.renders > shape.trial_render:+ if not shape.doubled_ran:+ # a stored verdict skipped the trial: this render's first doubled call is the shape's warm-up+ shape.doubled_ran = True+ shape.trial_render = self.renders+ if _graph_count() != graphs and self.render_compiled is None and not self.offloaded:+ self.pending_disable = (+ "the doubled batch compiled a new graph (the denoiser runs compiled code)"+ )+ elif _graph_count() != graphs and self.renders > shape.trial_render:self.pending_disable = "the doubled batch compiled a graph after its warm-up render"self.kept = {"out": other,@@ -877,6 +1086,7 @@ def _install(logger = logger,)router.offloaded = _offloaded(denoiser, offload_policy)+ router.identity = _load_identity(denoiser, offload_policy, speed_mode)try:_ROUTERS[pipe] = routerexcept TypeError:diff --git a/studio/backend/tests/test_diffusion_cfg_batch.py b/studio/backend/tests/test_diffusion_cfg_batch.pyindex f4e2d5f2f78..3976b232689 100644--- a/studio/backend/tests/test_diffusion_cfg_batch.py+++ b/studio/backend/tests/test_diffusion_cfg_batch.py@@ -777,3 +777,118 @@ def test_both_backends_pass_the_requested_speed_mode():assert calls and all("speed_mode = speed_mode" in c.split(")")[0] for c in calls), module.__name__+++_TRIAL = [+ 1,+ 1,+ 2,+ 1,+ 1,+ 2,+] # the discarded trial after steps 1-2 (two plain pairs, two doubled calls)+++def _switchable(+ monkeypatch,+ tmp_path,+ card = "FakeGPU",+):+ """A timed fake whose doubled-call cost the test moves (timing noise around the gate), on a named card, with+ the verdict store in ``tmp_path``. ``raising = False``: main has no store, so the behaviour is what fails there."""+ cost = {"batched": 1.2}+ model = _timed(monkeypatch, lambda b: 1.0 if b == 1 else cost["batched"])+ monkeypatch.setenv(cb.CFG_BATCH_ENV, "auto")+ monkeypatch.setenv("UNSLOTH_DIFFUSION_CFG_BATCH_VERDICTS_FILE", str(tmp_path / "verdicts.json"))+ monkeypatch.setattr(cb, "_card_name", lambda latent: card, raising = False)+ return model, cost+++def _render(pipe, model):+ start = len(model.calls)+ _flux_loop(model)+ cb.end_cfg_batch_render(pipe)+ return model.calls[start:]+++def _fresh_load(model_cls):+ """A new load of the same model (another process: only the verdict store survives)."""+ model = model_cls()+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None+ return pipe, model++[email protected]("first_wins", [True, False])+def test_a_borderline_shape_takes_one_path_in_every_process(monkeypatch, tmp_path, first_wins):+ model, cost = _switchable(monkeypatch, tmp_path)+ cost["batched"] = (+ 1.8 if first_wins else 1.9+ ) # 0.90 vs 0.95 of the plain pair: either side of the 0.92 gate+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None+ first = _render(pipe, model)+ assert first[4:10] == _TRIAL+ first_path = first[:4] + first[10:]+ # the next process times the same shape on the other side of the gate (noise): it must not re-decide+ cost["batched"] = 1.9 if first_wins else 1.8+ for _ in range(2):+ pipe2, model2 = _fresh_load(type(model))+ later = _render(pipe2, model2)+ assert later == first_path # no trial, the stored path+ assert cb.router_for(pipe2).last_shape.verdict == ("batch" if first_wins else "plain")+ assert first_path == ([1] * 4 + [2] * 4 if first_wins else [1] * 12)+++def test_memory_pressure_never_re_times_a_judged_shape(monkeypatch, tmp_path):+ model, cost = _switchable(monkeypatch, tmp_path)+ monkeypatch.setenv("UNSLOTH_DIFFUSION_CFG_BATCH_VERDICTS", "0") # the in-load latch alone+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None+ router = cb.router_for(pipe)+ assert _render(pipe, model)[10:] == [2] * 4+ real = router.orig_forward+ state = {"oom": True}++ def oom_once(*a, **k):+ if state["oom"] and int(k["hidden_states"].shape[0]) == 2:+ state["oom"] = False+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")+ return real(*a, **k)++ router.orig_forward = oom_once+ assert _render(pipe, model) == [1] * 12 # the OOM render fell back to plain+ router.orig_forward = real+ cost["batched"] = 1.9 # a re-timing now would turn the shape plain for good+ assert _render(pipe, model) == [1] * 4 + [2] * 4+ assert router.last_shape.off is None and router.last_shape.verdict == "batch"+++def test_no_room_mid_render_keeps_the_rest_of_that_render_plain(monkeypatch, tmp_path):+ model, _ = _switchable(monkeypatch, tmp_path)+ free = iter([True, True, True, False, True, True] + [True] * 20)+ monkeypatch.setattr(cb.CfgBatchRouter, "_memory_ok", lambda self, latent, shape: next(free))+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None+ calls = _render(pipe, model)+ # trial gate, verdict gate, step 3 batched, step 4 no room: steps 4-6 plain (not batched again at step 5)+ assert calls[10:] == [2] + [1] * 6+ assert _render(pipe, model) == [1] * 4 + [2] * 4 # the next render batches again, untimed+++def test_the_verdict_store_is_keyed_on_the_card_and_skipped_off_cuda(monkeypatch, tmp_path):+ model, cost = _switchable(monkeypatch, tmp_path)+ pipe = _Pipe(model)+ assert cb.install_cfg_batch(pipe, offload_policy = "none") is None+ _render(pipe, model)+ assert "verdict" in (tmp_path / "verdicts.json").read_text()+ cost["batched"] = 1.9+ monkeypatch.setattr(cb, "_card_name", lambda latent: "OtherGPU")+ pipe2, model2 = _fresh_load(type(model))+ assert _render(pipe2, model2)[4:10] == _TRIAL # another card is priced on its own+ assert "measured no gain" in cb.router_for(pipe2).last_shape.off+ monkeypatch.setattr(cb, "_card_name", lambda latent: None) # off CUDA: nothing timed is stored+ before = (tmp_path / "verdicts.json").read_text()+ pipe3, model3 = _fresh_load(type(model))+ assert _render(pipe3, model3)[4:10] == _TRIAL+ assert (tmp_path / "verdicts.json").read_text() == before