Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: Automatic prefers one card plus a second-card copy over a layer split for FLUX when the primary holds the whole denoiser (#16687)

Daniel Han09fd0d38ad3a
diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.py
index 5b397dafeee..99ee646b41e 100644
--- a/studio/backend/core/inference/diffusion_multigpu.py
+++ b/studio/backend/core/inference/diffusion_multigpu.py
@@ -753,6 +753,173 @@ def _one_card_rejection(plan: Any, primary: int) -> PlacementCandidate:
)
+# Automatic placement order: one card (plus a second-card copy for guided steps / even batches) before a layer split,
+# when a copy fits and streaming the denoiser on one card is hidden behind its compute: a layer split adds no
+# parallelism (its blocks run one card after the other), so it only removes the stream, and below this exposed share
+# the cards' own speed spread (T4s 10% apart under the same load) decides instead.
+STREAM_HIDDEN_FRACTION = 0.05
+STREAM_GATE_ENV = "UNSLOTH_DIFFUSION_SPLIT_STREAM_GATE"
+_GEMM_PROBES: dict = {}
+
+
+def _gemm_tflops(device_index: int, dtype: Any) -> Optional[float]:
+ """Dense GEMM TFLOPS of ``device_index`` in ``dtype`` (the denoiser's compute dtype: fp16 on a T4, where bf16 runs
+ emulated), best of a few runs, cached."""
+ key = (int(device_index), str(dtype))
+ if key in _GEMM_PROBES:
+ return _GEMM_PROBES[key]
+ result = None
+ try:
+ torch = _torch()
+ with torch.cuda.device(int(device_index)):
+ a = torch.randn(4096, 3072, dtype = dtype, device = "cuda")
+ b = torch.randn(3072, 3072, dtype = dtype, device = "cuda")
+ start, end = torch.cuda.Event(enable_timing = True), torch.cuda.Event(enable_timing = True)
+ torch.mm(a, b)
+ torch.cuda.synchronize()
+ best = float("inf")
+ for _ in range(3):
+ start.record()
+ for _ in range(10):
+ torch.mm(a, b)
+ end.record()
+ end.synchronize()
+ best = min(best, start.elapsed_time(end) / 10)
+ del a, b
+ if best > 0:
+ result = 2.0 * 4096 * 3072 * 3072 / (best / 1000.0) / 1e12
+ except Exception: # noqa: BLE001 - unmeasurable: no gate
+ result = None
+ _GEMM_PROBES[key] = result
+ return result
+
+
+def stream_exposure(
+ streamed_mib: int,
+ dense_params: float,
+ *,
+ h2d_gib_s: float,
+ tflops: float,
+ side_px: int = 1024,
+) -> tuple[float, float, float]:
+ """(transfer s, compute s, exposed share) of one denoise step streaming ``streamed_mib`` while computing a DiT of
+ ``dense_params`` parameters at ``side_px``. Exposed share = time the step waits on the stream / step time. Pure."""
+ from .diffusion_memory import _DIT_COMPUTE_EFFICIENCY, _DIT_PIXELS_PER_TOKEN
+
+ transfer = max(0, int(streamed_mib)) / 1024.0 / max(float(h2d_gib_s), 1e-6)
+ tokens = (int(side_px) // _DIT_PIXELS_PER_TOKEN) ** 2
+ compute = (
+ _DIT_COMPUTE_EFFICIENCY
+ * 2.0
+ * float(dense_params)
+ * tokens
+ / (max(float(tflops), 1e-6) * 1e12)
+ )
+ step = max(transfer, compute)
+ exposed = max(0.0, transfer - compute) / step if step > 0 else 0.0
+ return transfer, compute, exposed
+
+
+def _hidden_stream_reason(pipe: Any, plan: Any, sizes: dict, selection: Any) -> Optional[str]:
+ """Why Automatic keeps the one-card streamed plan instead of a layer split, or None (try the split as before).
+
+ Only for an automatic selection whose one-card plan streams the denoiser in groups (a capacity plan such as
+ sequential / whole-module offload keeps the split), priced at Studio's cost canvas from this card's measured
+ pinned H2D bandwidth and its GEMM throughput in the denoiser's dtype. ``UNSLOTH_DIFFUSION_SPLIT_STREAM_GATE=0``
+ restores the old choice."""
+ if not getattr(selection, "automatic", False):
+ return None
+ if (os.environ.get(STREAM_GATE_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
+ return None
+ if getattr(plan, "offload_policy", None) != "group":
+ return None
+ denoiser_cls = type(getattr(pipe, "transformer", None)).__name__
+ if denoiser_cls not in _DIT_BATCH_SPLIT_KWARGS:
+ # Only where a second-card copy can serve every kind of request this family runs (an even batch splits, a
+ # guided step pairs): elsewhere (Z-Image, measured 2.3-3.1x faster split) the split stays as on main.
+ return None
+ partner = _copy_partner(pipe, selection)
+ if partner is None:
+ # the copy does not fit an idle same-model card: the split stays (FLUX.2-klein-9B: 0-14% faster split)
+ return None
+ try:
+ from .diffusion_memory import _COST_CANVAS_PX, _offload_cost_measurements
+
+ measured = _offload_cost_measurements()
+ denoiser = getattr(pipe, "transformer", None)
+ if measured is None or denoiser is None:
+ return None
+ dtype = next((p.dtype for p in denoiser.parameters() if p.is_floating_point()), None)
+ dtype = getattr(denoiser, "dtype", None) or dtype
+ if dtype is None or not getattr(dtype, "is_floating_point", False) or dtype.itemsize < 2:
+ dtype = _torch().float16
+ if dtype == _torch().float32:
+ dtype = _torch().float16
+ tflops = _gemm_tflops(int(selection.primary), dtype)
+ if tflops is None:
+ return None
+ dit = _denoiser_mib(sizes)
+ resident = int(getattr(plan, "resident_transformer_mib", None) or 0)
+ streamed = max(0, dit - resident)
+ params = sum(p.numel() for p in denoiser.parameters())
+ transfer, compute, exposed = stream_exposure(
+ streamed, params, h2d_gib_s = measured[0], tflops = tflops, side_px = _COST_CANVAS_PX
+ )
+ except Exception: # noqa: BLE001 - unpriced: the split is tried as before
+ return None
+ try:
+ _STREAM_ESTIMATES[pipe] = {
+ "streamed_mib": streamed,
+ "h2d_gib_s": round(measured[0], 2),
+ "tflops": round(tflops, 1),
+ "transfer_s": round(transfer, 3),
+ "compute_s": round(compute, 3),
+ "exposed": round(exposed, 3),
+ }
+ except TypeError:
+ pass
+ if exposed >= STREAM_HIDDEN_FRACTION:
+ return None
+ if streamed > 0:
+ # The primary would stream part of the denoiser while the partner could hold it whole: a single image then
+ # depends on which card runs cooler (FLUX.1-dev split -7.7% to +4.4%, Klein-9B 0% to +14% across T4 VMs), so
+ # main's split stays and one image is never slower than main. The order applies when the primary holds the
+ # whole denoiser (only encoders stream), where one card is the resident speed and the copy adds batches.
+ return None
+ return (
+ f"one card plus a copy on cuda:{partner}: streaming is hidden behind compute ({_gib(streamed)} per step in "
+ f"{transfer:.2f} s against {compute:.2f} s, {exposed:.0%} exposed at {_COST_CANVAS_PX}px), so a split would "
+ f"not be faster, while the copy splits even batches and guided steps; {STREAM_GATE_ENV}=0 tries the split"
+ )
+
+
+def _copy_partner(pipe: Any, selection: Any) -> Optional[int]:
+ """The idle same-model card a whole copy of the denoiser fits on (:func:`automatic_partner`), or None. Sized from
+ nvidia-smi for the secondaries, so no context is opened on them."""
+ try:
+ denoiser = getattr(pipe, "transformer", None)
+ need = (
+ sum(t.numel() * t.element_size() for t in (*denoiser.parameters(), *denoiser.buffers()))
+ >> 20
+ )
+ probes = probe_devices((selection.primary, *selection.secondaries))
+ primary = probes.get(int(selection.primary))
+ cards = [
+ probes[o]
+ for o in selection.secondaries
+ if probes.get(o) is not None and probes[o].free_mib is not None
+ ]
+ if primary is None:
+ return None
+ pick, _why = automatic_partner(primary, cards, need)
+ return pick
+ except Exception: # noqa: BLE001 - unsized: the split is tried as before
+ return None
+
+
+_STREAM_ESTIMATES: "weakref.WeakKeyDictionary[Any, dict]" = weakref.WeakKeyDictionary()
+
+
def refine_plan_for_multigpu(
pipe: Any,
plan: Any,
@@ -868,6 +1035,19 @@ def refine_plan_for_multigpu(
rejected.append(_one_card_rejection(plan, primary))
def layer_split_or_plan() -> Any:
+ hidden = None if layer_force else _hidden_stream_reason(pipe, plan, sizes, selection)
+ if hidden is not None:
+ rejected.append(PlacementCandidate("layer_split", hidden))
+ _record_decision(
+ pipe,
+ selection,
+ "offload",
+ f"one card with '{plan.offload_policy}' offload on cuda:{primary} ({hidden})",
+ rejected,
+ )
+ if logger is not None:
+ logger.info("diffusion.multigpu: layer split not used (%s)", hidden)
+ return plan
refined = refine_plan_for_layer_split(
pipe,
plan,
@@ -1463,6 +1643,12 @@ def placement_status(pipe: Any) -> Optional[dict[str, Any]]:
if info is None and cfg is None and declined is None and layer is None and decision is None:
return None
out: dict[str, Any] = dict(info or {})
+ try:
+ stream = _STREAM_ESTIMATES.get(pipe)
+ except TypeError:
+ stream = None
+ if stream is not None:
+ out["stream_estimate"] = dict(stream)
from .diffusion_layer_split import layer_split_stats
live = {
diff --git a/studio/backend/tests/test_diffusion_auto_multigpu_policy.py b/studio/backend/tests/test_diffusion_auto_multigpu_policy.py
index fd032927ca7..dfef681df37 100644
--- a/studio/backend/tests/test_diffusion_auto_multigpu_policy.py
+++ b/studio/backend/tests/test_diffusion_auto_multigpu_policy.py
@@ -449,6 +449,85 @@ def test_weight_only_lora_path_mirrors_before_it_would_drop_the_replica():
)
+# --- layer split only where streaming is exposed ------------------------------------------------------------------
+
+
+ "streamed_mib, params, h2d, tflops, side, expect_hidden",
+ [
+ # FLUX.1-dev int8 on a T4: about 7 GiB streamed per step at ~11 GiB/s against 12B params of compute at 1024px
+ (7 * 1024, 12e9, 11.0, 40.0, 1024, True),
+ # the same model at 512px: still hidden (about 3% exposed)
+ (7 * 1024, 12e9, 11.0, 40.0, 512, True),
+ # a small DiT streaming most of its weights (Z-Image-Turbo at 512px): exposed
+ (10 * 1024, 6e9, 11.0, 40.0, 512, False),
+ # nothing streamed: hidden
+ (0, 9e9, 11.0, 40.0, 1024, True),
+ ],
+)
+def test_stream_exposure_table(streamed_mib, params, h2d, tflops, side, expect_hidden):
+ transfer, compute, exposed = mg.stream_exposure(
+ streamed_mib, params, h2d_gib_s = h2d, tflops = tflops, side_px = side
+ )
+ assert (exposed < mg.STREAM_HIDDEN_FRACTION) is expect_hidden
+ assert 0.0 <= exposed <= 1.0 and transfer >= 0 and compute > 0
+
+
+def test_split_gate_applies_only_to_automatic_group_plans(monkeypatch):
+ class Plan:
+ offload_policy = "group"
+ resident_transformer_mib = (
+ 8 * 1024
+ ) # the whole denoiser resident on the primary (only encoders stream)
+
+ class Sel:
+ automatic = True
+ primary = 0
+
+ class Explicit(Sel):
+ automatic = False
+
+ class Pipe:
+ transformer = FluxTransformer2DModel().half()
+
+ class Other:
+ transformer = torch.nn.Linear(64, 64).half()
+
+ sizes = {"transformer": (8 * 1024, "dit")}
+ from core.inference import diffusion_memory as dm
+
+ monkeypatch.setattr(dm, "_offload_cost_measurements", lambda: (11.0, 10.0))
+ monkeypatch.setattr(mg, "_copy_partner", lambda pipe, sel: 1)
+ monkeypatch.setattr(mg, "_gemm_tflops", lambda o, d: 1e-9)
+ # a family whose copy cannot serve its requests (no batch split): the split stays
+ assert mg._hidden_stream_reason(Other(), Plan(), sizes, Sel()) is None
+ monkeypatch.setattr(mg, "_gemm_tflops", lambda o, d: 1e-9) # compute dominates: hidden
+ pipe = Pipe()
+ assert "hidden behind compute" in mg._hidden_stream_reason(pipe, Plan(), sizes, Sel())
+ monkeypatch.setattr(
+ mg, "_copy_partner", lambda pipe, sel: None
+ ) # the copy does not fit: the split stays
+ assert mg._hidden_stream_reason(pipe, Plan(), sizes, Sel()) is None
+ monkeypatch.setattr(mg, "_copy_partner", lambda pipe, sel: 1)
+ assert mg._hidden_stream_reason(pipe, Plan(), sizes, Explicit()) is None
+ capacity = Plan()
+ capacity.offload_policy = "sequential"
+ assert mg._hidden_stream_reason(pipe, capacity, sizes, Sel()) is None
+ monkeypatch.setenv(mg.STREAM_GATE_ENV, "0")
+ assert mg._hidden_stream_reason(pipe, Plan(), sizes, Sel()) is None
+ monkeypatch.delenv(mg.STREAM_GATE_ENV)
+ streaming = Plan()
+ streaming.resident_transformer_mib = (
+ 1024 # 7 GiB of the denoiser streams on the primary: main's split stays
+ )
+ assert mg._hidden_stream_reason(pipe, streaming, sizes, Sel()) is None
+ monkeypatch.setattr(
+ mg, "_gemm_tflops", lambda o, d: 1e9
+ ) # compute negligible: exposed, split tried
+ assert mg._hidden_stream_reason(pipe, streaming, sizes, Sel()) is None
+ assert mg._STREAM_ESTIMATES[pipe]["exposed"] > 0.9
+
+
def test_dynamic_nvfp4_layers_keep_the_batch_whole():
"""A layer that takes its activation scale from each call's amax (ComfyUI's dynamic NVFP4) would quantise each half
against its own maximum: the split is refused and the render matches one card."""