Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Ordered optimizer steps and decline reasons, using the same predicates the primitives apply;
primitives still decide at apply time and the plan never overrides a registered family"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Optional
from . import targets
from .profile import ModelProfile, profile_model
# Load order of the optimizer steps (design doc, section A). Names match the audit / ``speed_optims`` keys.
STEP_ORDER: tuple[str, ...] = (
"patches",
"attention_backend",
"step_cache",
"compile_cache",
"vae_single_frame",
"channels_last",
"vae_fp16_decode",
"vae_fused",
"cudnn_benchmark",
"fp16_accum",
"fused_qkv",
"compiled",
"compiled_dequant",
"int8_gemm",
"compiled_vae_decode",
"tf32",
"rocm_query_chunks",
"cuda_graph",
"block_graphs",
"bg_compile",
)
GRAPH_BREAK_DECLINE = "graph-break suspects in a discovered block forward"
SPEED_OFF = "off"
SPEED_EAGER = "eager"
SPEED_DEFAULT = "default"
SPEED_MAX = "max"
@dataclass
class PlannedStep:
name: str
attempt: bool
reason: Optional[str] = None
@dataclass
class OptimizationPlan:
backend: str
family: Optional[str]
speed_mode: str
steps: list[PlannedStep] = field(default_factory = list)
profiles: list[ModelProfile] = field(default_factory = list)
generic: bool = False
# The compile was declined for the model itself (no candidate, every profile declines), not for the device.
compile_declined_by_model: bool = False
def step(self, name: str) -> Optional[PlannedStep]:
for s in self.steps:
if s.name == name:
return s
return None
def reason(self, name: str) -> Optional[str]:
s = self.step(name)
return s.reason if s is not None else None
def family_has_data(family: Any) -> bool:
"""Whether ``family`` comes from a registry (image or video); a bare / synthesized family takes the generic path."""
name = getattr(family, "name", None)
if not isinstance(name, str) or not name:
return False
try:
from ..diffusion_families import _FAMILIES as image_families # noqa: PLC0415
from ..video_families import _FAMILIES as video_families # noqa: PLC0415
except Exception: # noqa: BLE001
return True
return any(f is family or f.name == name for f in tuple(image_families) + tuple(video_families))
# Generic path (a family with no registry data): compile discovered repeated blocks under the runtime guard.
DISCOVER_ENV = "UNSLOTH_DIFFUSION_AUTOOPT_DISCOVER"
def discovery_enabled() -> bool:
import os # noqa: PLC0415
return (os.environ.get(DISCOVER_ENV) or "").strip().lower() not in ("0", "off", "false", "no")
ROCM_PROBE_FAILED = "ROCm regional compile probe failed on this install"
def compile_ineligible_reason(
target: Any,
family: Any,
*,
probe: bool = True,
) -> Optional[str]:
"""Why ``compile_eligible`` refuses, mirroring its checks in order; None when eligible.
``probe=False`` reads only a recorded ROCm probe result and never runs the allocating probe."""
from .. import diffusion_speed as ds # noqa: PLC0415
if not ds.torch_compile_runtime_available():
return "torch.compile unavailable in this process (TORCHDYNAMO_DISABLE or no Triton)"
if not bool(getattr(target, "supports_default_torch_compile", False)):
return "device does not support the default torch.compile profile"
if not bool(getattr(family, "supports_torch_compile", True)):
return "family opts out of torch.compile"
rocm_failed = ROCM_PROBE_FAILED
dtype = getattr(target, "dtype", None)
if ds._is_bfloat16(dtype):
return None if ds.rocm_compile_probe_passes(target, probe = probe) else rocm_failed
if bool(getattr(family, "fp16_incompatible", False)):
return "fp16-incompatible family stays eager outside bf16"
if ds._is_float16(dtype) and ds._fp16_compile_capable(target):
return None if ds.rocm_compile_probe_passes(target, probe = probe) else rocm_failed
return f"dtype {str(dtype).replace('torch.', '')} is not compiled on this device"
def build_plan(
pipe: Any,
target: Any,
*,
family: Any,
backend: str,
speed_mode: str,
is_gguf: bool = False,
offload_active: bool = False,
) -> OptimizationPlan:
"""Predict each step's decision and reason from the same predicates the primitives use."""
from .. import diffusion_speed as ds # noqa: PLC0415
mode = ds.normalize_speed_mode(speed_mode)
generic = not family_has_data(family)
dits = targets.denoiser_dits(pipe)
candidates = dits or [m for m in [targets.denoiser_unet(pipe)] if m is not None]
if generic and not candidates:
candidates = targets.discovered_only(pipe, family)
profiles = [
p for p in (profile_model(m) for m in candidates if hasattr(m, "forward")) if p is not None
]
plan = OptimizationPlan(
backend = backend,
family = getattr(family, "name", None),
speed_mode = mode,
profiles = profiles,
generic = generic,
)
def add(
name: str,
attempt: bool,
reason: Optional[str] = None,
) -> None:
plan.steps.append(PlannedStep(name, attempt, reason))
if mode == SPEED_OFF:
for name in STEP_ORDER:
if name not in ("patches", "attention_backend", "step_cache"):
add(name, False, "speed off")
return plan
on_cuda = getattr(target, "device", None) == "cuda"
compile_reason: Optional[str] = None
probe_pending = False
if mode == SPEED_EAGER:
compile_reason = "eager tier does not compile"
elif (
mode == SPEED_DEFAULT
and is_gguf
and on_cuda
and getattr(family, "supports_torch_compile", True)
and not ds.rocm_compile_switched_off(target)
):
compile_reason = "GGUF compiles the dequant op chain instead"
else:
# The ROCm probe compiles a block on the card: run it only once the model itself can be compiled.
compile_reason = compile_ineligible_reason(target, family, probe = False)
probe_pending = compile_reason is None
if compile_reason is None and ds.fp16_unet_offloaded(
target, pipe, offload_active = offload_active
):
compile_reason = "offloaded fp16 U-Net stays eager (fused QKV costs more transfer than compile saves)"
if compile_reason is None and not candidates:
compile_reason = "no denoiser transformer or whole-compile U-Net"
if compile_reason is None and dits:
declines = [p.compile_decline() for p in profiles]
if generic and discovery_enabled():
# No data: discovered (unverified) blocks get a guarded trial compile instead of a decline.
declines = [
None if (p.discovered_blocks and not p.repeated_blocks) else d
for p, d in zip(profiles, declines)
]
if declines and all(declines):
compile_reason = declines[0]
if probe_pending and compile_reason is not None:
plan.compile_declined_by_model = True
add("compiled", compile_reason is None, compile_reason)
if generic and compile_reason is None:
# No measured data: a discovered block whose forward host-syncs would graph-break on every call.
for p in profiles:
if not p.declared_blocks and p.graph_break_suspects:
suspects = "; ".join(
f"{k}: {', '.join(v)}" for k, v in p.graph_break_suspects.items()
)
step = plan.step("compiled")
step.attempt = False
step.reason = f"{GRAPH_BREAK_DECLINE} ({p.class_name}: {suspects})"
step = plan.step("compiled")
if (
probe_pending
and step is not None
and step.attempt
and not ds.rocm_compile_probe_passes(target)
):
step.attempt = False
step.reason = ROCM_PROBE_FAILED
rocm_off = ds.rocm_compile_switched_off(target)
add(
"compiled_dequant",
bool(is_gguf and on_cuda and not rocm_off),
"not a GGUF transformer"
if not is_gguf
else ("UNSLOTH_DIFFUSION_ROCM_COMPILE=0" if rocm_off else None),
)
tf32_tier = mode in (SPEED_DEFAULT, SPEED_MAX)
tf32_on = ds.tf32_enabled()
add(
"tf32",
tf32_tier and on_cuda and tf32_on,
None
if tf32_tier and tf32_on
else (f"{ds.TF32_ENV}=0" if tf32_tier else "compiled tiers only"),
)
if not getattr(family, "cudnn_benchmark", True):
add(
"cudnn_benchmark",
False,
"family keeps cudnn.benchmark off (per-process conv pick changes pixels)",
)
elif not on_cuda:
add("cudnn_benchmark", False, "not a CUDA device")
else:
add("cudnn_benchmark", True)
return plan
def compile_cache_kwargs(
pipe: Any,
*,
speed_mode: str,
fullgraph: bool,
offload_below_hooks: bool,
deferred: bool = False,
) -> dict[str, Any]:
"""The compile kwargs the compile cache fingerprints. ``deferred``: the image backend's 3rd-image profile, which
always compiles the ``default`` tier with explicit ``dynamic=True``."""
from .. import diffusion_speed as ds # noqa: PLC0415
mode = SPEED_DEFAULT if deferred else ds.normalize_speed_mode(speed_mode)
transformer = getattr(pipe, "transformer", None)
kwargs: dict[str, Any] = {
# A step cache or planned offload graph-breaks, so the bundle keys on the fullgraph the compile uses.
"fullgraph": bool(fullgraph),
# max compiles DiTs with automatic dynamic (None), as does the default tier for torchao weights.
"dynamic": ds.compile_dynamic(
transformer, True if deferred else (None if mode == SPEED_MAX else True)
),
"mode": "max-autotune-no-cudagraphs" if mode == SPEED_MAX else "default",
"vae_decode": ds.vae_decode_compile_allowed(pipe, mode),
}
if offload_below_hooks:
# An offloaded denoiser compiles its blocks below the hooks, a different graph than a traced-hook bundle.
kwargs["below_hooks"] = True
if ds.unet_compiles_regionally(ds._denoiser_unet(pipe), max_autotune = mode == SPEED_MAX):
# Per-block U-Net graphs: never served a whole-module bundle (nor the other way round).
kwargs["unet_graph"] = "blocks"
return kwargs