Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

24.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Opt-in static step skip (transformer_cache=static): schedule fixed per generation outside the
forward, keeps fullgraph compile and CUDA graph; GraphedForward installed later goes under it"""

from __future__ import annotations

import contextlib
import inspect
import os
from functools import update_wrapper
from typing import Any, Optional

from .diffusion_cache import TC_STATIC

MODE_REUSE = "reuse"
MODE_TAYLOR1 = "taylor1"
SKIP_MODES = (MODE_REUSE, MODE_TAYLOR1)

DEFAULT_MODE = MODE_TAYLOR1
DEFAULT_HEAD = 0.2
DEFAULT_TAIL = 0.1
DEFAULT_EVERY = 2

# Fewer steps leave no middle worth skipping, and a distilled few-step model has no step to spare.
STATIC_MIN_STEPS = 12
# Prefix-KV DiTs (Qwen-Image-2.1, FLUX.2 klein KV) return extra prompt rows on step 0, so reuse starts at step 1.
MIN_HEAD_STEPS = 2

ENV_MODE = "UNSLOTH_STATIC_SKIP_MODE"
ENV_HEAD = "UNSLOTH_STATIC_SKIP_HEAD"
ENV_TAIL = "UNSLOTH_STATIC_SKIP_TAIL"
ENV_EVERY = "UNSLOTH_STATIC_SKIP_EVERY"

_SLOT = "_unsloth_static_skip"


def _torch():
    import torch
    return torch


def _is_tensor(obj: Any) -> bool:
    try:
        is_tensor = getattr(_torch(), "is_tensor", None)
    except Exception:  # noqa: BLE001 - no torch, no tensors
        return False
    return bool(callable(is_tensor) and is_tensor(obj))


def static_schedule(
    steps: Optional[int],
    *,
    head: float = DEFAULT_HEAD,
    tail: float = DEFAULT_TAIL,
    every: int = DEFAULT_EVERY,
    min_steps: int = STATIC_MIN_STEPS,
) -> tuple:
    """Per step, compute (True) or skip (False); empty (compute all) below ``min_steps`` (never below
    ``STATIC_MIN_STEPS``)."""
    try:
        n = int(steps)  # type: ignore[arg-type]
        every = int(every)
        floor = max(STATIC_MIN_STEPS, int(min_steps))
    except (TypeError, ValueError):
        return ()
    if n < floor or every < 2:
        return ()
    first = max(MIN_HEAD_STEPS, round(n * float(head)))
    last = max(1, round(n * float(tail)))
    return tuple(i < first or i >= n - last or (i - first) % every == 0 for i in range(n))


def static_skip_settings(env: Optional[dict] = None, logger: Any = None) -> dict:
    env = os.environ if env is None else env
    out = {
        "mode": DEFAULT_MODE,
        "head": DEFAULT_HEAD,
        "tail": DEFAULT_TAIL,
        "every": DEFAULT_EVERY,
    }
    mode = str(env.get(ENV_MODE, "") or "").strip().lower()
    if mode in SKIP_MODES:
        out["mode"] = mode
    elif mode and logger is not None:
        logger.warning("diffusion.step_skip: ignoring %s=%r", ENV_MODE, mode)
    for key, name in (("head", ENV_HEAD), ("tail", ENV_TAIL)):
        raw = env.get(name)
        if raw in (None, ""):
            continue
        try:
            value = float(raw)
            if not 0.0 <= value < 1.0:
                raise ValueError(raw)
            out[key] = value
        except (TypeError, ValueError):
            if logger is not None:
                logger.warning("diffusion.step_skip: ignoring %s=%r", name, raw)
    if out["head"] + out["tail"] >= 1.0:
        out["head"], out["tail"] = DEFAULT_HEAD, DEFAULT_TAIL
    raw = env.get(ENV_EVERY)
    if raw not in (None, ""):
        try:
            value = int(raw)
            if value < 2:
                raise ValueError(raw)
            out["every"] = value
        except (TypeError, ValueError):
            if logger is not None:
                logger.warning("diffusion.step_skip: ignoring %s=%r", ENV_EVERY, raw)
    return out


def _every_override_valid(env: dict) -> bool:
    try:
        return int(env.get(ENV_EVERY)) >= 2
    except (TypeError, ValueError):
        return False


def auto_static_settings(
    plan: dict,
    env: Optional[dict] = None,
    logger: Any = None,
) -> dict:
    """Knobs for an AUTO static skip: the family's measured ``every`` and ``min_steps`` over the defaults; an
    explicit ``UNSLOTH_STATIC_SKIP_*`` value still wins (benchmarking)."""
    env = os.environ if env is None else env
    out = static_skip_settings(env, logger)
    if out["every"] == DEFAULT_EVERY and plan.get("every") and not _every_override_valid(env):
        out["every"] = int(plan["every"])
    out["min_steps"] = int(plan.get("min_steps") or STATIC_MIN_STEPS)
    out["auto"] = True
    return out


def _call_signature(args: tuple, kwargs: dict) -> tuple:
    """Reuse key: latent shape, prefix-KV mode (an ``extract`` call returns a longer sequence), container."""
    hidden = kwargs.get("hidden_states", args[0] if args else None)
    if _is_tensor(hidden):
        shape = tuple(hidden.shape)
    elif isinstance(hidden, (list, tuple)) and hidden and all(_is_tensor(h) for h in hidden):
        shape = tuple(tuple(h.shape) for h in hidden)
    else:
        shape = None
    mode = kwargs.get("kv_cache_mode")
    if not isinstance(mode, (str, type(None))):
        mode = repr(mode)
    return (shape, mode, kwargs.get("return_dict", True) is False)


def _same_shape_tensors(items: list) -> bool:
    return (
        bool(items)
        and all(_is_tensor(t) for t in items)
        and len({(tuple(t.shape), t.dtype, t.device) for t in items}) == 1
    )


def _split_output(out: Any) -> tuple:
    """(noise prediction, rebuild) for a reproducible container, else (None, None) (FLUX.2 klein KV's extract step)."""
    if _is_tensor(out):
        return out, lambda v: v
    if type(out) is tuple:
        if len(out) == 1 and _is_tensor(out[0]):
            return out[0], lambda v: (v,)
        if len(out) == 1 and type(out[0]) is list and _same_shape_tensors(out[0]):
            return _torch().stack(out[0]), lambda v: (list(v.unbind(0)),)
        # Joint (video, audio) predictions (MiniMax-H3), per stream; a non-tensor member (klein KV cache) declines.
        if len(out) >= 2 and all(_is_tensor(v) for v in out):
            return _Joint(out), lambda v: tuple(v.parts)
        return None, None
    keys = getattr(out, "keys", None)
    if callable(keys) and callable(getattr(out, "to_tuple", None)):
        try:
            names = list(keys())
            if len(names) != 1:
                return None, None
            name = names[0]
            value = out[name]
        except Exception:  # noqa: BLE001 - not the mapping it looked like
            return None, None
        cls = type(out)
        if _is_tensor(value):
            return value, lambda v: cls(**{name: v})
        # Z-Image img2img / inpaint omit return_dict=False: Transformer2DModelOutput(sample=list).
        if type(value) is list and _same_shape_tensors(value):
            return _torch().stack(value), lambda v: cls(**{name: list(v.unbind(0))})
        return None, None
    return None, None


class _Joint:
    """Several predictions from one call, stored and extrapolated stream by stream."""

    __slots__ = ("parts",)

    def __init__(self, parts: Any) -> None:
        self.parts = tuple(parts)

    @property
    def shape(self) -> tuple:
        return tuple(tuple(p.shape) for p in self.parts)

    def detach(self) -> "_Joint":
        return _Joint(p.detach() for p in self.parts)

    def clone(self) -> "_Joint":
        return _Joint(p.clone() for p in self.parts)


_TIMESTEP_NAMES = ("timestep", "timesteps", "t")


def _timestep_slot(signature: Any) -> tuple:
    """(name, positional index or None) of the timestep parameter; Z-Image passes it positionally."""
    try:
        params = list(signature.parameters.values())
    except Exception:  # noqa: BLE001
        return "timestep", None
    names = {p.name: i for i, p in enumerate(params)}
    for name in _TIMESTEP_NAMES:
        if name in names:
            index = names[name]
            positional = all(
                p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD) for p in params[: index + 1]
            )
            return name, index if positional else None
    return "timestep", None


def _timestep_of(
    args: tuple,
    kwargs: dict,
    slot: tuple = ("timestep", None),
) -> Any:
    """The call's timestep as a one-element float32 tensor on its own device (no host sync).

    The max: a per-token ``mask * t`` (Wan2.2 TI2V, LTX) is 0 on conditioned tokens; taylor1 would silently reuse."""
    name, index = slot
    t = kwargs.get(name)
    if t is None and index is not None and index < len(args):
        t = args[index]
    try:
        if _is_tensor(t):
            return t.detach().reshape(-1).float().amax().reshape(1).clone()
        if isinstance(t, (int, float)) and not isinstance(t, bool):
            return _torch().tensor([float(t)], dtype = _torch().float32)
    except Exception:  # noqa: BLE001 - no timestep, taylor1 falls back to reuse
        return None
    return None


# MiniMax-H3 token_tags of the (video, audio) output streams; each stream steps its own shifted schedule.
_JOINT_STREAM_TAGS = (0, 2)


def _stream_timesteps(kwargs: dict) -> Optional[tuple]:
    """Per-stream target timestep: min over the stream's rows (conditioning rows sit at or above it); no host sync."""
    torch = _torch()
    ts, idx, tags = kwargs.get("timestep"), kwargs.get("timestep_indices"), kwargs.get("token_tags")
    if not (_is_tensor(ts) and _is_tensor(idx) and _is_tensor(tags)):
        return None
    try:
        rows = ts.detach().reshape(-1).float().index_select(0, idx.detach().reshape(-1).long())
        tags = tags.detach().reshape(-1).to(rows.device)
        inf = torch.full_like(rows, float("inf"))
        return tuple(
            torch.where(tags == tag, rows, inf).amin().reshape(1).clone()
            for tag in _JOINT_STREAM_TAGS
        )
    except Exception:  # noqa: BLE001 - shared timestep instead
        return None


def _pick_t(t: Any, i: int, n: int) -> Any:
    if isinstance(t, tuple):
        return t[i] if len(t) == n else _torch().stack(t).amax().reshape(1)
    return t


def _extrapolate(v0: Any, t0: Any, v1: Any, t1: Any, t: Any) -> Any:
    if isinstance(v1, _Joint):
        n = len(v1.parts)
        return _Joint(
            _extrapolate(a, _pick_t(t0, i, n), b, _pick_t(t1, i, n), _pick_t(t, i, n))
            for i, (a, b) in enumerate(zip(v0.parts, v1.parts))
        )
    t0, t1, t = (_pick_t(x, 0, 0) for x in (t0, t1, t))
    torch = _torch()
    device = v1.device
    t0, t1, t = t0.to(device), t1.to(device), t.to(device)
    step = t1 - t0
    safe = torch.where(step != 0, step, torch.ones_like(step))
    ratio = torch.where(step != 0, (t - t1) / safe, torch.zeros_like(step))
    # A stream with no rows in the call reads inf: reuse instead of a NaN.
    ratio = torch.where(torch.isfinite(ratio), ratio, torch.zeros_like(ratio))
    ratio = ratio.reshape([1] * v1.ndim) if v1.ndim else ratio.reshape(())
    v1f = v1.float()
    return (v1f + (v1f - v0.float()) * ratio).to(v1.dtype)


class _Stored:
    __slots__ = ("sig", "t", "value", "rebuild")

    def __init__(self, sig: tuple, t: Any, value: Any, rebuild: Any) -> None:
        self.sig, self.t, self.value, self.rebuild = sig, t, value, rebuild


class StaticStepSkip:
    """Outer forward layer skipping scheduled calls per CFG branch (``cache_context`` name, else call ordinal)."""

    _unsloth_outer_forward = True

    def __init__(
        self,
        module: Any,
        *,
        inner: Any = None,
        mode: str = DEFAULT_MODE,
        head: float = DEFAULT_HEAD,
        tail: float = DEFAULT_TAIL,
        every: int = DEFAULT_EVERY,
        min_steps: int = STATIC_MIN_STEPS,
        auto: bool = False,
        logger: Any = None,
    ) -> None:
        # First, so nothing update_wrapper copies over can shadow the state set below.
        cls_forward = type(module).forward.__get__(module)
        try:
            update_wrapper(self, cls_forward)
        except Exception:  # noqa: BLE001
            pass
        self._t_slot = ("timestep", None)
        try:
            # Pipelines filter kwargs by inspect.signature(transformer.forward); keep the real one visible.
            self.__signature__ = inspect.signature(cls_forward)
            self._t_slot = _timestep_slot(self.__signature__)
        except Exception:  # noqa: BLE001
            pass
        self.module = module
        self.inner = inner
        self.mode = mode if mode in SKIP_MODES else DEFAULT_MODE
        self.head, self.tail, self.every = float(head), float(tail), int(every)
        self.min_steps = int(min_steps)
        self.auto = bool(auto)
        self.logger = logger
        self.armed = True
        self.context: Any = None
        self._warned_container = False
        self.stats = {"calls": 0, "computed": 0, "skipped": 0}
        self.last_stats = dict(self.stats)
        # Account whose generation produced the counters; status shows them to that account only.
        self.owner: Optional[str] = None
        # Of the last armed forward: the post-render reset empties the plan but status still reports it.
        self.planned_skips = 0
        self.reset(None)

    def reset(
        self,
        steps: Optional[int],
        *,
        step_signal: bool = False,
        keep_stats: bool = False,
        owner: Optional[str] = None,
        compute_all: bool = False,
    ) -> "StaticStepSkip":
        """Start a forward of ``steps`` denoise steps (None: compute all); ``keep_stats`` sums across chunks.
        ``owner`` (arming only) rebinds the counters; a new owner never inherits the previous one's.
        ``compute_all``: count this generation but skip nothing (an auto skip on an unmeasured workflow)."""
        self.plan = (
            static_schedule(
                steps,
                head = self.head,
                tail = self.tail,
                every = self.every,
                min_steps = self.min_steps,
            )
            if self.armed and not compute_all
            else ()
        )
        skips = [i for i, compute in enumerate(self.plan) if not compute]
        self.last_skip = skips[-1] if skips else -1
        if steps is not None:
            self.planned_skips = len(skips)
        self.step_signal = bool(step_signal)
        # An empty schedule still counts, so status never reports the previous generation's skips.
        self.counting = steps is not None
        self.steps_ended = 0
        self.ordinal = 0
        self.counters: dict = {}
        self.history: dict = {}
        if keep_stats:
            return self
        if self.stats["calls"] or self.counting:
            # Kept for status past the per-call reset; arming a new generation clears it (no stale counts).
            self.last_stats = dict(self.stats)
        self.stats = {"calls": 0, "computed": 0, "skipped": 0}
        if steps is not None and owner != self.owner:
            # Counters cleared before the owner moves: static_skip_view reads the owner first.
            self.last_stats = dict(self.stats)
            self.owner = owner
        return self

    def step_end(self) -> None:
        self.steps_ended += 1
        self.ordinal = 0

    def describe(self) -> dict:
        return {
            "mode": self.mode,
            "head": self.head,
            "tail": self.tail,
            "every": self.every,
            "min_steps": self.min_steps,
            "auto": self.auto,
            "armed": bool(self.armed),
            "planned_skips": self.planned_skips,
            "stats": dict(self.stats if self.stats["calls"] else self.last_stats),
        }

    def _forward(self, args: tuple, kwargs: dict) -> Any:
        inner = self.inner
        if inner is None:
            return type(self.module).forward(self.module, *args, **kwargs)
        return inner(*args, **kwargs)

    def __call__(self, *args, **kwargs):
        if not self.plan:
            if self.counting:
                self.stats["calls"] += 1
                self.stats["computed"] += 1
            return self._forward(args, kwargs)
        if self.context is not None:
            key = self.context
        else:
            key = f"#{self.ordinal}" if self.step_signal else None
        self.ordinal += 1
        if self.step_signal:
            step = self.steps_ended
        else:
            step = self.counters.get(key, 0)
            self.counters[key] = step + 1
        self.stats["calls"] += 1
        sig = _call_signature(args, kwargs)
        history = self.history.get(key)
        if (
            0 <= step < len(self.plan)
            and not self.plan[step]
            and history
            and history[-1].sig == sig
        ):
            try:
                out = self._skipped(history, args, kwargs)
            except Exception as exc:  # noqa: BLE001 - a skip that cannot be built computes instead
                out = None
                if self.logger is not None:
                    self.logger.debug("diffusion.step_skip: computing step %s (%s)", step, exc)
            if out is not None:
                self.stats["skipped"] += 1
                return out
        out = self._forward(args, kwargs)
        self.stats["computed"] += 1
        if step < self.last_skip:
            self._remember(key, sig, args, kwargs, out)
        return out

    def _remember(self, key: Any, sig: tuple, args: tuple, kwargs: dict, out: Any) -> None:
        value, rebuild = _split_output(out)
        if value is None:
            self.history.pop(key, None)
            if not self._warned_container and self.logger is not None:
                self._warned_container = True
                self.logger.info(
                    "diffusion.step_skip: %s returned a %s, which a skipped step cannot "
                    "reproduce; those calls compute",
                    type(self.module).__name__,
                    type(out).__name__,
                )
            return
        # Cloned: the pipeline may keep or modify the tensor it was handed.
        t = (_stream_timesteps(kwargs) if isinstance(value, _Joint) else None) or _timestep_of(
            args, kwargs, self._t_slot
        )
        stored = _Stored(sig, t, value.detach().clone(), rebuild)
        keep = 2 if self.mode == MODE_TAYLOR1 else 1
        history = self.history.setdefault(key, [])
        history.append(stored)
        del history[:-keep]

    def _skipped(self, history: list, args: tuple, kwargs: dict) -> Any:
        last = history[-1]
        value = None
        if self.mode == MODE_TAYLOR1 and len(history) >= 2:
            prev = history[-2]
            t = (
                _stream_timesteps(kwargs) if isinstance(last.value, _Joint) else None
            ) or _timestep_of(args, kwargs, self._t_slot)
            # Only same-shape outputs combine: a prefix-KV step 0 is longer than every later step.
            if (
                prev.sig == last.sig
                and tuple(prev.value.shape) == tuple(last.value.shape)
                and None not in (t, prev.t, last.t)
            ):
                value = _extrapolate(prev.value, prev.t, last.value, last.t, t)
        if value is None:
            value = last.value.clone()
        return last.rebuild(value)


def _context_wrapper(skip: StaticStepSkip, module: Any, prior: Any) -> Any:
    target = prior if prior is not None else getattr(type(module), "cache_context")

    @contextlib.contextmanager
    def cache_context(*args, **kwargs):
        name = args[0] if args else kwargs.get("name")
        previous = skip.context
        skip.context = name
        try:
            if prior is not None:
                with target(*args, **kwargs):
                    yield
            else:
                with target(module, *args, **kwargs):
                    yield
        finally:
            skip.context = previous

    cache_context._unsloth_static_skip = skip
    return cache_context


def _find(pipe: Any) -> Optional[StaticStepSkip]:
    transformer = getattr(pipe, "transformer", None)
    try:
        skip = transformer.__dict__.get(_SLOT) if transformer is not None else None
    except Exception:  # noqa: BLE001 - no instance dict
        return None
    return skip if isinstance(skip, StaticStepSkip) else None


def install_static_step_skip(
    pipe: Any,
    *,
    settings: Optional[dict] = None,
    logger: Any = None,
) -> Optional[str]:
    """``TC_STATIC`` or None (uncached). Never sets ``_unsloth_step_cache``, which runs the CUDA graph eager."""
    transformer = getattr(pipe, "transformer", None)
    if transformer is None:
        _warn(logger, "pipeline has no transformer")
        return None
    # A second denoiser takes part of the CFG trajectory this layer would not see.
    for attr in ("transformer_2", "unconditional_transformer"):
        other = getattr(pipe, attr, None)
        if other is not None and other is not transformer:
            _warn(logger, f"pipeline runs a second denoiser ({attr})")
            return None
    if not callable(getattr(type(transformer), "forward", None)):
        _warn(logger, "transformer has no forward")
        return None
    knobs = settings if settings is not None else static_skip_settings(logger = logger)
    existing = _find(pipe)
    if existing is not None:
        existing.armed = True
        existing.mode = knobs.get("mode", existing.mode)
        existing.head = float(knobs.get("head", existing.head))
        existing.tail = float(knobs.get("tail", existing.tail))
        existing.every = int(knobs.get("every", existing.every))
        existing.min_steps = int(knobs.get("min_steps", STATIC_MIN_STEPS))
        existing.auto = bool(knobs.get("auto", False))
        existing.reset(None)
        return TC_STATIC
    try:
        slots = transformer.__dict__
        skip = StaticStepSkip(
            transformer,
            inner = slots.get("forward"),
            mode = knobs.get("mode", DEFAULT_MODE),
            head = knobs.get("head", DEFAULT_HEAD),
            tail = knobs.get("tail", DEFAULT_TAIL),
            every = knobs.get("every", DEFAULT_EVERY),
            min_steps = knobs.get("min_steps", STATIC_MIN_STEPS),
            auto = knobs.get("auto", False),
            logger = logger,
        )
        prior_ctx = slots.get("cache_context")
        skip._prior_ctx = prior_ctx
        skip._ctx = None
        if prior_ctx is not None or callable(getattr(type(transformer), "cache_context", None)):
            skip._ctx = _context_wrapper(skip, transformer, prior_ctx)
            slots["cache_context"] = skip._ctx
        # Written into the instance dict, like GraphedForward: nn.Module.__setattr__ would inspect it.
        slots["forward"] = skip
        slots[_SLOT] = skip
    except Exception as exc:  # noqa: BLE001 - best-effort, the load proceeds uncached
        _warn(logger, str(exc))
        return None
    if logger is not None:
        logger.info(
            "diffusion.step_skip: static engaged (mode=%s head=%s tail=%s every=%s min_steps=%s%s)",
            skip.mode,
            skip.head,
            skip.tail,
            skip.every,
            skip.min_steps,
            ", auto" if skip.auto else "",
        )
    return TC_STATIC


def reset_static_step_skip(
    pipe: Any,
    steps: Optional[int],
    *,
    step_signal: bool = False,
    keep_stats: bool = False,
    owner: Optional[str] = None,
    compute_all: bool = False,
) -> bool:
    skip = _find(pipe)
    if skip is None:
        return False
    skip.reset(
        steps,
        step_signal = step_signal,
        keep_stats = keep_stats,
        owner = owner,
        compute_all = compute_all,
    )
    return True


def mark_step_end(pipe: Any) -> None:
    skip = _find(pipe)
    if skip is not None:
        skip.step_end()


def static_skip_is_auto(pipe: Any) -> bool:
    """Whether the installed layer came from the auto policy (measured per workflow) rather than an explicit ask."""
    skip = _find(pipe)
    return bool(skip is not None and skip.auto)


def static_skip_stats(pipe: Any) -> Optional[dict]:
    skip = _find(pipe)
    return skip.describe() if skip is not None else None


def static_skip_view(pipe: Any) -> tuple:
    """(owner, stats) read in that order, so a new owner is never paired with the previous owner's counts."""
    skip = _find(pipe)
    if skip is None:
        return None, None
    owner = skip.owner
    return owner, skip.describe()


def uninstall_static_step_skip(pipe: Any) -> bool:
    """Restore what the layer wrapped; if something wrapped it later (an offload hook), disarm to a passthrough."""
    skip = _find(pipe)
    if skip is None:
        return False
    skip.armed = False
    skip.reset(None)
    try:
        slots = skip.module.__dict__
        if slots.get("forward") is skip:
            if skip.inner is None:
                slots.pop("forward", None)
            else:
                slots["forward"] = skip.inner
        ctx = getattr(skip, "_ctx", None)
        if ctx is not None and slots.get("cache_context") is ctx:
            prior = getattr(skip, "_prior_ctx", None)
            if prior is None:
                slots.pop("cache_context", None)
            else:
                slots["cache_context"] = prior
        slots.pop(_SLOT, None)
    except Exception:  # noqa: BLE001 - unload must not fail; a disarmed layer is a passthrough
        return False
    return True


def _warn(logger: Any, why: str) -> None:
    if logger is not None:
        logger.warning("diffusion.step_skip: static unavailable (%s); running uncached", why)