Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

20.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Fault injection over the VAE OOM ladder: an OOM or a generic error at EVERY mutation point of the recovery and of
its restore, on a fake pipeline (a resident component, resident offload groups, CUDA graphs, a VAE with an
instance-bound ``_decode`` override and a later ``decode`` wrapper, a sliced batch decode). After each decode
(returned or raised) the system must be consistent: every module on its card or with a working deferred return that
resolves the card; resident groups all re-pinned or all streamed with a deferred re-pin; the wrapper stack exact after
uninstall; later slices on the recovered path; the surfaced error the decode's own OOM when nothing recovered.

CPU only: a float64 tensor stands in for one on the card, float32 for one on the host."""

import itertools
import types

import pytest

torch = pytest.importorskip("torch")

from core.inference import decode_layers as L  # noqa: E402
from core.inference import diffusion_cuda_graph as G  # noqa: E402
from core.inference import diffusion_memory as M  # noqa: E402
from core.inference import diffusion_vae_oom as O  # noqa: E402

CARD, HOST = torch.float64, torch.float32


class FakeOOM(RuntimeError):
    def __init__(self, where = ""):
        super().__init__(f"CUDA out of memory. Tried to allocate 2.00 GiB ({where})")


class Injected(ValueError):
    pass


class Device:
    type = "cuda"


DEV = Device()


class World:
    """The injector: counts mutation points; raises at the ``at``-th one (``kind`` "oom" or "error")."""

    def __init__(
        self,
        at = None,
        kind = "oom",
    ):
        self.at, self.kind, self.n, self.points, self.fired = at, kind, 0, [], None

    def __call__(self, point):
        self.points.append(point)
        self.n += 1
        if self.at is not None and self.n in (
            self.at if isinstance(self.at, tuple) else (self.at,)
        ):
            self.fired = point
            raise FakeOOM(point) if self.kind == "oom" else Injected(point)


def _move(module, dtype, world, tag):
    """Parameter by parameter, a mutation point before each: a fault leaves the module part moved."""
    for i, p in enumerate(list(module.parameters())):
        world(f"{tag}:param{i}")
        p.data = p.data.to(dtype)


class Group:
    def __init__(self):
        self._unsloth_resident = True


class Vae(torch.nn.Module):
    spatial_compression_ratio = 8

    def __init__(self, budget):
        super().__init__()
        self.use_tiling = False
        self.tile_sample_min_height = 256
        self.tile_sample_stride_height = 192
        self.a = torch.nn.Linear(64, 8, bias = False)
        self.b = torch.nn.Linear(64, 8, bias = False)
        self.budget = budget
        self.fake_hooks = False
        self.slices = []

    def enable_tiling(self):
        self.use_tiling = True

    def _decode(
        self,
        z,
        return_dict = True,
    ):
        streamed = self.fake_hooks
        need = {False: 9, True: 5}[bool(self.use_tiling)] - (3 if streamed else 0)
        self.budget.world(f"decode:slice{self.budget.slice}:{'tiled' if self.use_tiling else 'untiled'}"
                          f"{':streamed' if streamed else ''}")  # fmt: skip
        if self.budget.used() + need > self.budget.cap:
            raise FakeOOM("decode")
        self.slices.append((self.budget.slice, bool(self.use_tiling), streamed))
        return (z + (0.5 if self.use_tiling else 0.0),)

    def decode(
        self,
        z,
        return_dict = True,
    ):
        out = []
        for i in range(z.shape[0]):
            self.budget.slice = i
            out.append(self._decode(z[i : i + 1])[0])
        return (torch.cat(out),)


class Pipe:
    pass


class Budget:
    """Units of card memory: the transformer 4, each resident group 1, the VAE weights 3, plus the decode's need."""

    def __init__(self, cap, world):
        self.cap, self.world, self.slice = cap, world, 0
        self.transformer = self.groups = self.vae = None

    def used(self):
        own = getattr(self.transformer, "_fake_groups", None)
        if own is not None:
            u = sum(2 for g in own if g._unsloth_resident)
        else:
            u = 4 if all(p.dtype == CARD for p in self.transformer.parameters()) else 0
        u += sum(1 for g in self.groups if g._unsloth_resident and g not in (own or ()))
        u += 0 if self.vae.fake_hooks else 3
        return u


def build(
    cap,
    world,
    monkeypatch,
    tiled = False,
    variant = "base",
):
    budget = Budget(cap, world)
    transformer = torch.nn.Linear(8, 8).to(CARD)
    text_encoder = torch.nn.Linear(8, 8, bias = False).to(CARD)
    text_encoder._diffusers_hook = types.SimpleNamespace(hooks = {"group_offloading": 1})
    text_encoder._unsloth_resident_room = 1
    groups = [Group() for _ in range(3)]
    vae = Vae(budget).to(CARD)
    vae.use_tiling = tiled
    budget.transformer, budget.groups, budget.vae = transformer, groups, vae
    pipe = Pipe()
    pipe.components = {"transformer": transformer, "text_encoder": text_encoder, "vae": vae}
    pipe.vae, pipe.transformer, pipe._unsloth_cuda_graphs = vae, transformer, ("graph",)

    def release(
        pipe_,
        need,
        logger = None,
        reason = "",
    ):
        # diffusion_memory.release_resident_groups's contract: bookkeeping before each demotion; a failure part way
        # is caught and the restore for what was released is returned. Its re-pin swallows its own failures (as
        # _keep_groups_resident does); ``strict`` reports groups still streamed.
        released = []

        def restore(strict = False):
            try:
                for g in released:
                    world("groups:repin")
                    g._unsloth_resident = True
            except Exception:  # noqa: BLE001
                pass
            if strict and any(not g._unsloth_resident for g in released):
                raise RuntimeError("re-pinning resident offload groups left some streamed")

        try:
            for g in reversed(groups):
                if g._unsloth_resident:
                    released.append(g)
                    world("groups:release")
                    g._unsloth_resident = False
        except Exception:  # noqa: BLE001
            return restore if released else None
        return restore if released else None

    def apply_offload(v, device):
        world("vae_hooks:add")
        v.fake_hooks = True
        _move(v, HOST, world, "vae_out")

    def remove_offload(v, device):
        world("vae_hooks:remove")
        v.fake_hooks = False
        _move(v, CARD, world, "vae_back")

    def park(v, device):
        v.fake_hooks = False
        _move(v, HOST, world, "vae_park")
        world("vae_hooks:readd")
        v.fake_hooks = True

    def unhook():
        for g in transformer.__dict__.pop("_fake_groups", None) or ():
            groups.remove(g)
        transformer.__dict__.pop("_diffusers_hook", None)
        transformer.__dict__.pop("_unsloth_resident_room", None)

    def hook_resident(
        pipe_,
        device,
        logger = None,
        resident = True,
    ):
        if getattr(transformer, "_fake_groups", None) is not None:
            return False
        world("torchao:hook")
        own = [Group(), Group()]
        for g in own:
            g._unsloth_resident = False
        transformer._fake_groups = own
        groups.extend(own)
        transformer._diffusers_hook = types.SimpleNamespace(hooks = {"group_offloading": 1})
        try:
            _move(transformer, HOST, world, "torchao_out")
            if resident:
                _move(transformer, CARD, world, "torchao_keep")
                for g in own:
                    g._unsloth_resident = True
                transformer._unsloth_resident_room = 1
        except Exception:  # noqa: BLE001
            unhook()
            _move(transformer, CARD, world, "torchao_back")
            return False
        return True

    def return_torchao(
        pipe_,
        device,
        logger = None,
        torch_hooks = None,
    ):
        own = getattr(transformer, "_fake_groups", None)
        if own is not None:
            _move(transformer, CARD, world, "torchao_repin")
            for g in own:
                g._unsloth_resident = True
            transformer._unsloth_resident_room = 1
            return
        unhook()
        _move(transformer, CARD, world, "torchao_back")

    monkeypatch.setattr(M, "release_resident_groups", release)
    monkeypatch.setattr(M, "hook_resident_denoiser", hook_resident)
    monkeypatch.setattr(M, "return_torchao_denoiser", return_torchao, raising = False)
    monkeypatch.setattr(M, "module_forward_hook_ids", lambda m: {}, raising = False)
    monkeypatch.setattr(O, "_holds_torchao", lambda m: variant == "torchao" and m is transformer)
    monkeypatch.setattr(G, "reset_all", lambda handles, strict = False: world("graphs:reset"))
    monkeypatch.setattr(O, "_to_host", lambda m: _move(m, HOST, world, "out"))
    monkeypatch.setattr(O, "_return_to", lambda m, d: _move(m, CARD, world, "back"))
    monkeypatch.setattr(O, "_apply_vae_offload", apply_offload)
    monkeypatch.setattr(O, "_remove_vae_offload", remove_offload)
    monkeypatch.setattr(O, "_park_vae", park)
    monkeypatch.setattr(O, "_whole_on", lambda m, d: all(p.dtype == CARD for p in m.parameters()))
    monkeypatch.setattr(
        O, "_whole_on_host", lambda m: all(p.dtype == HOST for p in m.parameters()), raising = False
    )
    monkeypatch.setattr(
        O, "_vae_hooked", lambda m: bool(getattr(m, "fake_hooks", False)), raising = False
    )
    monkeypatch.setattr(O, "_hooked", lambda m: bool(getattr(m, "fake_hooks", False)) or (
        getattr(m, "_diffusers_hook", None) is not None))  # fmt: skip
    monkeypatch.setattr(O, "_host_free_mib", lambda: 1 if variant == "host_short" else 1 << 20)
    monkeypatch.setattr(O, "_empty_cache", lambda d: None)
    monkeypatch.setattr(O, "_device_of", lambda z: DEV)
    monkeypatch.setattr(O, "_release_cache", lambda z, vae = None: None)

    override = types.MethodType(lambda self, z, *a, **k: Vae._decode(self, z, *a, **k), vae)
    vae._decode = override
    before = {k: vae.__dict__.get(k, "MISSING") for k in ("_decode", "decode")}
    assert O.install(vae, pipe = pipe)
    scope = vae.__dict__["decode"]

    def later(*a, **k):
        return scope(*a, **k)

    later.__wrapped__ = scope
    vae.decode = later
    return pipe, budget, before, override, later


STEPS = {(False, False): 0, (True, False): 1, (True, True): 2}


def check(pipe, budget, before, override, later, world, outcome):
    vae, transformer, text_encoder = (
        pipe.vae,
        pipe.components["transformer"],
        pipe.components["text_encoder"],
    )
    ctx = f"fault {world.kind} at {world.fired!r}, cap {budget.cap}, outcome {outcome!r}"
    # surfaced error: the decode's own OOM when nothing recovered; an injected error only from a decode attempt
    if isinstance(outcome, BaseException):
        if isinstance(outcome, Injected):
            assert world.fired and world.fired.startswith("decode:"), ctx
        else:
            assert isinstance(outcome, FakeOOM), ctx
    # later slices stay on the recovered path (never back to a cheaper mode than an earlier slice used)
    modes = [STEPS[(t, s)] for _, t, s in vae.slices]
    assert modes == sorted(modes), (ctx, vae.slices)
    # a VAE left split (its repair is REQUIRED) never decodes before it is repaired: while the repair still fails,
    # the next decode raises that clear error instead of decoding with the inconsistent VAE
    vdef = vae.__dict__.get(O._DEFERRED_ATTR)
    if vdef is not None:
        # optional only when it is checked to stream whole from the host; anything else is a required repair
        streams = vae.fake_hooks and all(p.dtype == HOST for p in vae.parameters())
        assert vdef.required == (not streams), (ctx, vdef.required, streams)
    if vdef is not None and vdef.required:
        world.at, world.kind = (world.n + 1, world.n + 2), "oom"
        vae.slices.clear()
        with pytest.raises(RuntimeError, match = "cannot return"):
            vae.decode(torch.zeros(1, 4, 1, 2, 2))
        assert vae.slices == [], ctx
    world.at = None  # memory frees up: everything deferred must now find its way back
    budget.cap = 10**6
    deferred = transformer.__dict__.get(O._DEFERRED_ATTR)
    if getattr(transformer, "_fake_groups", None) is not None and deferred is None:
        assert all(p.dtype == (CARD if all(g._unsloth_resident for g in transformer._fake_groups) else HOST)
                   for p in transformer.parameters()), ctx  # fmt: skip
    elif deferred is None:
        assert all(p.dtype == CARD for p in transformer.parameters()), ctx
    else:
        assert (
            deferred.execution_device is DEV and transformer.__dict__.get("_hf_hook") is deferred
        ), ctx
        transformer(torch.zeros(1, 8, dtype = CARD))
        assert all(p.dtype == CARD for p in transformer.parameters()), ctx
        assert (
            O._DEFERRED_ATTR not in transformer.__dict__ and not transformer._forward_pre_hooks
        ), ctx
    resident = [g._unsloth_resident for g in budget.groups]
    if not all(resident):
        anchors = [m for m in (transformer, text_encoder) if O._DEFERRED_ATTR in m.__dict__]
        assert anchors, ctx
        for m in anchors:
            m(torch.zeros(1, 8, dtype = CARD))
        assert all(g._unsloth_resident for g in budget.groups), ctx
    vdef = vae.__dict__.get(O._DEFERRED_ATTR)
    if vdef is None:
        assert not vae.fake_hooks and all(p.dtype == CARD for p in vae.parameters()), ctx
    else:
        assert O.settle_deferred(vae), ctx
        assert not vae.fake_hooks and all(p.dtype == CARD for p in vae.parameters()), ctx
    assert O._SESSION_ATTR not in vae.__dict__, ctx
    # the wrapper stack: uninstall leaves exactly what was there plus the later wrapper (a pass-through below it)
    O.uninstall(vae)
    assert vae.__dict__.get("_decode") is override and vae.__dict__.get("decode") is later, ctx
    assert L.LAYERS_ATTR not in vae.__dict__ and O._RECORD_ATTR not in vae.__dict__, ctx
    vae.slices.clear()
    assert vae.decode(torch.zeros(2, 4, 1, 2, 2))[0].shape[0] == 2 and len(vae.slices) == 2, ctx


def _run(
    cap,
    at,
    kind,
    monkeypatch,
    tiled = False,
    verify = True,
    variant = "base",
):
    world = World(at, kind)
    with monkeypatch.context() as mp:
        pipe, budget, before, override, later = build(cap, world, mp, tiled, variant)
        mp.setattr(O, "_FAULT", world)
        try:
            out = pipe.vae.decode(torch.zeros(3, 4, 1, 2, 2))
            outcome = out[0].shape
        except (FakeOOM, Injected) as exc:
            outcome = exc
        world.decode_points = world.n
        if verify:
            check(pipe, budget, before, override, later, world, outcome)
    return world


CAPS = (20, 13, 8, 2, 1)
VARIANTS = ("base", "torchao", "host_short")


def _cases():
    import pytest as _pytest

    mp = _pytest.MonkeyPatch()
    cases = []
    try:
        for variant, cap, tiled in itertools.product(VARIANTS, CAPS, (False, True)):
            n = _run(cap, None, "oom", mp, tiled, verify = False, variant = variant).decode_points
            cases += [
                (variant, cap, tiled, at, kind)
                for at, kind in itertools.product(range(1, n + 1), ("oom", "error"))
            ]
    finally:
        mp.undo()
    return cases


CASES = _cases()


def test_fault_injection_covers_every_mutation_point():
    assert len(CASES) >= 900, len(CASES)
    kinds = {p.split(":")[0] for v, cap, tiled in itertools.product(VARIANTS, CAPS, (False, True))
             for p in _points(cap, tiled, v)}  # fmt: skip
    for point in ("graphs", "groups", "out", "back", "vae_hooks", "vae_out", "vae_back", "decode",
                  "graphs_reset", "resident_groups", "moved", "vae_offload", "torchao", "torchao_hooks",
                  "torchao_out", "torchao_keep"):  # fmt: skip
        assert point in kinds, (point, kinds)


def _points(
    cap,
    tiled,
    variant = "base",
):
    mp = pytest.MonkeyPatch()
    try:
        world = _run(cap, None, "oom", mp, tiled, verify = False, variant = variant)
    finally:
        mp.undo()
    return world.points[: world.decode_points]


@pytest.mark.parametrize("variant,cap,tiled,at,kind", CASES)
def test_fault_injected_recovery_stays_consistent(variant, cap, tiled, at, kind, monkeypatch):
    _run(cap, at, kind, monkeypatch, tiled, variant = variant)


@pytest.mark.parametrize("variant", VARIANTS)
@pytest.mark.parametrize("cap", CAPS)
def test_unfaulted_ladder_reaches_its_step(cap, variant, monkeypatch):
    world = World()
    with monkeypatch.context() as mp:
        pipe, budget, *_ = build(cap, world, mp, variant = variant)
        try:
            pipe.vae.decode(torch.zeros(3, 4, 1, 2, 2))
            raised = None
        except FakeOOM as exc:
            raised = exc
        stats = pipe.vae._unsloth_vae_oom_stats
        expect = {20: None, 13: "freed", 8: "tiled", 2: "offload", 1: None}[cap]
        if variant == "host_short":
            expect = {20: None, 13: "tiled", 8: None, 2: None, 1: None}[cap]
            if cap in (8, 2):
                assert raised is not None
                return
        assert stats.get("recovered_by") == expect, stats
        assert (raised is not None) == (cap == 1)
        if cap != 20:
            assert len({(t, s) for _, t, s in pipe.vae.slices}) <= 1


def _torchao_double_cases():
    """Item 4285966491: the torchao hook conversion fails (first fault, inside hook_resident_denoiser) AND its own
    return to the card fails (second fault, any later point): the transformer is left split between host and card
    with no hooks, and the session must still leave it on the card, streamed under hooks, or with a deferred return."""
    mp = pytest.MonkeyPatch()
    cases = []
    try:
        for cap, tiled in itertools.product((13, 8, 2), (False, True)):
            pts = _run(cap, None, "oom", mp, tiled, verify = False, variant = "torchao").points
            firsts = [
                i + 1 for i, p in enumerate(pts) if p.startswith(("torchao_out:", "torchao_keep:"))
            ]
            for a1 in firsts:
                n = _run(cap, a1, "oom", mp, tiled, verify = False, variant = "torchao").decode_points
                cases += [
                    (cap, tiled, a1, a2, kind)
                    for a2 in range(a1 + 1, n + 1)
                    for kind in ("oom", "error")
                ]
    finally:
        mp.undo()
    return cases


TORCHAO_DOUBLE = _torchao_double_cases()


def test_torchao_double_faults_reach_the_split_state():
    assert len(TORCHAO_DOUBLE) >= 50, len(TORCHAO_DOUBLE)
    mp = pytest.MonkeyPatch()
    try:
        cap, tiled, a1, a2, kind = next(c for c in TORCHAO_DOUBLE if c[4] == "oom")
        world = _run(cap, (a1, a1 + 1), "oom", mp, tiled, verify = False, variant = "torchao")
    finally:
        mp.undo()
    assert world.points[a1].startswith("torchao_back:"), world.points[a1 - 1 : a1 + 2]


@pytest.mark.parametrize("cap,tiled,a1,a2,kind", TORCHAO_DOUBLE)
def test_torchao_conversion_and_its_return_both_fail(cap, tiled, a1, a2, kind, monkeypatch):
    _run(cap, (a1, a2), kind, monkeypatch, tiled, variant = "torchao")


_RESTORE_POINTS = (":undo", ":settle", "back:", "vae_back:", "vae_park:", "vae_hooks:remove", "vae_hooks:readd",
                   "groups:repin", "torchao_back:", "torchao_repin:", "groups:release")  # fmt: skip


def _restore_double_cases():
    """Item 4287727812: a restore (undo) that fails, then a second fault anywhere after it (in the settle / park /
    deferred path): every Change's settle + defer pairing must leave a coherent module or a REQUIRED repair."""
    mp = pytest.MonkeyPatch()
    cases = []
    try:
        for variant, cap, tiled in itertools.product(VARIANTS, (13, 8, 2), (False, True)):
            pts = _run(cap, None, "oom", mp, tiled, verify = False, variant = variant).points
            firsts = [i + 1 for i, p in enumerate(pts) if any(k in p for k in _RESTORE_POINTS)]
            for a1 in firsts:
                n = _run(cap, a1, "oom", mp, tiled, verify = False, variant = variant).decode_points
                cases += [(variant, cap, tiled, a1, a2) for a2 in range(a1 + 1, min(n, a1 + 8) + 1)]
    finally:
        mp.undo()
    return cases


RESTORE_DOUBLE = _restore_double_cases()


def test_restore_double_faults_cover_the_park_path():
    assert len(RESTORE_DOUBLE) >= 200, len(RESTORE_DOUBLE)
    assert any(c[0] == "base" and c[2] in (False, True) for c in RESTORE_DOUBLE)


@pytest.mark.parametrize("variant,cap,tiled,a1,a2", RESTORE_DOUBLE)
def test_restore_and_its_settle_both_fail(variant, cap, tiled, a1, a2, monkeypatch):
    _run(cap, (a1, a2), "oom", monkeypatch, tiled, variant = variant)