Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Layer split planner (core/inference/diffusion_layer_split_plan.py), CPU only, fake card memory.
The pure packer is driven with card limits directly; the post-load refinement through ``refine_plan_for_multigpu``
with patched loaded sizes, block layouts and secondary limits; the engine is a recording fake."""
from __future__ import annotations
import dataclasses
import pytest
import torch
from core.inference import diffusion_layer_split_plan as lsp
from core.inference import diffusion_memory as dm
from core.inference import diffusion_multigpu as mg
L = lsp.DenoiserLayout
@pytest.fixture(autouse = True)
def _env(monkeypatch):
monkeypatch.delenv(mg.MULTIGPU_ENV, raising = False)
monkeypatch.delenv(mg.CFG_PARALLEL_ENV, raising = False)
monkeypatch.delenv(lsp.LAYER_SPLIT_ENV, raising = False)
# pin the host so the fake [0, 1] cards plan on any machine (ROCm included)
monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)
monkeypatch.setattr(mg, "_capability", lambda o: None)
monkeypatch.setattr(lsp, "_is_rocm", lambda: False)
monkeypatch.setattr(mg, "_rocm_torch", lambda: False)
yield
def _check(split, limits):
"""Every card within its limit, every block placed exactly once in contiguous ranges."""
for o, req in split.card_required_mib:
assert req <= limits[o], (o, req, limits[o])
for name in dict.fromkeys(r.denoiser for r in split.ranges):
rs = [r for r in split.ranges if r.denoiser == name]
assert rs[0].start == 0
for a, b in zip(rs, rs[1:]):
assert a.stop == b.start
return split
def test_packs_a_dit_too_big_for_one_card_over_two():
limits = {0: 18000, 1: 18000}
split, reason = lsp.plan_layer_split(
denoisers = {"transformer": L(1000, (1000,) * 23)},
companions = {"text_encoder": (4000, "text_encoder"), "vae": (300, "other")},
primary = 0,
card_limits = limits,
headroom_mib = 2000,
overhead_mib = 500,
)
assert split is not None, reason
_check(split, limits)
assert split.cards == (0, 1)
assert split.ranges[-1].stop == 23
assert split.text_encoders_card == 1
weights = dict(split.card_weights_mib)
assert sum(weights.values()) == 1000 + 23000 + 4000 + 300
assert weights[0] == 1000 + 300 + sum(r.mib for r in split.ranges if r.card == 0)
assert split.boundaries == 2
assert "layer split" in reason
def test_text_encoders_too_big_for_any_card_stream_and_the_blocks_still_split():
kw = dict(
denoisers = {"transformer": L(1000, (1100,) * 54)},
companions = {"text_encoder": (45000, "text_encoder"), "vae": (300, "other")},
primary = 0,
card_limits = {0: 55000, 1: 55000},
headroom_mib = 6000,
overhead_mib = 1000,
)
split, reason = lsp.plan_layer_split(**kw)
assert split is not None, reason
assert split.text_encoders_streamed and split.text_encoders_card is None
assert split.as_public_dict()["text_encoders_streamed"] is True
assert {r.card for r in split.ranges} == {0, 1}
_check(split, kw["card_limits"])
kw["card_limits"] = {0: 55000, 1: 110000}
split, reason = lsp.plan_layer_split(**kw)
assert split is not None, reason
assert not split.text_encoders_streamed and split.text_encoders_card == 1
def test_streamed_encoders_are_paged_not_moved_and_undo_removes_their_hooks(monkeypatch):
calls = {"stream": [], "engine": [], "removed": []}
class Comp(torch.nn.Module):
def __init__(self):
super().__init__()
self.w = torch.nn.Parameter(torch.zeros(1))
self.moved = []
def to(self, *a, **k):
self.moved.append(a[0] if a else k.get("device"))
return self
class Pipe:
def __init__(self):
self.transformer = Comp()
self.text_encoder = Comp()
self.vae = Comp()
self.components = {
"transformer": self.transformer,
"text_encoder": self.text_encoder,
"vae": self.vae,
}
pipe = Pipe()
split = lsp.LayerSplit(
primary = 0,
cards = (0, 1),
ranges = (
lsp.BlockRange("transformer", 0, 1, 0, 1),
lsp.BlockRange("transformer", 1, 2, 1, 1),
),
text_encoders_card = None,
card_required_mib = ((0, 1), (1, 1)),
card_limit_mib = ((0, 9), (1, 9)),
text_encoders_streamed = True,
)
from core.inference import diffusion_layer_split as engine
monkeypatch.setattr(
dm,
"stream_text_encoders_onto",
lambda p, d, logger = None: calls["stream"].append(str(d)) or ["text_encoder"],
)
monkeypatch.setattr(dm, "_remove_group_offload_hooks", lambda m: calls["removed"].append(m))
monkeypatch.setattr(lsp, "split_blocks", lambda m: [("b.0", None), ("b.1", None)])
monkeypatch.setattr(
engine, "place_layer_split", lambda p, **kw: calls["engine"].append(kw["assignment"]) or {}
)
monkeypatch.setattr(engine, "undo_layer_split", lambda p, attr = "transformer": False)
import contextlib
from core.inference import diffusion_fast_load
monkeypatch.setattr(
diffusion_fast_load, "fast_upload", lambda mods, dev, logger = None: contextlib.nullcontext()
)
lsp.place_layer_split(pipe, primary = "cuda:0", split = split)
assert pipe.text_encoder.moved == [] # never moved onto a card: paged by its hooks
assert pipe.vae.moved == [torch.device("cuda", 0)]
assert calls["stream"] == ["cuda:0"]
assert calls["engine"] == [{"b.0": 0, "b.1": 1}]
record = mg._PLACEMENTS[pipe]
assert record["streamed"] == ["text_encoder"] and record["moved"] == []
assert record["placement"]["text_encoder"].startswith("cpu (streamed")
lsp.undo_layer_split(pipe)
assert calls["removed"] == [pipe.text_encoder]
assert pipe not in mg._PLACEMENTS
def test_every_denoiser_keeps_a_block_off_the_primary():
limits = {0: 100000, 1: 6000}
kw = dict(
companions = {"text_encoder": (2000, "text_encoder"), "vae": (300, "other")},
primary = 0,
card_limits = limits,
headroom_mib = 1000,
overhead_mib = 0,
working_set_mib = 1000,
secondary_overhead_mib = 0,
)
for denoisers in (
{"transformer": L(0, (1000,) * 10)},
{"transformer": L(0, (1000,) * 10), "transformer_2": L(0, (1000,) * 10)},
):
split, reason = lsp.plan_layer_split(denoisers = denoisers, **kw)
assert split is not None, reason
_check(split, limits)
for name in denoisers:
assert any(r.card != 0 for r in split.ranges if r.denoiser == name), (
name,
split.ranges,
)
def test_too_small_for_the_combined_cards_declines():
split, reason = lsp.plan_layer_split(
denoisers = {"transformer": L(1000, (1000,) * 40)},
companions = {"vae": (300, "other")},
primary = 0,
card_limits = {0: 18000, 1: 18000},
headroom_mib = 2000,
overhead_mib = 500,
)
assert split is None
assert "does not fit" in reason
def test_no_secondary_or_unknown_blocks_declines():
assert (
lsp.plan_layer_split(
denoisers = {"transformer": L(0, (1000,))},
companions = {},
primary = 0,
card_limits = {0: 18000},
headroom_mib = 0,
overhead_mib = 0,
)[0]
is None
)
assert (
lsp.plan_layer_split(
denoisers = {"transformer": L(1000, ())},
companions = {},
primary = 0,
card_limits = {0: 18000, 1: 18000},
headroom_mib = 0,
overhead_mib = 0,
)[0]
is None
)
def test_mixed_card_sizes_cut_in_proportion_to_free_capacity():
limits = {0: 20000, 1: 68000}
split, _ = lsp.plan_layer_split(
denoisers = {"transformer": L(1000, (1000,) * 60)},
companions = {},
primary = 0,
card_limits = limits,
headroom_mib = 3000,
overhead_mib = 500,
)
_check(split, limits)
on = {o: sum(r.mib for r in split.ranges if r.card == o) for o in (0, 1)}
reserve = lsp.secondary_reserve_mib()
caps = {0: 20000 - 1000 - 3000 - max(500, reserve), 1: 68000 - 2000 - reserve}
for o in (0, 1):
assert abs(on[o] / caps[o] - 60000 / sum(caps.values())) < 1000 / caps[o] + 1e-9
def test_uses_a_third_card_only_when_two_are_not_enough():
limits = {0: 18000, 1: 18000, 2: 18000}
two, _ = lsp.plan_layer_split(
denoisers = {"transformer": L(1000, (1000,) * 25)},
companions = {},
primary = 0,
card_limits = limits,
headroom_mib = 2000,
overhead_mib = 500,
)
assert two.cards == (0, 1)
three, _ = lsp.plan_layer_split(
denoisers = {"transformer": L(1000, (1000,) * 40)},
companions = {},
primary = 0,
card_limits = limits,
headroom_mib = 2000,
overhead_mib = 500,
)
_check(three, limits)
assert three.cards == (0, 1, 2)
assert three.boundaries == 3
def test_largest_secondary_is_used_first():
split, _ = lsp.plan_layer_split(
denoisers = {"transformer": L(1000, (1000,) * 25)},
companions = {},
primary = 0,
card_limits = {0: 18000, 1: 9000, 2: 30000},
headroom_mib = 2000,
overhead_mib = 500,
)
assert split.cards == (0, 2)
def test_two_experts_are_each_cut_with_the_headroom_charged_once():
limits = {0: 40000, 1: 40000}
split, reason = lsp.plan_layer_split(
denoisers = {"transformer": L(500, (700,) * 40), "transformer_2": L(500, (700,) * 40)},
companions = {"text_encoder": (6000, "text_encoder"), "vae": (300, "other")},
primary = 0,
card_limits = limits,
headroom_mib = 4000,
overhead_mib = 500,
)
assert split is not None, reason
_check(split, limits)
assert {r.denoiser for r in split.ranges if r.card == 1} == {"transformer", "transformer_2"}
weights = dict(split.card_weights_mib)
required = dict(split.card_required_mib)
reserve = lsp.secondary_reserve_mib()
assert required[0] == weights[0] + 4000 + max(500, reserve)
assert required[1] == weights[1] + max(1400, 1024) + reserve
def test_phase_headroom_prices_decode_on_the_primary_and_encode_on_the_te_card():
split, _ = lsp.plan_layer_split(
denoisers = {"transformer": L(1000, (1000,) * 23)},
companions = {"text_encoder": (4000, "text_encoder"), "vae": (300, "other")},
primary = 0,
card_limits = {0: 20000, 1: 20000},
headroom_mib = 0,
overhead_mib = 500,
phase_headroom_mib = (3000, 1500, 2500),
)
weights = dict(split.card_weights_mib)
required = dict(split.card_required_mib)
reserve = lsp.secondary_reserve_mib()
assert split.text_encoders_card == 1
assert required[0] == weights[0] + max(1500, 2500) + max(500, reserve)
assert required[1] == weights[1] + 3000 + reserve
def test_secondary_is_charged_a_block_working_set_not_the_primary_activation_peak():
denoisers = {"transformer": L(1500, (1070,) * 56)}
limits = {0: 80000, 1: 80000}
split, reason = lsp.plan_layer_split(
denoisers = denoisers,
companions = {"vae": (200, "other")},
primary = 0,
card_limits = limits,
headroom_mib = 30000,
overhead_mib = 2048,
)
assert split is not None, reason
weights = dict(split.card_weights_mib)
required = dict(split.card_required_mib)
assert lsp.block_working_set_mib(denoisers, 30000) == 2140
assert required[1] == weights[1] + 2140 + lsp.secondary_reserve_mib()
assert required[0] == weights[0] + 30000 + 2048
assert lsp.block_working_set_mib(denoisers, 1500) == 1500
assert lsp.block_working_set_mib({"transformer": L(0, (100,))}, 0) == 1024
def test_reserve_follows_comfyui_minimum_inference_memory(monkeypatch):
monkeypatch.setattr(lsp.sys, "platform", "linux")
assert lsp.secondary_reserve_mib() == 1200
monkeypatch.setattr(lsp.sys, "platform", "win32")
assert lsp.secondary_reserve_mib() == 1500
def test_layouts_count_storage_bytes_so_quantized_blocks_are_smaller():
class Block(torch.nn.Module):
def __init__(self, dtype):
super().__init__()
self.w = torch.nn.Parameter(torch.empty(4096, 4096, dtype = dtype), requires_grad = False)
class Den(torch.nn.Module):
_no_split_modules = ["Block"]
def __init__(self, dtype):
super().__init__()
self.proj_out = torch.nn.Linear(4096, 1024, dtype = torch.bfloat16)
self.transformer_blocks = torch.nn.ModuleList(Block(dtype) for _ in range(3))
self.single_transformer_blocks = torch.nn.ModuleList(Block(dtype) for _ in range(2))
class Pipe:
transformer = Den(torch.bfloat16)
class Quant:
transformer = Den(torch.int8)
bf16 = lsp._denoiser_layouts(Pipe())["transformer"]
int8 = lsp._denoiser_layouts(Quant())["transformer"]
assert bf16.block_mib == (32,) * 5
assert int8.block_mib == (16,) * 5
assert bf16.stem_mib == 9
def test_a_denoiser_without_a_block_list_has_no_layout():
class Pipe:
transformer = torch.nn.Linear(4, 4)
assert lsp._denoiser_layouts(Pipe()) is None
@dataclasses.dataclass
class _Target:
supports_model_cpu_offload: bool = True
ordinal: int = 0
device: str = "cuda"
backend: str = "cuda"
is_cuda_torch_device: bool = True
def _memory(free = 24000, total = 24576):
return dm.DeviceMemory(
backend = "cuda", device = "cuda", memory_kind = "discrete_vram", free_mib = free, total_mib = total
)
def _plan(**kw):
args = dict(
target = _Target(),
device_memory = _memory(),
model_dense_mib = 24000,
companion_dense_mib = 4300,
text_encoder_dense_mib = 4000,
runtime_headroom_mib = 2000,
)
args.update(kw)
return dm.plan_diffusion_memory(**args)
_BIG = {
"transformer": (24000, "dit"),
"text_encoder": (4000, "text_encoder"),
"vae": (300, "other"),
}
class _Engine:
def __init__(self, fail = False):
self.fail = fail
self.placed = []
self.undone = 0
def place(
self,
pipe,
*,
primary,
split,
logger = None,
):
self.placed.append((primary, split))
if self.fail:
raise RuntimeError("boom")
return {}
def undo(self, pipe):
self.undone += 1
@pytest.fixture
def engine(monkeypatch):
eng = _Engine()
monkeypatch.setattr(lsp, "engine_available", lambda: True)
monkeypatch.setattr(lsp, "place_layer_split", eng.place)
monkeypatch.setattr(lsp, "undo_layer_split", eng.undo)
return eng
def _cards(
monkeypatch,
sizes = _BIG,
sec_limit = 20000,
layouts = None,
):
monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: sec_limit)
monkeypatch.setattr(
mg, "best_secondary", lambda selection, probes = None, **_: selection.secondaries[0]
)
monkeypatch.setattr(
lsp,
"_denoiser_layouts",
lambda pipe: layouts if layouts is not None else {"transformer": L(1000, (1000,) * 23)},
)
class _Pipe:
"""Weak-referenceable stand-in; the refinement only reads it through the patched size / layout readers."""
components: dict = {}
def to(self, *a, **k):
return self
def _refine(
plan,
pipe = None,
ordinals = (0, 1),
):
pipe = pipe if pipe is not None else _Pipe()
with mg.selection_scope(mg.make_selection(0, list(ordinals))):
return mg.refine_plan_for_multigpu(pipe, plan), pipe
def test_one_card_offload_becomes_a_layer_split(monkeypatch, engine):
plan = _plan()
assert lsp.plan_offloads_denoiser(plan)
_cards(monkeypatch)
refined, pipe = _refine(plan)
assert refined is not plan
assert refined.offload_policy == dm.OFFLOAD_NONE
assert refined.layer_split is not None and refined.secondary_ordinal is None
assert refined.fallback_plan is plan
assert refined.layer_split.cards == (0, 1)
public = refined.as_public_dict()
assert public["layer_split"]["cards"] == [0, 1]
assert "layer_split" not in plan.as_public_dict()
assert any("layer split" in r for r in refined.reasons)
def test_no_engine_keeps_the_one_card_plan(monkeypatch):
monkeypatch.setattr(lsp, "engine_available", lambda: False)
_cards(monkeypatch)
plan = _plan()
assert _refine(plan)[0] is plan
def test_no_selection_or_kill_switches_keep_the_plan(monkeypatch, engine):
_cards(monkeypatch)
plan = _plan()
assert mg.refine_plan_for_multigpu(_Pipe(), plan) is plan
assert _refine(plan, ordinals = (0,))[0] is plan
monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "0")
assert _refine(plan)[0] is plan
monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "auto")
monkeypatch.setenv(mg.MULTIGPU_ENV, "0")
assert _refine(plan)[0] is plan
assert engine.placed == []
def test_keep_multiple_models_loaded_gets_no_selection(monkeypatch, engine):
_cards(monkeypatch)
plan = _plan()
monkeypatch.setattr(mg, "models_kept_loaded", lambda: True)
seen = []
class Backend:
@mg.with_load_selection
def _run_load(self, **kwargs):
seen.append(mg.current_selection())
return mg.refine_plan_for_multigpu(_Pipe(), plan)
assert Backend()._run_load(gpu_ordinal = 0, _gpu_ordinals = [0, 1]) is plan
assert seen == [None]
def test_a_dit_that_fits_one_card_keeps_todays_plan(monkeypatch, engine):
small = {
"transformer": (8000, "dit"),
"text_encoder": (4000, "text_encoder"),
"vae": (300, "other"),
}
_cards(monkeypatch, sizes = small)
plan = _plan(model_dense_mib = 8000)
assert plan.offload_policy == dm.OFFLOAD_NONE
assert _refine(plan)[0] is plan
streamed_te = dataclasses.replace(
_plan(),
offload_policy = dm.OFFLOAD_GROUP,
stream_transformer = False,
stream_text_encoders = True,
)
_cards(monkeypatch, sec_limit = 1000) # no companion assignment either
assert _refine(streamed_te)[0] is streamed_te
assert engine.placed == []
def test_quantized_storage_that_fits_one_card_is_a_companion_split_not_a_layer_split(
monkeypatch, engine
):
quant = {
"transformer": (12000, "dit"),
"text_encoder": (4000, "text_encoder"),
"vae": (300, "other"),
}
_cards(monkeypatch, sizes = quant)
refined, _ = _refine(_plan())
assert refined.layer_split is None
assert refined.secondary_ordinal == 1
def test_companion_split_is_preferred_over_a_layer_split(monkeypatch, engine):
fits = {
"transformer": (14000, "dit"),
"text_encoder": (8000, "text_encoder"),
"vae": (300, "other"),
}
_cards(monkeypatch, sizes = fits)
refined, _ = _refine(_plan())
assert refined.secondary_ordinal == 1 and refined.layer_split is None
def test_cards_too_small_keep_the_offload_plan_and_record_why(monkeypatch, engine):
_cards(monkeypatch, sec_limit = 6000)
plan = _plan()
refined, pipe = _refine(plan)
assert refined is plan
status = lsp.layer_split_status(pipe)
assert "does not fit" in status["declined"]
assert mg.placement_status(pipe)["layer_split"] == status
def test_rocm_discrete_splits_and_explicit_modes_and_legacy_cpu_offload_keep_the_plan(
monkeypatch, engine
):
_cards(monkeypatch)
plan = _plan()
_rocm_host(monkeypatch, {0: _RX7900, 1: _RX7900})
refined, pipe = _refine(plan)
assert refined.layer_split is not None and refined.layer_split.cards == (0, 1)
monkeypatch.setattr(lsp, "_is_rocm", lambda: False)
monkeypatch.setattr(mg, "_rocm_torch", lambda: False)
balanced = _plan(requested_mode = "balanced")
assert _refine(balanced)[0] is balanced
legacy = _plan(explicit_offload = True)
assert legacy.estimates.get("explicit_cpu_offload")
assert _refine(legacy)[0] is legacy
assert engine.placed == []
def test_force_cuts_a_denoiser_one_card_would_hold(monkeypatch, engine):
small = {"transformer": (8000, "dit"), "vae": (300, "other")}
_cards(monkeypatch, sizes = small, layouts = {"transformer": L(500, (500,) * 15)})
plan = _plan(model_dense_mib = 8000, companion_dense_mib = 300, text_encoder_dense_mib = 0)
assert plan.offload_policy == dm.OFFLOAD_NONE
monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "force")
refined, _ = _refine(plan)
assert refined.layer_split is not None and refined.layer_split.cards == (0, 1)
def test_video_two_expert_cut_when_neither_expert_fits_a_card(monkeypatch, engine):
sizes = {
"transformer": (28500, "dit"),
"transformer_2": (28500, "dit"),
"text_encoder": (6000, "text_encoder"),
"vae": (300, "other"),
}
layouts = {"transformer": L(500, (700,) * 40), "transformer_2": L(500, (700,) * 40)}
_cards(monkeypatch, sizes = sizes, sec_limit = 34000, layouts = layouts)
plan = _plan(
device_memory = _memory(free = 40000, total = 40960),
model_dense_mib = 57000,
companion_dense_mib = 6300,
text_encoder_dense_mib = 6000,
)
refined, _ = _refine(plan, ordinals = (0, 1, 2))
split = refined.layer_split
assert split is not None
assert {r.denoiser for r in split.ranges} == {"transformer", "transformer_2"}
def _layer_plan(monkeypatch, engine):
_cards(monkeypatch)
refined, pipe = _refine(_plan())
assert refined.layer_split is not None
return refined, pipe
def test_apply_records_the_split_and_cfg_parallel_declines(monkeypatch, engine):
plan, pipe = _layer_plan(monkeypatch, engine)
assert lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")
assert engine.placed[0][0] == "cuda:0" and engine.placed[0][1] is plan.layer_split
assert lsp.is_layer_split(pipe) and mg.is_split(pipe)
status = mg.placement_status(pipe)
assert status["layer_split"]["cards"] == [0, 1]
assert status["layer_split"]["boundaries"] == 2
with mg.selection_scope(mg.make_selection(0, [0, 1])):
assert (
mg.cfg_parallel_decline(pipe, offload_policy = "none", selection = mg.current_selection())
== "denoiser is split across cards (layer split)"
)
lsp.forget_layer_split(pipe)
assert not lsp.is_layer_split(pipe)
def test_failed_placement_applies_the_one_card_fallback(monkeypatch, engine):
plan, pipe = _layer_plan(monkeypatch, engine)
engine.fail = True
applied = []
real = dm.apply_memory_plan
def spy(p, pl, **kw):
applied.append(pl)
if pl is plan:
return real(p, pl, **kw)
return pl.offload_policy, False
monkeypatch.setattr(dm, "apply_memory_plan", spy)
policy, _ = dm.apply_memory_plan(pipe, plan, device = "cuda", placement_device = "cuda:0")
assert engine.undone == 1
assert applied == [plan, plan.fallback_plan]
assert policy == plan.fallback_plan.offload_policy
assert dm.applied_plan(pipe, plan) is plan.fallback_plan
assert not lsp.is_layer_split(pipe)
assert "placement failed" in lsp.layer_split_status(pipe)["declined"]
def test_video_guard_reads_per_card_floors_from_the_layer_split(monkeypatch, engine):
from core.inference import video as V
plan, pipe = _layer_plan(monkeypatch, engine)
assert V._video_split_card_floors(pipe) is None
lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")
floors = V._video_split_card_floors(pipe)
split = plan.layer_split
weights = dict(split.card_weights_mib)
required = dict(split.card_required_mib)
reserve = dict(split.card_reserve_mib)
assert floors == (
(0, weights[0], True, True, True, None, None),
(1, weights[1], False, False, True, required[1] - weights[1] - reserve[1], reserve[1]),
)
assert reserve[1] > 0
def test_video_guard_prices_a_block_card_by_its_planned_headroom():
from core.inference import video as V
cards = ((0, 13300, True, True, True, None), (1, 20000, False, False, True, 3200))
free = {0: 60000, 1: 23500}
kw = dict(family = "wan", available_mib = lambda o: free[o], planned_shape = (832, 480, 33))
assert V.video_split_shortfall_message(cards, width = 832, height = 480, frames = 33, **kw) is None
message = V.video_split_shortfall_message(cards, width = 832, height = 480, frames = 66, **kw)
assert message is not None and "cuda:1" in message
assert V.video_split_shortfall_message(cards, width = 416, height = 240, frames = 17, **kw) is None
def test_video_guard_scales_only_the_working_set_not_the_fixed_reserve():
from core.inference import video as V
cards = ((0, 13300, True, True, True, None, None), (1, 20000, False, False, True, 700, 2500))
free = {0: 60000, 1: 24000}
kw = dict(family = "wan", available_mib = lambda o: free[o], planned_shape = (832, 480, 33))
assert V.video_split_shortfall_message(cards, width = 832, height = 480, frames = 66, **kw) is None
free[1] = 20000 + 700 + 2400
message = V.video_split_shortfall_message(cards, width = 832, height = 480, frames = 33, **kw)
assert message is not None and "cuda:1" in message
def test_a_from_pipe_sibling_sees_the_split_through_its_shared_denoiser(monkeypatch, engine):
from core.inference import diffusion_layer_split as engine_mod
plan, pipe = _layer_plan(monkeypatch, engine)
shared = torch.nn.Linear(2, 2)
pipe.transformer = shared
lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")
sibling = type("Sibling", (), {"transformer": shared})()
monkeypatch.setattr(engine_mod, "is_layer_split", lambda m: m is shared)
assert lsp.applied_layer_split(sibling) is plan.layer_split
assert lsp.secondary_block_cards(sibling) == (1,)
assert mg.is_split(sibling)
mg.undo_split(pipe)
assert lsp.applied_layer_split(pipe) is None and lsp.applied_layer_split(sibling) is None
def test_a_failed_replacement_does_not_report_the_previous_split(monkeypatch, engine):
plan, pipe = _layer_plan(monkeypatch, engine)
assert lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")
assert lsp.is_layer_split(pipe)
engine.fail = True
assert not lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")
assert not lsp.is_layer_split(pipe) and not mg.is_split(pipe)
assert "placement failed" in lsp.layer_split_status(pipe)["declined"]
def test_an_expert_wholly_on_the_primary_is_placed_plainly(monkeypatch):
import contextlib
from core.inference import diffusion_fast_load
from core.inference import diffusion_layer_split as engine_mod
class Comp(torch.nn.Module):
def __init__(self):
super().__init__()
self.w = torch.nn.Parameter(torch.zeros(1))
self.moved = []
def to(self, *a, **k):
self.moved.append(a[0] if a else k.get("device"))
return self
class Pipe:
def __init__(self):
self.transformer = Comp()
self.transformer_2 = Comp()
self.vae = Comp()
self.components = {
"transformer": self.transformer,
"transformer_2": self.transformer_2,
"vae": self.vae,
}
pipe = Pipe()
split = lsp.LayerSplit(
primary = 0,
cards = (0, 1),
ranges = (
lsp.BlockRange("transformer", 0, 1, 0, 1),
lsp.BlockRange("transformer", 1, 2, 1, 1),
lsp.BlockRange("transformer_2", 0, 2, 0, 2),
),
text_encoders_card = None,
card_required_mib = ((0, 1), (1, 1)),
card_limit_mib = ((0, 9), (1, 9)),
)
placed = []
monkeypatch.setattr(lsp, "split_blocks", lambda m: [("b.0", None), ("b.1", None)])
monkeypatch.setattr(
engine_mod,
"place_layer_split",
lambda p, **kw: placed.append((kw["attr"], kw["assignment"])) or {},
)
monkeypatch.setattr(
diffusion_fast_load, "fast_upload", lambda mods, dev, logger = None: contextlib.nullcontext()
)
out = lsp.place_layer_split(pipe, primary = "cuda:0", split = split)
assert placed == [
("transformer", {"b.0": 0, "b.1": 1})
] # only the expert that is cut reaches the engine
assert out["transformer_2"]["whole_on_primary"] is True
assert pipe.transformer_2.moved == [torch.device("cuda", 0)]
def test_offload_installers_refuse_or_replace_a_split(monkeypatch):
from core.inference import diffusion_layer_split as engine_mod
dit = torch.nn.Linear(2, 2)
pipe = type("P", (), {"transformer": dit, "components": {"transformer": dit}})()
monkeypatch.setattr(engine_mod, "is_layer_split", lambda m: m is dit)
monkeypatch.setattr(dm, "_offload_groups", lambda m: [])
monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda p: True)
assert dm.hook_resident_denoiser(pipe, "cuda:0") is False
undone = []
monkeypatch.setattr(lsp, "undo_layer_split", lambda p: undone.append(p))
dm._apply_group_offload(pipe, "cpu", None)
assert undone == [pipe]
def test_image_guard_checks_every_other_card_running_blocks(monkeypatch, engine):
from core.inference import diffusion as D
plan, pipe = _layer_plan(monkeypatch, engine)
free = {"mib": 100000}
monkeypatch.setattr(
D,
"reclaimable_snapshot_device_memory",
lambda target: dm.DeviceMemory(
backend = "cuda",
device = "cuda",
memory_kind = "discrete_vram",
free_mib = free["mib"],
total_mib = 102400,
),
)
kw = {"width": 1024, "height": 1024, "family": "flux2"}
assert D._guard_layer_cards(pipe, object(), kw) == 0 # not placed: nothing to check
lsp.apply_layer_split_plan(pipe, plan, placement = "cuda:0")
assert D._guard_layer_cards(pipe, object(), kw) == 1
free["mib"] = 10
with pytest.raises(ValueError, match = "cuda:1"):
D._guard_layer_cards(pipe, object(), kw)
def test_two_denoisers_that_fit_only_jointly_are_cut_jointly():
kw = dict(
companions = {},
primary = 0,
card_limits = {0: 3, 1: 3},
headroom_mib = 0,
overhead_mib = 0,
working_set_mib = 0,
secondary_overhead_mib = 0,
)
denoisers = {"transformer": L(0, (1, 1)), "transformer_2": L(0, (1, 2, 1))}
split, reason = lsp.plan_layer_split(denoisers = denoisers, **kw)
assert split is not None, reason
per_card = {}
for r in split.ranges:
per_card[r.card] = per_card.get(r.card, 0) + r.mib
assert per_card[0] <= 3 and per_card[1] <= 3
for name, layout in denoisers.items():
mine = sorted((r.start, r.stop) for r in split.ranges if r.denoiser == name)
assert mine[0][0] == 0 and mine[-1][1] == len(layout.block_mib) # every block placed once
assert any(r.card != 0 for r in split.ranges if r.denoiser == name)
assert lsp._search_cuts([(1, 1), (1, 2, 1)], [2, 2]) is None # truly infeasible stays declined
def test_dual_denoisers_may_keep_one_wholly_on_the_primary():
# A fits only on the primary, B only on the secondary: forcing both to cut would decline a valid placement
assert lsp._search_cuts([(2, 2), (3, 3)], [4, 6]) == [[0, 2, 2], [0, 0, 2]]
assert lsp._search_cuts([(2, 2), (3, 3)], [10, 0]) is None # nothing off the primary: no split
split, reason = lsp.plan_layer_split(
denoisers = {"transformer": L(0, (2, 2)), "transformer_2": L(0, (3, 3))},
companions = {},
primary = 0,
card_limits = {0: 4, 1: 6},
headroom_mib = 0,
overhead_mib = 0,
phase_headroom_mib = (0, 0, 0),
working_set_mib = 0,
secondary_overhead_mib = 0,
)
assert split is not None, reason
_check(split, {0: 4, 1: 6})
assert {(r.denoiser, r.card) for r in split.ranges} == {
("transformer", 0),
("transformer_2", 1),
}
def test_failed_placement_drops_fused_records_left_on_another_card(monkeypatch):
from types import SimpleNamespace
from torch import nn
def attn(fused_device):
a = nn.Module()
a.to_q, a.to_k, a.to_v = nn.Linear(4, 4), nn.Linear(4, 4), nn.Linear(4, 4)
a.__dict__["_unsloth_fused_qkv"] = (
nn.Linear(4, 12, device = fused_device),
a.to_q,
a.to_k,
a.to_v,
)
return a
dit = nn.Module()
dit.stale, dit.intact = attn("meta"), attn("cpu")
pipe = SimpleNamespace(transformer = dit, to = lambda *a, **k: None)
def boom(*a, **k):
raise RuntimeError("out of memory")
monkeypatch.setattr(lsp, "place_layer_split", boom)
monkeypatch.setattr(lsp, "undo_layer_split", lambda p: None)
assert (
lsp.apply_layer_split_plan(pipe, SimpleNamespace(layer_split = object()), placement = "cuda:0")
is False
)
assert "_unsloth_fused_qkv" not in dit.stale.__dict__
assert "_unsloth_fused_qkv" in dit.intact.__dict__ # still on its parts' device: kept
def test_an_arch_pinned_attention_backend_keeps_blocks_off_other_architectures(monkeypatch, engine):
# flash3 is Hopper only: a B200 secondary would reject it
from core.inference.diffusion_attention import ATTENTION_BACKEND_ATTR
caps = {0: (9, 0), 1: (10, 0), 2: (9, 0)}
monkeypatch.setattr(mg, "_capability", lambda o: caps.get(int(o)))
_cards(monkeypatch)
pipe = _Pipe()
pipe.transformer = torch.nn.Linear(2, 2)
setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, "_flash_3_hub")
plan = _plan()
refined, _ = _refine(plan, pipe = pipe, ordinals = (0, 1))
assert refined is plan # the only secondary is another architecture: no split
assert "attention backend _flash_3_hub" in lsp._DECLINES[pipe]
refined, _ = _refine(plan, pipe = pipe, ordinals = (0, 1, 2))
assert refined.layer_split is not None and 1 not in refined.layer_split.cards
setattr(
pipe.transformer, ATTENTION_BACKEND_ATTR, "_native_cudnn"
) # runs on every admitted card
refined, _ = _refine(plan, pipe = pipe, ordinals = (0, 1))
assert refined.layer_split is not None and 1 in refined.layer_split.cards
# ROCm: real classifiers (utils.hardware's unified test, rocm_bf16's gfx table) over fake device properties.
_RX7900 = ("gfx1100", 0) # 7900 XTX, discrete
_MI300 = ("gfx942", 0)
_RX6900 = ("gfx1030", 0) # RDNA2: no native bf16
_STRIX_HALO = ("gfx1151", 1) # APU: the driver's integrated flag set
def _rocm_host(monkeypatch, cards):
from types import SimpleNamespace
monkeypatch.setattr(lsp, "_is_rocm", lambda: True)
monkeypatch.setattr(mg, "_rocm_torch", lambda: True)
monkeypatch.setattr(torch.cuda, "is_bf16_supported", lambda *a, **k: True)
monkeypatch.delenv("UNSLOTH_STUDIO_ROCM_BF16", raising = False)
def props(o = None):
arch, integrated = cards[int(o)]
return SimpleNamespace(
gcnArchName = f"{arch}:sramecc+:xnack-", is_integrated = integrated, name = "AMD Radeon"
)
monkeypatch.setattr(torch.cuda, "get_device_properties", props)
# the per-card SDPA probe spawns HIP children; every card has a fused kernel unless a test says otherwise
from core.inference import diffusion_attention as da
monkeypatch.setattr(da, "sdpa_math_only", lambda target: False)
def test_rocm_two_discrete_cards_plan_the_same_split_as_cuda(monkeypatch, engine):
_cards(monkeypatch)
cuda, _ = _refine(_plan())
_rocm_host(monkeypatch, {0: _RX7900, 1: _RX7900})
rocm, pipe = _refine(_plan())
assert rocm.layer_split is not None
assert rocm.layer_split.ranges == cuda.layer_split.ranges
assert rocm.layer_split.card_required_mib == cuda.layer_split.card_required_mib
assert lsp.layer_split_status(pipe) is None # no decline recorded
def test_rocm_two_apus_decline_with_the_unified_reason(monkeypatch, engine):
_cards(monkeypatch)
_rocm_host(monkeypatch, {0: _STRIX_HALO, 1: _STRIX_HALO})
plan = _plan()
refined, pipe = _refine(plan)
assert refined is plan and engine.placed == []
declined = lsp.layer_split_status(pipe)["declined"]
assert declined.startswith(lsp.UNIFIED_DECLINE) and "cuda:0" in declined
def test_rocm_unified_primary_plan_records_why_and_cuda_stays_silent(monkeypatch, engine):
_cards(monkeypatch)
unified = dm.DeviceMemory(
backend = "cuda", device = "cuda", memory_kind = "unified_memory", free_mib = 90000, total_mib = 98000
)
plan = dataclasses.replace(_plan(), device_memory = unified)
monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "force")
refined, pipe = _refine(plan)
assert refined is plan and lsp.layer_split_status(pipe) is None # CUDA: unchanged
_rocm_host(monkeypatch, {0: _STRIX_HALO, 1: _STRIX_HALO})
refined, pipe = _refine(plan)
assert refined is plan and engine.placed == []
assert lsp.layer_split_status(pipe)["declined"].startswith(lsp.UNIFIED_DECLINE)
def test_rocm_discrete_primary_skips_an_apu_secondary(monkeypatch, engine):
_cards(monkeypatch)
_rocm_host(monkeypatch, {0: _RX7900, 1: _STRIX_HALO, 2: _RX7900})
refined, _ = _refine(_plan(), ordinals = (0, 1, 2))
assert refined.layer_split is not None and 1 not in refined.layer_split.cards
assert 2 in refined.layer_split.cards
refined, pipe = _refine(_plan(), ordinals = (0, 1))
assert refined.layer_split is None
assert "cuda:1" in lsp.layer_split_status(pipe)["declined"]
def test_rocm_kill_switch_keeps_the_plan_and_records_nothing(monkeypatch, engine):
_cards(monkeypatch)
_rocm_host(monkeypatch, {0: _RX7900, 1: _RX7900})
monkeypatch.setenv(lsp.LAYER_SPLIT_ENV, "0")
plan = _plan()
refined, pipe = _refine(plan)
assert refined is plan and engine.placed == [] and lsp.layer_split_status(pipe) is None
def test_rocm_mixed_arch_selection_compares_dtype_not_gfx_order(monkeypatch):
# CUDA's capability order would drop MI300 (9, 4) under a 7900 XTX (11, 0) primary
_rocm_host(monkeypatch, {0: _RX7900, 1: _MI300, 2: _RX6900})
monkeypatch.setattr(mg, "_capability", lambda o: {0: (11, 0), 1: (9, 4), 2: (10, 3)}[int(o)])
assert mg.make_selection(0, [0, 1]) == mg.MultiGpuSelection(primary = 0, secondaries = (1,))
assert mg.make_selection(1, [1, 0]) == mg.MultiGpuSelection(primary = 1, secondaries = (0,))
# a bf16 primary keeps no RDNA2 card (it would emulate the primary's dtype); an fp16 primary keeps any
assert mg.make_selection(0, [0, 2]) is None
assert mg.make_selection(2, [2, 0, 1]) == mg.MultiGpuSelection(primary = 2, secondaries = (0, 1))
def test_rocm_mixed_arch_native_sdpa_splits_and_flash_stays_on_the_primary_target(
monkeypatch, engine
):
from core.inference.diffusion_attention import ATTENTION_BACKEND_ATTR
_cards(monkeypatch)
_rocm_host(monkeypatch, {0: _RX7900, 1: _MI300, 2: _RX7900})
pipe = _Pipe()
pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)
refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1))
assert refined.layer_split is not None and refined.layer_split.cards == (0, 1)
probed = []
monkeypatch.setattr(
lsp, "_rocm_flash_runs_on", lambda o, dtype: probed.append((o, dtype)) or o != 2
)
for tag in ("flash", "aiter"):
setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, tag)
refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1))
assert refined.layer_split is None
assert f"attention backend {tag}" in lsp._DECLINES[pipe]
setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, "aiter")
refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1, 2))
assert refined.layer_split is not None and 1 not in refined.layer_split.cards
setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, "flash")
probed.clear()
refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1, 2))
assert probed == [
(2, torch.bfloat16)
] # the same-target card is probed; the MI300 never reaches the probe
assert refined.layer_split is None # flash failed on cuda:2: no secondary left
def test_windows_rocm_secondary_budget_uses_the_capped_free(monkeypatch):
"""WDDM: mem_get_info reports free ~ total whatever is resident; the secondary is priced from what torch has not
reserved there (utils.hardware.trusted_mem_get_info)."""
import sys
from utils.hardware import hardware as hw
gib = 1 << 30
monkeypatch.setattr(sys, "platform", "win32")
monkeypatch.setattr(hw, "IS_ROCM", True)
monkeypatch.setattr(
torch.cuda, "mem_get_info", lambda o = None: (24 * gib - (64 << 20), 24 * gib)
)
monkeypatch.setattr(torch.cuda, "memory_reserved", lambda o = None: 10 * gib)
memory = mg.secondary_device_memory(1)
assert memory.total_mib == 24 * 1024 and memory.free_mib == 14 * 1024
assert mg.secondary_resident_limit_mib(1, "auto") < 14 * 1024
monkeypatch.setattr(hw, "IS_ROCM", False)
assert mg.secondary_device_memory(1).free_mib == 24 * 1024 - 64
def test_rocm_apu_without_the_integrated_flag_gets_no_blocks(monkeypatch, engine):
# Windows gfx1153 Krackan Point 2 on a runtime that leaves is_integrated 0: still an APU by its gfx target
_cards(monkeypatch)
_rocm_host(monkeypatch, {0: _RX7900, 1: ("gfx1153", 0), 2: _RX7900})
refined, _ = _refine(_plan(), ordinals = (0, 1, 2))
assert refined.layer_split is not None and 1 not in refined.layer_split.cards
refined, pipe = _refine(_plan(), ordinals = (0, 1))
assert refined.layer_split is None
assert lsp.layer_split_status(pipe)["declined"].startswith(lsp.UNIFIED_DECLINE)
def test_rocm_math_only_secondary_gets_no_blocks_beside_a_fused_primary(monkeypatch, engine):
# mixed gfx, native SDPA: cuda:1 has no fused kernel at bf16, so math SDPA would hold the full score matrix there
_cards(monkeypatch)
_rocm_host(monkeypatch, {0: _RX7900, 1: _MI300, 2: _RX7900})
pipe = _Pipe()
pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)
from core.inference import diffusion_attention as da
probed = []
monkeypatch.setattr(
da,
"sdpa_math_only",
lambda t: probed.append((t.torch_device, t.dtype)) or t.torch_device == "cuda:1",
)
refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1, 2))
assert refined.layer_split is not None and 1 not in refined.layer_split.cards
assert 2 in refined.layer_split.cards
assert ("cuda:1", torch.bfloat16) in probed # probed per card at the denoiser's dtype
refined, pipe = _refine(_plan(), pipe = pipe, ordinals = (0, 1))
assert refined.layer_split is None
declined = lsp.layer_split_status(pipe)["declined"]
assert declined.startswith(lsp.MATH_ONLY_DECLINE) and "cuda:1" in declined
# a math-only primary already priced the quadratic peak: matching cards keep the split
monkeypatch.setattr(da, "sdpa_math_only", lambda t: True)
refined, _ = _refine(_plan(), pipe = pipe, ordinals = (0, 1))
assert refined.layer_split is not None and refined.layer_split.cards == (0, 1)
def test_cuda_split_never_runs_the_rocm_sdpa_probe(monkeypatch, engine):
_cards(monkeypatch)
from core.inference import diffusion_attention as da
monkeypatch.setattr(da, "sdpa_math_only", lambda t: pytest.fail("ROCm probe on CUDA"))
refined, _ = _refine(_plan())
assert refined.layer_split is not None
_TWO_EXPERTS = { # only moving the second expert fits both cards
"transformer": (9000, "dit"),
"transformer_2": (9000, "dit"),
"text_encoder": (4000, "text_encoder"),
"vae": (500, "other"),
}
def test_rocm_second_expert_moves_only_to_a_card_that_runs_the_attention_backend(
monkeypatch, engine
):
from core.inference import diffusion_attention as da
from core.inference.diffusion_attention import ATTENTION_BACKEND_ATTR
_cards(monkeypatch, sizes = _TWO_EXPERTS, sec_limit = 16000)
plan = _plan()
cuda, _ = _refine(plan)
# the component split moves the expert
assert "transformer_2" in (cuda.secondary_components or ())
_rocm_host(monkeypatch, {0: _RX7900, 1: _MI300})
pipe = _Pipe()
pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)
pipe.transformer_2 = torch.nn.Linear(2, 2).to(torch.bfloat16)
rocm, _ = _refine(_plan(), pipe = pipe)
assert "transformer_2" in (rocm.secondary_components or ()) # native SDPA, fused on both cards
for tag in ("flash", "aiter"): # pinned to gfx1100: the gfx942 card must not run the expert
setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, tag)
setattr(pipe.transformer_2, ATTENTION_BACKEND_ATTR, tag)
rocm, _ = _refine(_plan(), pipe = pipe)
assert "transformer_2" not in (rocm.secondary_components or ())
setattr(pipe.transformer, ATTENTION_BACKEND_ATTR, None)
setattr(pipe.transformer_2, ATTENTION_BACKEND_ATTR, None)
monkeypatch.setattr(da, "sdpa_math_only", lambda t: t.torch_device == "cuda:1")
rocm, _ = _refine(_plan(), pipe = pipe)
assert "transformer_2" not in (rocm.secondary_components or ())
@pytest.fixture
def _real_rocm_sdpa_probe(monkeypatch):
"""Real probe + guard + switches; only ROCm detection and child answers faked (cuda:0 fused, cuda:1 math-only)."""
from core import _torchao_stub
from core.inference import diffusion_attention as da
switches = (
(torch.backends.cuda.flash_sdp_enabled, torch.backends.cuda.enable_flash_sdp),
(
torch.backends.cuda.mem_efficient_sdp_enabled,
torch.backends.cuda.enable_mem_efficient_sdp,
),
)
before = [read() for read, _ in switches]
for _, write in switches:
write(True)
caches = (da._SDPA_PROBE_CACHE, da._ROCM_GUARD_DISABLED, da._ROCM_PROBE_INCOMPLETE)
for cache in caches:
cache.clear()
monkeypatch.delenv(da.ROCM_FUSED_SDPA_ALLOW_ENV, raising = False)
monkeypatch.setattr(_torchao_stub, "_module_is_rocm", lambda t: True)
monkeypatch.setattr(
da,
"_run_rocm_sdpa_children",
lambda device, dtype: (
(da.SDPA_MATH, da.SDPA_FLASH, da.SDPA_MEM_EFFICIENT)
if device == "cuda:0"
else (da.SDPA_MATH,)
),
)
yield lambda: tuple(read() for read, _ in switches)
for (_, write), value in zip(switches, before):
write(value)
for cache in caches:
cache.clear()
def test_rocm_secondary_probe_leaves_the_primary_sdpa_switches_alone(_real_rocm_sdpa_probe):
# A rejected math-only secondary must not turn fused SDPA off process-wide.
from types import SimpleNamespace
from core.inference import diffusion_attention as da
flags = _real_rocm_sdpa_probe
pipe = _Pipe()
pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)
assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == ()
assert flags() == (True, True)
assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == ()
assert flags() == (True, True)
assert not da.sdpa_math_only(
SimpleNamespace(device = "cuda", torch_device = "cuda:0", dtype = torch.bfloat16)
)
assert da.guard_rocm_fused_sdpa(
SimpleNamespace(device = "cuda", torch_device = "cuda:1", dtype = torch.bfloat16)
) == (da.SDPA_FLASH, da.SDPA_MEM_EFFICIENT)
assert flags() == (False, False)
def test_rocm_kept_secondary_still_gets_the_fused_sdpa_guard(_real_rocm_sdpa_probe, monkeypatch):
# cuda:1 is kept despite a failed flash launch, so flash goes off process-wide.
from core.inference import diffusion_attention as da
flags = _real_rocm_sdpa_probe
monkeypatch.setattr(
da,
"_run_rocm_sdpa_children",
lambda device, dtype: (
(da.SDPA_MATH, da.SDPA_FLASH, da.SDPA_MEM_EFFICIENT)
if device == "cuda:0"
else (da.SDPA_MATH, da.SDPA_MEM_EFFICIENT)
),
)
pipe = _Pipe()
pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)
assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == (1,)
assert flags() == (False, True)
assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == (1,)
assert flags() == (False, True)
def test_rocm_kept_secondaries_share_a_fused_sdpa_kernel(_real_rocm_sdpa_probe, monkeypatch):
# Keeping both would leave only math in common; cuda:2 goes.
from core.inference import diffusion_attention as da
flags = _real_rocm_sdpa_probe
answers = {
"cuda:0": (da.SDPA_MATH, da.SDPA_FLASH, da.SDPA_MEM_EFFICIENT),
"cuda:1": (da.SDPA_MATH, da.SDPA_FLASH),
"cuda:2": (da.SDPA_MATH, da.SDPA_MEM_EFFICIENT),
}
monkeypatch.setattr(da, "_run_rocm_sdpa_children", lambda device, dtype: answers[device])
pipe = _Pipe()
pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)
assert lsp._rocm_drop_math_only(pipe, 0, (1, 2)) == (1,)
assert flags() == (True, False)
assert lsp._rocm_drop_math_only(pipe, 0, (1, 2)) == (1,)
assert flags() == (True, False)
class _MixedPrecisionDiT(torch.nn.Module):
"""MiniMax-H3: fp32 input stem before bf16 attention blocks."""
def __init__(self):
super().__init__()
self.proj_in = torch.nn.Linear(2, 4)
attn = torch.nn.Module()
attn.to_q = torch.nn.Linear(4, 4).to(torch.bfloat16)
self.blocks = torch.nn.ModuleList([torch.nn.ModuleDict({"attn": attn})])
def test_rocm_probe_runs_at_the_attention_dtype_not_an_fp32_stem(monkeypatch):
from core.inference import diffusion_attention as da
pipe = _Pipe()
pipe.transformer = _MixedPrecisionDiT()
assert next(pipe.transformer.parameters()).dtype == torch.float32
assert lsp._rocm_attention_dtype(pipe) == torch.bfloat16
probed = []
monkeypatch.setattr(
da,
"sdpa_math_only",
lambda t: probed.append(t.dtype) or t.dtype == torch.float32 or t.torch_device == "cuda:1",
)
assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == ()
assert set(probed) == {torch.bfloat16}
class _PackedLinear(torch.nn.Linear):
"""diffusers' GGUFLinear surface: packed uint8 weight, ``compute_dtype``, no bias."""
def __init__(
self,
fin,
fout,
compute_dtype = None,
):
super().__init__(fin, fout, bias = False)
self.compute_dtype = compute_dtype
self.weight = torch.nn.Parameter(
torch.zeros(fout, fin, dtype = torch.uint8), requires_grad = False
)
def _pack_attention(model, compute_dtype):
for module in list(model.modules()):
for attr in ("to_q", "to_k", "to_v", "to_gate", "to_qkv"):
linear = getattr(module, attr, None)
if isinstance(linear, torch.nn.Linear):
packed = _PackedLinear(linear.in_features, linear.out_features, compute_dtype)
setattr(module, attr, packed)
return model
def _math_at_fp32_and_on_cuda1(monkeypatch):
from core.inference import diffusion_attention as da
probed = []
monkeypatch.setattr(
da,
"sdpa_math_only",
lambda t: probed.append(t.dtype) or t.dtype == torch.float32 or t.torch_device == "cuda:1",
)
return probed
@pytest.fixture(scope = "session")
def krea2():
# Must import before the 8-card fixture: xformers / torchao device checks fail against it.
return pytest.importorskip("diffusers.models.transformers.transformer_krea2")
def _tiny_krea2(krea2):
model = krea2.Krea2Transformer2DModel(
in_channels = 4,
num_layers = 2,
attention_head_dim = 16,
num_attention_heads = 2,
num_key_value_heads = 1,
intermediate_size = 32,
timestep_embed_dim = 8,
text_hidden_dim = 8,
num_text_layers = 2,
text_num_attention_heads = 2,
text_num_key_value_heads = 2,
text_intermediate_size = 16,
num_layerwise_text_blocks = 1,
num_refiner_text_blocks = 1,
axes_dims_rope = (4, 6, 6),
).to(torch.bfloat16)
for name, module in model.named_modules():
if name and any(k in name.split(".")[-1] for k in model._keep_in_fp32_modules):
module.to(torch.float32)
return model
def test_rocm_probe_reads_a_quantized_projection_not_its_fp32_qk_norm(monkeypatch, krea2):
model = _tiny_krea2(krea2)
attn = next(m for m in model.modules() if getattr(m, "to_q", None) is not None)
assert attn.norm_q.weight.dtype == torch.float32
pipe = _Pipe()
pipe.transformer = _pack_attention(model, torch.bfloat16)
assert lsp._rocm_attention_dtype(pipe) == torch.bfloat16
probed = _math_at_fp32_and_on_cuda1(monkeypatch)
assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == ()
assert set(probed) == {torch.bfloat16}
def test_rocm_probe_dtype_skips_fp32_pinned_modules_when_projections_carry_no_dtype(
monkeypatch, krea2
):
pipe = _Pipe()
pipe.transformer = _pack_attention(_tiny_krea2(krea2), None)
assert lsp._attention_dtype(pipe.transformer) is None
assert lsp._rocm_attention_dtype(pipe) == torch.bfloat16
probed = _math_at_fp32_and_on_cuda1(monkeypatch)
assert lsp._rocm_drop_math_only(pipe, 0, (1,)) == ()
assert set(probed) == {torch.bfloat16}
def test_rocm_probe_dtype_packed_projection_with_fp32_norm_synthetic():
attn = torch.nn.Module()
attn.norm_q = torch.nn.RMSNorm(4)
attn.to_q = _PackedLinear(4, 4, torch.bfloat16)
dit = torch.nn.Module()
dit.attn = attn
pipe = _Pipe()
pipe.transformer = dit
assert lsp._rocm_attention_dtype(pipe) == torch.bfloat16
def _refine_rocm_split(monkeypatch, limit_mib):
monkeypatch.setattr(lsp, "_is_rocm", lambda: True)
monkeypatch.setattr(lsp, "engine_available", lambda: True)
monkeypatch.setattr(
lsp, "_rocm_discrete_secondaries", lambda pipe, primary, cards, logger: (1,)
)
monkeypatch.setattr(
lsp,
"_denoiser_layouts",
lambda pipe: {"transformer": L(1000, (1000,) * 23)},
)
pipe = _Pipe()
pipe.transformer = torch.nn.Linear(2, 2).to(torch.bfloat16)
plan = _plan()
return plan, lsp.refine_plan_for_layer_split(
pipe,
plan,
selection = mg.make_selection(0, [0, 1]),
sizes = {"transformer": (24_000, "dit")},
primary_limit_mib = limit_mib,
secondary_limit_mib = lambda _o: limit_mib,
headroom_mib = 2000,
overhead_mib = 500,
)
def test_rocm_split_declined_on_memory_keeps_the_primarys_fused_sdpa(
_real_rocm_sdpa_probe, monkeypatch
):
# cuda:1 lacks flash, but the split does not fit, so the one-card load keeps flash on.
from core.inference import diffusion_attention as da
flags = _real_rocm_sdpa_probe
monkeypatch.setattr(
da,
"_run_rocm_sdpa_children",
lambda device, dtype: (
(da.SDPA_MATH, da.SDPA_FLASH, da.SDPA_MEM_EFFICIENT)
if device == "cuda:0"
else (da.SDPA_MATH, da.SDPA_MEM_EFFICIENT)
),
)
plan, refined = _refine_rocm_split(monkeypatch, 4000)
assert refined is plan
assert flags() == (True, True)
plan, refined = _refine_rocm_split(monkeypatch, 20_000)
assert refined is not plan and refined.layer_split.cards == (0, 1)
assert flags() == (False, True)