Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

114 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Multi-GPU diffusion layer (core/inference/diffusion_multigpu.py), CPU only.

Cross-card behaviour is exercised with fake devices: the planner reads budgets through injected callables, placement
and the boundary through a patched card predicate, and CFG-parallel with a replica on the same (CPU) device, which
checks the routing, the prefetch / constant caches and every fallback without a second card."""

from __future__ import annotations

import contextlib
import dataclasses
import threading
from collections import OrderedDict

import pytest
import torch

from core.inference import diffusion_memory as dm
from core.inference import diffusion_multigpu as mg


def _from_diffusers(module: str, name: str):
    # Not importorskip: importing diffusers can raise RuntimeError (bitsandbytes without CUDA), an environment fact.
    # Only the import is skippable; a symbol missing from an importable diffusers is a failure.
    import importlib
    try:
        loaded = importlib.import_module(module)
    except (ImportError, RuntimeError) as exc:
        pytest.skip(f"diffusers is not importable here: {type(exc).__name__}")
    return getattr(loaded, name)


_REAL_DEVICE_COUNT = torch.cuda.device_count


@pytest.fixture(autouse = True)
def _env(monkeypatch):
    monkeypatch.delenv(mg.MULTIGPU_ENV, raising = False)
    monkeypatch.delenv(mg.CFG_PARALLEL_ENV, raising = False)
    # Keep make_selection's real-host filters from collapsing fake [0, 1] selections on a one-GPU host.
    monkeypatch.setattr(torch.cuda, "device_count", lambda: 8)
    monkeypatch.setattr(mg, "_capability", lambda o: None)
    monkeypatch.setattr(mg, "_rocm_torch", lambda: False)
    yield


@pytest.fixture
def real_cuda(monkeypatch):
    # The fake count makes a CUDA wheel with no GPU report CUDA available; Dynamo and the diffusers import then touch it.
    monkeypatch.setattr(torch.cuda, "device_count", _REAL_DEVICE_COUNT)


def test_one_card_or_the_kill_switch_means_no_selection(monkeypatch):
    assert mg.make_selection(0, [0]) is None
    assert mg.make_selection(None, [0, 1]) is None
    assert mg.make_selection(1, [0, 1]) == mg.MultiGpuSelection(primary = 1, secondaries = (0,))
    monkeypatch.setenv(mg.MULTIGPU_ENV, "0")
    assert mg.make_selection(1, [0, 1]) is None
    assert mg.cfg_parallel_mode() == "off"


def test_selection_is_scoped_to_the_load_thread():
    sel = mg.make_selection(0, [0, 1])
    seen = {}
    with mg.selection_scope(sel):
        assert mg.current_selection() == sel
        t = threading.Thread(target = lambda: seen.setdefault("other", mg.current_selection()))
        t.start()
        t.join()
    assert mg.current_selection() is None
    assert seen["other"] is None


def test_resolve_selected_cuda_ordinals_translates_the_mask(monkeypatch):
    import utils.hardware.hardware as hw
    from core.inference import diffusion_device as dd

    monkeypatch.setattr(
        hw,
        "_get_parent_visible_gpu_spec",
        lambda: {"raw": None, "numeric_ids": [4, 5], "supports_explicit_gpu_ids": True},
    )
    monkeypatch.setattr(hw, "get_physical_gpu_count", lambda: 8)
    assert dd.resolve_selected_cuda_ordinals([5, 4]) == [0, 1]
    assert dd.resolve_selected_cuda_ordinals([]) == []
    with pytest.raises(ValueError):
        dd.resolve_selected_cuda_ordinals([0])


def _split(**overrides):
    kwargs = dict(
        single_card_policy = "group",
        model_dense_mib = 20000,
        companion_dense_mib = 9000,
        runtime_headroom_mib = 2000,
        base_overhead_mib = 1000,
        primary_limit_mib = 14000,
        secondary_limit_mib = lambda o: 12750,
        selection = mg.MultiGpuSelection(primary = 0, secondaries = (1,)),
    )
    kwargs.update(overrides)
    return mg.plan_companion_split(**kwargs)


def test_split_when_the_denoiser_alone_fits_the_primary():
    split = _split()
    assert split is not None and split.secondary_ordinal == 1
    assert split.primary_required_mib == 11000 + 2000 + 1000
    assert split.secondary_required_mib == 9000 + 3000


@pytest.mark.parametrize(
    "overrides",
    [
        dict(primary_limit_mib = 13000),  # denoiser + headroom do not fit the primary either
        dict(secondary_limit_mib = lambda o: 11999),  # companions + headroom = 12000
        dict(secondary_limit_mib = lambda o: None),
        dict(single_card_policy = "none"),
        dict(companion_dense_mib = None),
        dict(selection = None),
    ],
)
def test_no_split_keeps_single_card_behaviour(overrides):
    assert _split(**overrides) is None


def test_text_encoders_alone_when_the_decode_needs_the_runtime_room():
    # Secondary can't hold decode headroom: encoders alone fit, denoiser+VAE go on primary.
    kwargs = dict(
        companion_dense_mib = 11500,
        text_encoder_dense_mib = 11000,
        model_dense_mib = 16000,
        runtime_headroom_mib = 4000,
        primary_limit_mib = 12000,
        secondary_limit_mib = lambda o: 13600,
    )
    split = _split(**kwargs)
    assert split is not None and split.components == mg.TEXT_ENCODERS_ONLY
    assert split.secondary_required_mib == 11000 + mg.TEXT_ENCODER_HEADROOM_MIB + 1000
    assert split.primary_required_mib == 5000 + 4000 + 1000
    # without the encoder share the planner cannot price it and keeps one card
    assert _split(**{**kwargs, "text_encoder_dense_mib": None}) is None


def test_measured_phases_price_each_card_for_what_it_runs():
    kwargs = dict(
        model_dense_mib = 7630 + 7500 + 160,
        companion_dense_mib = 7660,
        text_encoder_dense_mib = 7500,
        runtime_headroom_mib = 8192,
        base_overhead_mib = 2048,
        primary_limit_mib = 14400,
        secondary_limit_mib = lambda o: 12495,
    )
    assert _split(**kwargs) is None
    split = _split(**kwargs, phase_headroom_mib = (1516, 1160, 2645))
    assert split is not None and split.components is None
    assert split.primary_required_mib == 7630 + 1160 + 2048
    assert split.secondary_required_mib == 7660 + 2645 + 2048


def test_force_splits_even_when_one_card_fits(monkeypatch):
    monkeypatch.setenv(mg.MULTIGPU_ENV, "force")
    assert _split(single_card_policy = "none") is not None


def test_split_picks_the_secondary_with_the_most_room():
    split = _split(
        selection = mg.MultiGpuSelection(primary = 0, secondaries = (1, 2)),
        secondary_limit_mib = lambda o: {1: 12750, 2: 34000}[o],
    )
    assert split.secondary_ordinal == 2


@dataclasses.dataclass
class _Target:
    supports_model_cpu_offload: bool = True
    ordinal: int = 0
    device: str = "cuda"
    backend: str = "cuda"
    is_cuda_torch_device: bool = True


def _memory(free = 24000, total = 24576):
    return dm.DeviceMemory(
        backend = "cuda", device = "cuda", memory_kind = "discrete_vram", free_mib = free, total_mib = total
    )


def _plan(**kw):
    args = dict(
        target = _Target(),
        device_memory = _memory(),
        model_dense_mib = 20000,
        companion_dense_mib = 9000,
        text_encoder_dense_mib = 8500,
        runtime_headroom_mib = 2000,
    )
    args.update(kw)
    return dm.plan_diffusion_memory(**args)


def test_planner_unchanged_without_a_selection():
    plan = _plan()
    assert plan.offload_policy != dm.OFFLOAD_NONE
    assert plan.secondary_ordinal is None


def test_planner_splits_under_a_selection(monkeypatch):
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)
    single = _plan()
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        plan = _plan()
        balanced = _plan(requested_mode = "balanced")
        legacy = _plan(explicit_offload = True)
        other = _plan(target = _Target(ordinal = 1))
    assert plan.offload_policy == dm.OFFLOAD_NONE and plan.secondary_ordinal == 1
    assert plan.vae_tiling is False and plan.vae_slicing is False
    assert plan.as_public_dict()["secondary_ordinal"] == 1
    assert "multi-GPU" in plan.reasons[-1]
    for p in (balanced, legacy, other):
        assert p.secondary_ordinal is None
    assert balanced.offload_policy == dm.OFFLOAD_GROUP
    assert other.offload_policy == single.offload_policy


def test_planner_split_declined_when_the_secondary_is_small(monkeypatch):
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 3400)
    single = _plan()
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        plan = _plan()
    assert plan == single


def test_planner_split_off_on_unified_memory(monkeypatch):
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)
    unified = dm.DeviceMemory(
        backend = "cuda", device = "cuda", memory_kind = "unified_memory", free_mib = 16000, total_mib = 16384
    )
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        plan = _plan(device_memory = unified)
    assert plan.secondary_ordinal is None


def test_assignment_prefers_moving_the_companions():
    sizes = {
        "transformer": (10000, "dit"),
        "text_encoder": (8000, "text_encoder"),
        "vae": (300, "other"),
    }
    moved = mg.choose_component_assignment(
        sizes,
        headroom_mib = 2000,
        overhead_mib = 1000,
        primary_limit_mib = 13500,
        secondary_limit_mib = 13000,
    )
    assert moved == ("text_encoder", "vae")


def test_assignment_can_move_only_the_text_encoders():
    sizes = {
        "transformer": (9000, "dit"),
        "text_encoder": (11000, "text_encoder"),
        "vae": (1400, "other"),
    }
    moved = mg.choose_component_assignment(
        sizes,
        headroom_mib = 4000,
        overhead_mib = 1000,
        primary_limit_mib = 16000,
        secondary_limit_mib = 13500,
    )
    assert moved == ("text_encoder",)


def test_place_split_resolves_text_encoders_only(monkeypatch):
    class Pipe:
        def __init__(self):
            self.transformer = torch.nn.Linear(2, 2)
            self.text_encoder = torch.nn.Linear(2, 2)
            self.text_encoder_2 = torch.nn.Linear(2, 2)
            self.vae = _Vae()

        @property
        def components(self):
            return dict(vars(self))

    monkeypatch.setattr(mg, "install_execution_device_patch", lambda: True)
    pipe = Pipe()
    real_device = torch.device
    monkeypatch.setattr(
        torch, "device", lambda *a: real_device("cpu") if a and a[0] == "cuda" else real_device(*a)
    )
    try:
        mg.place_split(pipe, primary = "cpu", secondary_ordinal = 1, names = mg.TEXT_ENCODERS_ONLY)
    finally:
        monkeypatch.setattr(torch, "device", real_device)
    assert mg.placement_status(pipe)["moved"] == ["text_encoder", "text_encoder_2"]
    assert "decode" not in pipe.vae.__dict__


def test_assignment_puts_the_second_expert_on_the_secondary():
    sizes = {
        "transformer": (12000, "dit"),
        "transformer_2": (12000, "dit"),
        "text_encoder": (5000, "text_encoder"),
        "vae": (500, "other"),
    }
    moved = mg.choose_component_assignment(
        sizes,
        headroom_mib = 2000,
        overhead_mib = 1000,
        primary_limit_mib = 21000,
        secondary_limit_mib = 16000,
    )
    assert moved == ("transformer_2",)
    moved = mg.choose_component_assignment(
        sizes,
        headroom_mib = 2000,
        overhead_mib = 1000,
        primary_limit_mib = 15500,
        secondary_limit_mib = 21000,
    )
    assert moved == ("transformer_2", "text_encoder", "vae")
    assert (
        mg.choose_component_assignment(
            sizes,
            headroom_mib = 2000,
            overhead_mib = 1000,
            primary_limit_mib = 9000,
            secondary_limit_mib = 9000,
        )
        is None
    )


def test_refine_plan_for_multigpu(monkeypatch):
    plan = _plan()
    assert plan.offload_policy != dm.OFFLOAD_NONE
    sizes = {
        "transformer": (9000, "dit"),
        "text_encoder": (8000, "text_encoder"),
        "vae": (300, "other"),
    }
    monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)
    monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None, **_: 1)
    assert mg.refine_plan_for_multigpu(object(), plan) is plan
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        refined = mg.refine_plan_for_multigpu(object(), plan)
    assert refined.offload_policy == dm.OFFLOAD_NONE
    assert refined.secondary_ordinal == 1
    assert refined.secondary_components == ("text_encoder", "vae")


class _Holder:
    def __init__(self, t):
        self.parameters = t
        self.mean = t * 2
        self.deterministic = False


@dataclasses.dataclass
class _DC:
    sample: torch.Tensor
    note: str = "x"


def test_move_to_device_keeps_container_types():
    t = torch.ones(2)
    Named = __import__("collections").namedtuple("Named", "a b")
    obj = {
        "tuple": (t, 1),
        "list": [t],
        "named": Named(t, "s"),
        "ordered": OrderedDict(x = t),
        "dc": _DC(t),
        "holder": _Holder(t),
        "plain": "keep",
    }
    out = mg.move_to_device(obj, torch.device("meta"))
    assert out["tuple"][0].device.type == "meta" and out["tuple"][1] == 1
    assert out["list"][0].device.type == "meta"
    assert (
        type(out["named"]) is Named
        and out["named"].a.device.type == "meta"
        and out["named"].b == "s"
    )
    assert isinstance(out["ordered"], OrderedDict) and out["ordered"]["x"].device.type == "meta"
    assert isinstance(out["dc"], _DC) and out["dc"].sample.device.type == "meta"
    assert isinstance(out["holder"], _Holder) and out["holder"].mean.device.type == "meta"
    assert out["holder"].deterministic is False and obj["holder"].parameters.device.type == "cpu"
    assert out["plain"] == "keep"
    assert mg.move_to_device(t, torch.device("cpu")) is t


def test_model_output_moves_with_its_attributes():
    BaseOutput = _from_diffusers("diffusers.utils", "BaseOutput")

    @dataclasses.dataclass
    class Out(BaseOutput):
        sample: torch.Tensor = None

    out = mg.move_to_device(Out(sample = torch.ones(1)), torch.device("meta"))
    assert out.sample.device.type == "meta" and out[0].device.type == "meta"


class _Vae(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.lin = torch.nn.Linear(2, 2)
        self.seen = []

    def forward(self, x):
        self.seen.append(("forward", x.device))
        return self.lin(x)

    def decode(
        self,
        z,
        return_dict = False,
    ):
        self.seen.append(("decode", z.device))
        return (self.lin(z),)


def test_boundary_moves_inputs_in_and_outputs_home(monkeypatch):
    vae = _Vae()
    home = torch.device("meta")
    moves = []

    def fake_move(
        obj,
        device,
        _depth = 0,
    ):
        moves.append(device)
        return obj

    monkeypatch.setattr(mg, "_crosses", lambda device, h: True)
    monkeypatch.setattr(mg, "move_to_device", fake_move)
    wrapped = mg.install_boundary(vae, home)
    assert set(wrapped) == {"forward", "decode"}
    vae.decode(torch.ones(1, 2))
    assert moves == [torch.device("cpu"), home]
    assert vae.seen[-1] == ("decode", torch.device("cpu"))
    vae(torch.ones(1, 2))
    assert moves[2:] == [torch.device("cpu"), home]
    assert mg.install_boundary(vae, home) == []
    mg.remove_boundary(vae)
    assert "decode" not in vae.__dict__ and "forward" not in vae.__dict__


def test_boundary_is_inert_on_the_same_card():
    vae = _Vae()
    mg.install_boundary(vae, torch.device("cpu"))
    x = torch.ones(1, 2)
    assert torch.equal(vae(x), vae.lin(x))


def test_execution_device_reports_the_primary_of_a_split_pipeline(monkeypatch, real_cuda):
    monkeypatch.setattr(mg, "_SPLIT_EVER_PLACED", True)
    DiffusionPipeline = _from_diffusers("diffusers", "DiffusionPipeline")

    class Pipe(DiffusionPipeline):
        def __init__(self, vae, transformer):
            super().__init__()
            self.register_modules(vae = vae, transformer = transformer)

    class Mod(torch.nn.Linear):
        @property
        def device(self):
            return self.weight.device

    vae = Mod(1, 1).to("meta")
    transformer = Mod(1, 1)
    pipe = Pipe(vae, transformer)
    assert mg.install_execution_device_patch()
    before = pipe.device
    vae._unsloth_split_home = torch.device("meta")
    assert pipe.device == torch.device("meta")
    del vae._unsloth_split_home
    transformer._unsloth_split_home = torch.device("cpu")
    assert pipe._execution_device == torch.device("cpu")
    assert pipe.device == torch.device("cpu")
    del transformer._unsloth_split_home
    assert pipe.device == before


def test_execution_device_reports_the_primary_of_a_split_modular_pipeline(monkeypatch, real_cuda):
    """ModularPipeline reports its first component's card (the text encoder, on the secondary): step 0 device mismatch."""
    monkeypatch.setattr(mg, "_SPLIT_EVER_PLACED", True)
    import importlib

    try:
        diffusers = importlib.import_module("diffusers")
    except (ImportError, RuntimeError) as exc:
        pytest.skip(f"diffusers is not importable here: {type(exc).__name__}")
    ModularPipeline = getattr(diffusers, "ModularPipeline", None)
    if ModularPipeline is None:
        pytest.skip("this diffusers has no modular pipelines")

    class Mod(torch.nn.Linear):
        @property
        def device(self):
            return self.weight.device

    class Pipe(ModularPipeline):
        def __init__(self, text_encoder, transformer):
            self._component_specs = {"text_encoder": None, "transformer": None}
            self.text_encoder = text_encoder
            self.transformer = transformer

    text_encoder = Mod(1, 1).to("meta")  # the secondary card of the split
    transformer = Mod(1, 1)  # the primary
    pipe = Pipe(text_encoder, transformer)
    assert pipe.device == torch.device("meta")  # stock: the first component's card
    mg.install_execution_device_patch()
    transformer._unsloth_split_home = torch.device("cpu")
    assert pipe._execution_device == torch.device("cpu")
    assert pipe.device == torch.device("cpu")
    del transformer._unsloth_split_home
    assert pipe.device == torch.device("meta")  # no split: diffusers' own answer again


def test_place_split_and_fallback(monkeypatch):
    class Pipe:
        def __init__(self):
            self.transformer = torch.nn.Linear(2, 2)
            self.text_encoder = torch.nn.Linear(2, 2)
            self.vae = _Vae()
            self.scheduler = object()

        @property
        def components(self):
            return {
                "transformer": self.transformer,
                "text_encoder": self.text_encoder,
                "vae": self.vae,
                "scheduler": self.scheduler,
            }

        def to(self, device):
            for m in (self.transformer, self.text_encoder, self.vae):
                m.to(device)

    pipe = Pipe()
    assert mg.companion_names(pipe) == ("text_encoder", "vae")
    moved_to = {}
    real_device = torch.device

    def fake_device(*args):
        if args and args[0] == "cuda":
            return real_device("cpu")
        return real_device(*args)

    monkeypatch.setattr(mg, "install_execution_device_patch", lambda: True)
    monkeypatch.setattr(torch, "device", fake_device)
    try:
        placement = mg.place_split(pipe, primary = "cpu", secondary_ordinal = 1)
    finally:
        monkeypatch.setattr(torch, "device", real_device)
    assert set(placement) == {"transformer", "text_encoder", "vae"}
    assert pipe.transformer._unsloth_split_home == torch.device("cpu")
    assert "decode" in pipe.vae.__dict__
    status = mg.placement_status(pipe)
    assert status["moved"] == ["text_encoder", "vae"]
    mg.undo_split(pipe)
    assert (
        "decode" not in pipe.vae.__dict__ and "_unsloth_split_home" not in pipe.transformer.__dict__
    )
    assert mg.placement_status(pipe) is None
    del moved_to

    @dataclasses.dataclass
    class P:
        secondary_ordinal: int = 1
        secondary_components: tuple = None

    def boom(*a, **k):
        raise RuntimeError("CUDA out of memory")

    monkeypatch.setattr(mg, "place_split", boom)
    assert mg.apply_split_plan(pipe, P(), placement = "cpu") is False


def test_apply_memory_plan_falls_back_to_model_offload(monkeypatch):
    calls = []

    class Pipe:
        components: dict = {}

        def enable_model_cpu_offload(self, device = None):
            calls.append(("offload", device))

    plan = dataclasses.replace(_plan(), offload_policy = dm.OFFLOAD_NONE, secondary_ordinal = 1)
    monkeypatch.setattr(mg, "apply_split_plan", lambda pipe, plan, placement, logger = None: False)
    monkeypatch.setattr(dm, "keep_cpu_weights_on_offload", lambda pipe, logger = None: 0)
    pipe = Pipe()
    policy, _tiling = dm.apply_memory_plan(pipe, plan, device = "cuda", placement_device = "cuda:0")
    assert policy == dm.OFFLOAD_MODEL and calls == [("offload", "cuda:0")]
    applied = dm.applied_plan(pipe, plan)
    assert applied.offload_policy == dm.OFFLOAD_MODEL and applied.secondary_ordinal is None

    placed = []
    monkeypatch.setattr(
        mg, "apply_split_plan", lambda pipe, plan, placement, logger = None: placed.append(1) or True
    )
    policy, _tiling = dm.apply_memory_plan(Pipe(), plan, device = "cuda", placement_device = "cuda:0")
    assert policy == dm.OFFLOAD_NONE and placed == [1]


class _Dit(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.proj = torch.nn.Linear(4, 4)
        self.calls = 0

    def forward(
        self,
        hidden_states,
        timestep,
        encoder_hidden_states,
        return_dict = False,
    ):
        self.calls += 1
        return (self.proj(hidden_states) * timestep + encoder_hidden_states.sum(),)


def _router(dit, **kw):
    replicas = []

    def factory():
        r = mg.build_replica(dit, torch.device("cpu"))
        replicas.append(r)
        return r

    router = mg.CfgParallelRouter(
        dit, primary = "cpu", secondary = "cpu", replica_factory = factory, **kw
    )
    dit.forward = router
    return router, replicas


def _cfg_loop(
    dit,
    latents,
    steps,
    branches = 2,
    scale = 4.0,
):
    cond = torch.randn(3, 4, generator = torch.Generator().manual_seed(1))
    uncond = torch.randn(3, 4, generator = torch.Generator().manual_seed(2))
    extra = torch.randn(3, 4, generator = torch.Generator().manual_seed(3))
    for i in range(steps):
        t = torch.tensor(1.0 - i / steps)
        outs = [
            dit(hidden_states = latents, timestep = t, encoder_hidden_states = e)[0]
            for e in (cond, uncond, extra)[:branches]
        ]
        if branches == 1:
            pred = outs[0]
        else:
            pred = outs[1] + scale * (outs[0] - outs[1])
            if branches == 3:
                pred = pred + 0.5 * (outs[0] - outs[2])
        latents = latents - 0.1 * pred
    return latents


@pytest.mark.parametrize("branches", [2, 3])
def test_cfg_parallel_is_exact_and_routes_odd_branches(branches):
    torch.manual_seed(0)
    dit = _Dit()
    latents = torch.randn(2, 4)
    reference = _cfg_loop(dit, latents.clone(), steps = 5, branches = branches)
    router, replicas = _router(dit)
    out = _cfg_loop(dit, latents.clone(), steps = 5, branches = branches)
    assert torch.equal(out, reference)
    assert len(replicas) == 1 and replicas[0] is not dit
    assert router.stats["steps_parallel"] == 4
    assert router.stats["replica_calls"] == 4 * (branches // 2)
    assert router.describe()["disabled_reason"] is None
    assert replicas[0].proj.weight is not dit.proj.weight
    assert torch.equal(replicas[0].proj.weight, dit.proj.weight)


def test_guidance_distilled_model_never_builds_a_replica():
    dit = _Dit()
    router, replicas = _router(dit)
    _cfg_loop(dit, torch.randn(2, 4), steps = 6, branches = 1)
    assert replicas == [] and router.replica is None
    assert router.stats["replica_calls"] == 0


def test_replica_failure_recomputes_on_the_primary():
    torch.manual_seed(0)
    dit = _Dit()
    latents = torch.randn(2, 4)
    reference = _cfg_loop(dit, latents.clone(), steps = 4)
    router, replicas = _router(dit)

    class Broken(torch.nn.Module):
        def forward(self, *a, **k):
            raise RuntimeError("replica kernel failed")

    router.replica_factory = lambda: Broken()
    out = _cfg_loop(dit, latents.clone(), steps = 4)
    assert torch.equal(out, reference)
    assert "replica call failed" in router.describe()["disabled_reason"]
    assert router.replica is None


def test_replica_declined_when_a_submodule_hook_captures_the_source():
    dit = _Dit()
    source = dit.proj
    dit.proj.register_forward_hook(lambda m, i, o: o + source.bias)
    router, replicas = _router(dit)
    _cfg_loop(dit, torch.randn(2, 4), steps = 3)
    assert replicas == []
    assert "proj._forward_hooks" in router.describe()["disabled_reason"]


def test_portable_hooks_and_root_hooks_are_accepted():
    torch.manual_seed(0)
    dit = _Dit()
    scale = 0.5
    hook = lambda m, i, o: o * scale  # noqa: E731 - captures only a float: shared safely
    hook.__module__ = mg.__name__
    dit.proj.register_forward_hook(hook)
    calls = []
    root_hook = lambda m, a: calls.append(1)  # noqa: E731 - root: runs on the source around every branch
    root_hook.__module__ = mg.__name__
    dit.register_forward_pre_hook(root_hook)
    assert mg._hooks_present(dit) is None
    latents = torch.randn(2, 4)
    reference = _cfg_loop(dit, latents.clone(), steps = 4)
    n_ref = len(calls)
    router, replicas = _router(dit)
    out = _cfg_loop(dit, latents.clone(), steps = 4)
    assert torch.equal(out, reference)
    assert router.stats["replica_calls"] == 3
    # the root hook ran exactly once per denoiser call, routed or not (stripped from the copy)
    assert len(calls) == 2 * n_ref
    assert not replicas[0]._forward_pre_hooks


def test_closure_forward_override_is_declined_but_bound_methods_are_rebound():
    import types

    dit = _Dit()
    source = dit.proj
    dit.proj.forward = lambda x: source.weight @ x  # a closure would keep reading the source module
    assert "instance override" in mg._hooks_present(dit)
    dit2 = _Dit()
    dit2.proj.forward = types.MethodType(
        lambda self, x: torch.nn.functional.linear(x, self.weight), dit2.proj
    )
    assert mg._hooks_present(dit2) is None
    replica = mg.build_replica(dit2, torch.device("cpu"))
    assert replica.proj.forward.__self__ is replica.proj


def test_replica_reapplies_recorded_compile(monkeypatch, real_cuda):
    mg.install_compile_recorder()
    dit = _Dit()
    dit.proj.compile(backend = "eager")
    assert dit.proj.__dict__["_unsloth_compile_args"] == ((), {"backend": "eager"})
    replica = mg.build_replica(dit, torch.device("cpu"))
    assert replica._unsloth_replica_compiled == 1
    assert replica.proj._compiled_call_impl is not None
    assert replica.proj._compiled_call_impl is not dit.proj._compiled_call_impl
    x = torch.randn(2, 4)
    assert torch.equal(replica.proj(x), dit.proj(x))


def test_compiled_without_recorded_arguments_is_refused():
    dit = _Dit()
    dit.proj._compiled_call_impl = lambda *a, **k: None
    with pytest.raises(RuntimeError, match = "recorded compile"):
        mg.build_replica(dit, torch.device("cpu"))


def test_worker_propagates_grad_mode_and_context_variables():
    var = __import__("contextvars").ContextVar("probe", default = "unset")
    worker = mg._ReplicaWorker(torch.device("cpu"))
    try:
        var.set("caller")
        with torch.inference_mode():
            seen = worker.call(
                lambda: (
                    torch.is_inference_mode_enabled(),
                    var.get(),
                    threading.current_thread().name,
                )
            )
        assert seen[0] is True and seen[1] == "caller" and seen[2].startswith("unsloth-cfg-replica")
        with pytest.raises(ValueError):
            worker.call(lambda: (_ for _ in ()).throw(ValueError("boom")))
    finally:
        worker.close()


def test_branch_tensors_are_prefetched_fresh_every_step():
    torch.manual_seed(0)
    dit = _Dit()
    latents = torch.randn(2, 4)
    reference = _cfg_loop(dit, latents.clone(), steps = 4)
    router, _replicas = _router(dit)
    copies = []
    real_prefetch = router._prefetch

    def spy(args, kwargs):
        real_prefetch(args, kwargs)
        copies.append(len(router.carry))

    router._prefetch = spy
    out = _cfg_loop(dit, latents.clone(), steps = 4)
    assert torch.equal(out, reference)
    # Prefetch runs before every primary branch from step 1. Step 1 has no carry yet (step 0 built the replica on its
    # odd branch); from step 2 the previous step's odd-branch tensors except its latent (timestep, uncond embeds).
    assert copies == [0, 2, 2]


def test_cfg_parallel_decline_reasons(monkeypatch):
    class Pipe:
        transformer = _Dit()

    sel = mg.MultiGpuSelection(primary = 0, secondaries = (1,))
    assert (
        mg.cfg_parallel_decline(Pipe(), offload_policy = "none", selection = None)
        == "one card selected"
    )
    assert "offloaded" in mg.cfg_parallel_decline(Pipe(), offload_policy = "model", selection = sel)
    assert "CUDA" in mg.cfg_parallel_decline(Pipe(), offload_policy = "group", selection = sel)
    hooked = Pipe()
    hooked.transformer = _Dit()
    source = hooked.transformer.proj
    hooked.transformer.proj.register_forward_pre_hook(lambda m, a: source)
    assert "hooked" in mg.cfg_parallel_decline(hooked, offload_policy = "group", selection = sel)
    assert "CUDA" in mg.cfg_parallel_decline(Pipe(), offload_policy = "none", selection = sel)

    class Unet:
        transformer = None

    assert "U-Net" in mg.cfg_parallel_decline(Unet(), offload_policy = "none", selection = sel)
    monkeypatch.setenv(mg.CFG_PARALLEL_ENV, "0")
    assert mg.CFG_PARALLEL_ENV in mg.cfg_parallel_decline(
        Pipe(), offload_policy = "none", selection = sel
    )


def test_status_reports_a_declined_cfg_parallel():
    class Pipe:
        transformer = _Dit()

    pipe = Pipe()
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        reason = mg.install_cfg_parallel(pipe, offload_policy = "model")
    assert "offloaded" in reason
    assert mg.placement_status(pipe) == {"cfg_parallel": {"armed": False, "declined": reason}}
    other = Pipe()
    assert mg.install_cfg_parallel(other, offload_policy = "none") == "one card selected"
    assert mg.placement_status(other) is None


def test_replica_is_rebuilt_when_the_weights_change():
    torch.manual_seed(0)
    dit = _Dit()
    latents = torch.randn(2, 4)
    router, replicas = _router(dit)
    _cfg_loop(dit, latents.clone(), steps = 3)
    assert len(replicas) == 1
    with torch.no_grad():
        dit.proj.weight.mul_(0.5)
    reference_router = dit.__dict__.pop("forward")
    reference = _cfg_loop(dit, latents.clone(), steps = 4)
    dit.forward = reference_router
    out = _cfg_loop(dit, latents.clone(), steps = 4)
    assert torch.equal(out, reference)
    assert router.stats["replica_rebuilds"] == 1 and len(replicas) == 2
    assert torch.equal(replicas[1].proj.weight, dit.proj.weight)


def test_adapter_scale_changes_the_fingerprint():
    dit = _Dit()
    before = mg.weights_fingerprint(dit)
    dit.proj.scaling = {"default": 1.0}
    with_adapter = mg.weights_fingerprint(dit)
    dit.proj.scaling = {"default": 0.5}
    assert len({before, with_adapter, mg.weights_fingerprint(dit)}) == 3


def test_with_load_selection_scopes_and_pops_the_ordinals():
    seen = {}

    class Backend:
        @mg.with_load_selection
        def _run_load(self, **kwargs):
            """body"""
            seen["kwargs"] = dict(kwargs)
            seen["selection"] = mg.current_selection()

    Backend()._run_load(gpu_ordinal = 1, _gpu_ordinals = [0, 1], repo_id = "x")
    assert seen["kwargs"] == {"gpu_ordinal": 1, "repo_id": "x"}
    assert seen["selection"] == mg.MultiGpuSelection(primary = 1, secondaries = (0,))
    assert mg.current_selection() is None
    Backend()._run_load(gpu_ordinal = 1, repo_id = "x")
    assert seen["selection"] is None
    import inspect

    assert '"""body"""' in inspect.getsource(Backend._run_load)


def test_portability_looks_inside_captured_containers():
    holder = {"m": torch.nn.Linear(1, 1)}
    assert not mg._portable(lambda m, a: holder)
    plain = {"scale": 2.0}
    # even scalar-only containers are shared mutable state between the source and the replica
    assert not mg._portable(lambda m, a: plain)
    frozen = ("scale", 2.0)
    assert mg._portable(lambda m, a: frozen)


class _BlockDit(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.proj_in = torch.nn.Linear(4, 4)
        self.blocks = torch.nn.ModuleList([torch.nn.Linear(4, 4) for _ in range(3)])

    def forward(
        self,
        hidden_states,
        timestep,
        encoder_hidden_states,
        return_dict = False,
    ):
        x = self.proj_in(hidden_states) * timestep
        for block in self.blocks:
            x = block(x) + encoder_hidden_states.sum()
        return (x,)


def test_replica_drops_group_offload_and_stays_exact():
    apply_group_offloading = _from_diffusers("diffusers.hooks", "apply_group_offloading")

    torch.manual_seed(0)
    dit = _BlockDit()
    latents = torch.randn(2, 4)
    reference = _cfg_loop(dit, latents.clone(), steps = 4)
    apply_group_offloading(
        dit,
        onload_device = torch.device("cpu"),
        offload_device = torch.device("cpu"),
        offload_type = "block_level",
        num_blocks_per_group = 1,
    )
    assert mg._hooks_present(dit) is None
    assert mg._registry_kind(dit) == "offload"
    router, replicas = _router(dit)
    out = _cfg_loop(dit, latents.clone(), steps = 4)
    assert torch.equal(out, reference)
    assert router.stats["replica_calls"] == 3
    replica = replicas[0]
    for sub in replica.modules():
        assert "_diffusers_hook" not in sub.__dict__
        assert "forward" not in sub.__dict__


def test_step_cache_registry_is_declined():
    class Registry:
        hooks = {"first_block_cache": object()}

    dit = _Dit()
    dit._diffusers_hook = Registry()
    assert "first_block_cache" in mg._hooks_present(dit)


def test_offload_torch_hooks_are_stripped_from_the_replica():
    dit = _Dit()
    source = dit.proj
    calls = []

    def prefetch_begin(module, args):
        calls.append(source)

    prefetch_begin.__module__ = "core.inference.diffusion_offload_prefetch"
    dit.proj.register_forward_pre_hook(prefetch_begin)
    assert mg._hooks_present(dit) is None
    replica = mg.build_replica(dit, torch.device("cpu"))
    assert not replica.proj._forward_pre_hooks
    replica(
        hidden_states = torch.ones(1, 4),
        timestep = torch.tensor(1.0),
        encoder_hidden_states = torch.ones(1, 4),
    )
    assert calls == []


def test_the_conditioning_block_cache_stays_with_the_source():
    """FLUX.3's conditioning-block cache lives in the source's __dict__ and holds source-card tensors; the replica gets
    the states through the forwarded call, so a copy would stay on the source card and never be cleared."""
    dit = _Dit()
    states = {"temb": torch.ones(2, 4)}
    dit.__dict__["_unsloth_cond_cache"] = {("key",): states}
    replica = mg.build_replica(dit, torch.device("cpu"))
    assert "_unsloth_cond_cache" not in replica.__dict__
    assert dit.__dict__["_unsloth_cond_cache"] == {("key",): states}
    out = replica(
        hidden_states = torch.ones(1, 4),
        timestep = torch.tensor(1.0),
        encoder_hidden_states = torch.ones(1, 4),
    )[0]
    assert torch.equal(
        out,
        dit(
            hidden_states = torch.ones(1, 4),
            timestep = torch.tensor(1.0),
            encoder_hidden_states = torch.ones(1, 4),
        )[0],
    )


def test_predicted_branch_hits_and_stays_exact():
    torch.manual_seed(0)
    dit = _Dit()
    latents = torch.randn(2, 4)
    reference = _cfg_loop(dit, latents.clone(), steps = 6)
    router, _replicas = _router(dit)
    out = _cfg_loop(dit, latents.clone(), steps = 6)
    assert torch.equal(out, reference)
    # the replica's first call (it may compile) runs unpredicted, after the primary branch
    assert router.stats["predicted_hits"] == 4 and router.stats["predicted_misses"] == 0
    assert router.stats["replica_calls"] == 5


def test_mispredicted_branch_is_recomputed_exactly():
    torch.manual_seed(0)
    dit = _Dit()
    latents = torch.randn(2, 4)
    uncond = torch.randn(3, 4)

    def loop(steps):
        x = latents.clone()
        cond = torch.randn(3, 4, generator = torch.Generator().manual_seed(1))
        for i in range(steps):
            t = torch.tensor(1.0 - i / steps)
            a = dit(hidden_states = x, timestep = t, encoder_hidden_states = cond)[0]
            # a fresh negative tensor every step (equal values, new object): the prediction cannot be trusted
            b = dit(hidden_states = x, timestep = t, encoder_hidden_states = uncond.clone())[0]
            x = x - 0.1 * (b + 4.0 * (a - b))
        return x

    reference = loop(5)
    router, _replicas = _router(dit)
    out = loop(5)
    assert torch.equal(out, reference)
    assert router.stats["predicted_hits"] == 0 and router.stats["predicted_misses"] == 3
    assert router.stats["replica_calls"] == 4


def test_inline_recomputed_scalars_are_predicted_by_value():
    torch.manual_seed(0)
    dit = _Dit()
    latents = torch.randn(2, 4)
    cond = torch.randn(3, 4)
    uncond = torch.randn(3, 4)

    def loop(steps):
        x = latents.clone()
        for i in range(steps):
            t = torch.tensor(1000.0 * (1.0 - i / steps))
            a = dit(hidden_states = x, timestep = t / 1000, encoder_hidden_states = cond)[0]
            b = dit(hidden_states = x, timestep = t / 1000, encoder_hidden_states = uncond)[0]
            x = x - 0.1 * (b + 4.0 * (a - b))
        return x

    reference = loop(6)
    router, _replicas = _router(dit)
    out = loop(6)
    assert torch.equal(out, reference)
    assert router.stats["predicted_hits"] == 4 and router.stats["predicted_misses"] == 0


def test_bitwise_equal_on_an_expanded_scalar():
    t = torch.tensor(941).expand(1)
    assert t.stride() == (0,)
    assert mg._bitwise_equal(t, t.clone())
    assert mg._bitwise_equal(t, torch.tensor(941).expand(1))
    assert not mg._bitwise_equal(t, torch.tensor([942]))
    assert mg._bitwise_equal(torch.tensor(float("nan")).expand(1), torch.tensor([float("nan")]))
    assert not mg._bitwise_equal(torch.tensor(-0.0).expand(1), torch.tensor([0.0]))
    assert mg._unchanged_since_launch([(t, t.clone())])
    assert not mg._unchanged_since_launch([(torch.tensor(942).expand(1), t.clone())])


def test_bitwise_equal_distinguishes_signed_zero():
    assert mg._bitwise_equal(torch.tensor([0.0]), torch.tensor([0.0]))
    assert mg._bitwise_equal(torch.tensor(0.5), torch.tensor(1.0) / 2)
    assert not mg._bitwise_equal(torch.tensor([0.0]), torch.tensor([-0.0]))
    assert not mg._bitwise_equal(torch.tensor([1.0]), torch.tensor([1.0], dtype = torch.float64))


def test_failed_split_applies_the_single_card_plan(monkeypatch):
    calls = []

    class Pipe:
        components: dict = {}

        def enable_model_cpu_offload(self, device = None):
            calls.append(("model", device))

    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        plan = _plan()
    assert plan.secondary_ordinal == 1
    assert plan.fallback_plan is not None and plan.fallback_plan.offload_policy == dm.OFFLOAD_GROUP
    monkeypatch.setattr(mg, "apply_split_plan", lambda pipe, plan, placement, logger = None: False)
    monkeypatch.setattr(
        dm,
        "_apply_group_offload",
        lambda pipe, placement, logger, **kw: calls.append(("group", placement)) or True,
    )
    pipe = Pipe()
    policy, _tiling = dm.apply_memory_plan(pipe, plan, device = "cuda", placement_device = "cuda:0")
    assert policy == dm.OFFLOAD_GROUP and calls == [("group", "cuda:0")]
    assert dm.applied_plan(pipe, plan) is plan.fallback_plan


def test_post_load_split_keeps_the_original_plan_as_fallback(monkeypatch):
    plan = _plan()
    sizes = {
        "transformer": (9000, "dit"),
        "text_encoder": (8000, "text_encoder"),
        "vae": (300, "other"),
    }
    monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)
    monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None, **_: 1)
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        refined = mg.refine_plan_for_multigpu(object(), plan)
    assert refined.fallback_plan is plan


def test_hook_with_a_mutable_closure_counter_is_declined():
    dit = _Dit()
    counter = [0]

    def count(module, args):
        counter[0] += 1

    dit.proj.register_forward_pre_hook(count)
    assert "proj._forward_pre_hooks" in mg._hooks_present(dit)
    router, replicas = _router(dit)
    _cfg_loop(dit, torch.randn(2, 4), steps = 3)
    assert replicas == [] and "hooks" in router.describe()["disabled_reason"]


def test_replica_is_dropped_when_the_source_gains_a_cache_hook():
    # transformer_cache=auto installs a stateful step cache on the SOURCE at generate time, after the replica exists
    torch.manual_seed(0)
    dit = _Dit()
    router, replicas = _router(dit)
    _cfg_loop(dit, torch.randn(2, 4), steps = 3)
    assert len(replicas) == 1

    class Registry:
        hooks = {"fbc_leader": object()}

    dit._diffusers_hook = Registry()
    _cfg_loop(dit, torch.randn(2, 4), steps = 2)
    assert router.replica is None
    assert "fbc_leader" in router.describe()["disabled_reason"]


def test_post_load_split_keeps_vae_tiling(monkeypatch):
    plan = dataclasses.replace(_plan(), vae_tiling = True, vae_slicing = True)
    sizes = {
        "transformer": (9000, "dit"),
        "text_encoder": (8000, "text_encoder"),
        "vae": (300, "other"),
    }
    monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)
    monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None, **_: 1)
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        refined = mg.refine_plan_for_multigpu(object(), plan)
    assert refined.secondary_ordinal == 1
    assert refined.vae_tiling is True and refined.vae_slicing is True


def test_replica_keeps_a_bound_root_forward_override():
    import types

    torch.manual_seed(0)
    dit = _Dit()

    def doubled(
        self,
        hidden_states,
        timestep,
        encoder_hidden_states,
        return_dict = False,
    ):
        return (2 * type(self).forward(self, hidden_states, timestep, encoder_hidden_states)[0],)

    dit.forward = types.MethodType(doubled, dit)
    latents = torch.randn(2, 4)
    reference = _cfg_loop(dit, latents.clone(), steps = 4)
    router, replicas = _router(dit)
    out = _cfg_loop(dit, latents.clone(), steps = 4)
    assert torch.equal(out, reference)
    assert router.stats["replica_calls"] == 3
    assert replicas[0].forward.__self__ is replicas[0]


def test_in_place_edit_after_launch_rejects_the_prediction():
    torch.manual_seed(0)
    dit = _Dit()
    latents = torch.randn(2, 4)
    cond = torch.randn(3, 4)
    uncond = torch.randn(3, 4)

    def loop(steps):
        x = latents.clone()
        neg = uncond.clone()
        for i in range(steps):
            t = torch.tensor(1.0 - i / steps)
            a = dit(hidden_states = x, timestep = t, encoder_hidden_states = cond)[0]
            neg.mul_(1.01)  # same object, new values, after the prediction was launched
            b = dit(hidden_states = x, timestep = t, encoder_hidden_states = neg)[0]
            x = x - 0.1 * (b + 4.0 * (a - b))
        return x

    with torch.inference_mode():
        reference = loop(5)
        router, _replicas = _router(dit)
        out = loop(5)
    assert torch.equal(out, reference)
    assert router.stats["predicted_hits"] == 0 and router.stats["predicted_misses"] == 3
    router.prefetched = {id(uncond): (uncond, uncond)}
    router.pending = {
        "leaves": [],
        "by_value": set(),
        "hook_state": None,
        "snapshots": [],
        "job": None,
    }
    assert router._take_prediction((), {"x": torch.zeros(1)}) is None
    assert router.prefetched == {}


def test_deferred_speed_profile_invalidates_the_replica(monkeypatch):
    # The 3rd generation compiles / patches the source denoiser; a replica built during the eager images must go.
    import types as _types

    from core.inference.diffusion import DiffusionBackend

    torch.manual_seed(0)
    dit = _Dit()
    router, replicas = _router(dit)

    class Pipe:
        transformer = dit

    pipe = Pipe()
    mg._CFG_ROUTERS[pipe] = router
    _cfg_loop(dit, torch.randn(2, 4), steps = 3)
    assert router.replica is not None
    state = _types.SimpleNamespace(
        pipe = pipe,
        speed_deferred = True,
        deferred_static_plan = None,
        resolved = {},
        transformer_cache = None,
    )
    fake_self = _types.SimpleNamespace(_engage_deferred_profile = lambda state: None)
    DiffusionBackend._engage_deferred_speed(fake_self, state)
    assert router.replica is None and router.disabled_reason is None
    _cfg_loop(dit, torch.randn(2, 4), steps = 2)
    assert len(replicas) == 2
    assert router.replica is not None

    def broken_profile(state):
        raise RuntimeError("compile failed")

    with pytest.raises(RuntimeError, match = "compile failed"):
        DiffusionBackend._engage_deferred_speed(
            _types.SimpleNamespace(_engage_deferred_profile = broken_profile), state
        )
    assert router.replica is None


class _FlagLinear(torch.nn.Linear):
    def forward(self, x):
        out = super().forward(x)
        return -out if getattr(self, "_unsloth_flip", False) else out


class _FlagDit(_Dit):
    def __init__(self):
        super().__init__()
        self.proj = _FlagLinear(4, 4)


def _flag_pre_hook(module, args, kwargs):
    module.proj._unsloth_flip = bool(kwargs["encoder_hidden_states"].sum() < 0) != bool(
        kwargs["timestep"] < 0.5
    )


def _flag_post_hook(module, args, output):
    module.proj.__dict__.pop("_unsloth_flip", None)


_flag_pre_hook.__module__ = mg.__name__
_flag_post_hook.__module__ = mg.__name__


def _plain_state_pre_hook(module, args, kwargs):
    # a foreign root hook keeping per-call state in an ordinary attribute, which the replica would never see
    module.proj.flag = bool(kwargs["encoder_hidden_states"].sum() < 0)


def test_foreign_root_hook_declines_the_replica():
    dit = _FlagDit()
    dit.register_forward_pre_hook(_plain_state_pre_hook, with_kwargs = True)
    assert "a hook outside Unsloth" in mg._hooks_present(dit)
    router, replicas = _router(dit)
    _cfg_loop(dit, torch.randn(2, 4), steps = 3)
    assert replicas == [] and router.stats["replica_calls"] == 0


def test_root_hook_state_on_submodules_reaches_the_replica():
    torch.manual_seed(0)
    dit = _FlagDit()
    dit.register_forward_pre_hook(_flag_pre_hook, with_kwargs = True)
    dit.register_forward_hook(_flag_post_hook)
    latents = torch.randn(2, 4)
    cond = torch.ones(3, 4)
    neg = -torch.ones(3, 4)

    def loop(steps):
        x = latents.clone()
        for i in range(steps):
            t = torch.tensor(1.0 - i / steps)
            a = dit(hidden_states = x, timestep = t, encoder_hidden_states = cond)[0]
            b = dit(hidden_states = x, timestep = t, encoder_hidden_states = neg)[0]
            x = x - 0.1 * (b + 4.0 * (a - b))
        return x

    reference = loop(5)
    router, _replicas = _router(dit)
    out = loop(5)
    assert torch.equal(out, reference)
    assert router.stats["replica_calls"] == 4


def test_runtime_controlnet_streams_beside_a_split_pipeline(monkeypatch):
    # A split reports policy "none", but its fit priced only the base pipeline: a ControlNet added at generate time
    # must stream as the single-card offload plan would have, not land whole on the primary.
    import types as _types

    from core.inference.diffusion import _controlnet_streams

    class Pipe:
        pass

    pipe = Pipe()
    state = _types.SimpleNamespace(pipe = pipe, offload_policy = "none", calibrated_placement = False)
    assert _controlnet_streams(state) is False  # one card, resident: unchanged
    mg._PLACEMENTS[pipe] = {"moved": ["text_encoder", "vae"]}
    assert _controlnet_streams(state) is True
    assert _controlnet_streams(_types.SimpleNamespace(pipe = Pipe(), offload_policy = "group")) is True


def test_generate_guard_checks_the_vae_card(monkeypatch):
    # The guard reads the primary's free memory; with the VAE on the secondary, the decode must be checked there.
    import types as _types

    from core.inference import diffusion as D

    seen = []

    @contextlib.contextmanager
    def scope(ordinal):
        seen.append(("scope", ordinal))
        yield

    def shortfall(**kw):
        seen.append(("guard", kw["device_memory"], kw.get("vae_tile_side")))
        return _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)

    monkeypatch.setattr(D, "diffusion_device_scope", scope)
    monkeypatch.setattr(D, "reclaimable_snapshot_device_memory", lambda target: "card1-free")
    monkeypatch.setattr(D, "raise_on_image_activation_shortfall", shortfall)
    monkeypatch.setattr(D, "engage_vae_tiling", lambda pipe, logger = None: ("restore", True))

    class Vae(torch.nn.Module):
        def __init__(self):
            super().__init__()
            self.w = torch.nn.Parameter(torch.zeros(1))

    pipe = _types.SimpleNamespace(vae = Vae())
    target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)
    monkeypatch.setattr(mg, "is_split", lambda p: True)
    monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))
    restore, tiled = D._guard_vae_card(
        pipe, target, {"device_memory": "card0-free", "vae_tile_side": 512}, None, None
    )
    assert ("scope", 1) in seen and ("guard", "card1-free", 512) in seen
    assert (restore, tiled) == ("restore", True)
    monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))
    seen.clear()
    assert D._guard_vae_card(pipe, target, {"device_memory": "card0-free"}, None, None) == (
        None,
        False,
    )
    assert seen == []


def test_secondary_of_lower_compute_capability_takes_no_component(monkeypatch):
    caps = {0: (8, 9), 1: (7, 5), 2: (8, 9), 3: (9, 0)}
    monkeypatch.setattr(mg, "_capability", lambda o: caps.get(int(o)))
    assert mg.make_selection(0, [0, 1, 2, 3]) == mg.MultiGpuSelection(primary = 0, secondaries = (2, 3))
    assert mg.make_selection(0, [0, 1]) is None
    assert mg.make_selection(1, [1, 0]) == mg.MultiGpuSelection(primary = 1, secondaries = (0,))
    assert mg.make_selection(0, [0, 7]) == mg.MultiGpuSelection(primary = 0, secondaries = (7,))


class _Cache:
    """A prefix-KV style cache: a value holder whose layer tensors the denoiser reads and writes."""

    def __init__(self):
        self.keys = [torch.ones(4)]
        self.writers: list = []


class _CacheDit(_Dit):
    def forward(
        self,
        hidden_states,
        timestep,
        encoder_hidden_states,
        kv_cache = None,
        return_dict = False,
    ):
        if kv_cache is not None:
            kv_cache.writers.append(threading.current_thread())
            hidden_states = hidden_states + kv_cache.keys[0]
        return super().forward(hidden_states, timestep, encoder_hidden_states)


def test_branch_with_a_cache_object_stays_on_the_primary():
    torch.manual_seed(0)
    dit = _CacheDit()
    cond, uncond = torch.randn(3, 4), torch.randn(3, 4)

    def loop(latents):
        cache = _Cache()
        for i in range(4):
            t = torch.tensor(1.0 - i / 4)
            outs = [
                dit(hidden_states = latents, timestep = t, encoder_hidden_states = e, kv_cache = cache)[0]
                for e in (cond, uncond)
            ]
            latents = latents - 0.1 * (outs[1] + 4.0 * (outs[0] - outs[1]))
        return latents, cache

    latents = torch.randn(2, 4)
    reference, _ = loop(latents.clone())
    router, replicas = _router(dit)
    out, cache = loop(latents.clone())
    assert torch.equal(out, reference)
    # the mutable cache is never handed to the replica thread, and no replica is paid for
    assert set(cache.writers) == {threading.current_thread()}
    assert replicas == [] and router.stats["replica_calls"] == 0
    assert router.stats["opaque_primary_calls"] == 4
    assert router.describe()["disabled_reason"] is None
    assert not mg._holds_opaque_tensors(((torch.ones(1),), {"a": [torch.ones(1)], "b": None}))
    assert mg._holds_opaque_tensors({"kv_cache": _Cache()})


class _KVLayer:
    """Qwen-Image-2.1's ``QwenImage21KVLayerCache``: one layer's prefix K, filled by the prefill."""

    def __init__(self):
        self.k = None


class _KVCache:
    """Qwen-Image-2.1's ``QwenImage21KVCache``: the per-layer holders sit in a list inside the cache object."""

    def __init__(self, layers):
        self.layer_caches = [_KVLayer() for _ in range(layers)]


class _PrefixKVDit(_Dit):
    """Step 0 stores the prefix K (``extract``); later steps concatenate it with the step's own (``cached``)."""

    # (thread, mode, cache) per layer read; on the class, so the replica's copy logs here too
    seen: list = []

    def forward(
        self,
        hidden_states,
        timestep,
        encoder_hidden_states,
        kv_cache = None,
        kv_cache_mode = None,
        return_dict = False,
    ):
        out = self.proj(hidden_states) * timestep
        prefix = self.proj(encoder_hidden_states)
        for layer in kv_cache.layer_caches:
            self.seen.append((threading.current_thread(), kv_cache_mode, kv_cache))
            if kv_cache_mode == "extract":
                layer.k = prefix.clone()
            else:
                out = out + torch.cat([layer.k, out], dim = 0)[: out.shape[0]].sum(0)
        return (out,)


def _kv_render(
    dit,
    latents,
    steps = 5,
):
    cond, neg = (
        torch.randn(3, 4, generator = torch.Generator().manual_seed(1)),
        torch.randn(3, 4, generator = torch.Generator().manual_seed(2)),
    )
    caches = (_KVCache(2), _KVCache(2))
    for i in range(steps):
        t = torch.tensor(1.0 - i / steps)
        mode = "extract" if i == 0 else "cached"
        a, b = (
            dit(
                hidden_states = latents,
                timestep = t,
                encoder_hidden_states = e,
                kv_cache = c,
                kv_cache_mode = mode,
            )[0]
            for e, c in zip((cond, neg), caches)
        )
        latents = latents - 0.1 * (b + 4.0 * (a - b))
    return latents, caches


def test_prefix_kv_cache_branch_runs_on_the_replica_against_its_own_copy():
    torch.manual_seed(0)
    dit = _PrefixKVDit()
    latents = torch.randn(2, 4)
    references = [_kv_render(dit, latents.clone())[0] for _ in range(2)]
    router, replicas = _router(dit)
    main = threading.current_thread()
    for render in range(2):
        _PrefixKVDit.seen.clear()
        out, (cond_cache, neg_cache) = _kv_render(dit, latents.clone())
        assert torch.equal(out, references[render])
        assert all(th is main for th, mode, _c in _PrefixKVDit.seen if mode == "extract")
        assert neg_cache.layer_caches[0].k is not None
        # the replica reads its own copy, never the pipeline's (mutable) object
        replica_caches = [c for th, mode, c in _PrefixKVDit.seen if th is not main]
        assert replica_caches and all(
            c is not neg_cache and c is not cond_cache for c in replica_caches
        )
        router.end_render()
    assert router.describe()["disabled_reason"] is None and router.describe()["armed"]
    assert router.stats["steps_parallel"] >= 6
    assert router.stats["kv_cache_copies"] == 2
    assert router.stats["kv_prefill_primary_calls"] == 2
    assert router.kv_mirrors == {}  # dropped at end_render


def test_prefix_kv_cache_too_large_for_the_replica_card_stays_on_the_primary(monkeypatch):
    torch.manual_seed(0)
    dit = _PrefixKVDit()
    latents = torch.randn(2, 4)
    reference = _kv_render(dit, latents.clone())[0]
    # the replica (no weights to speak of) passes admission with exactly 1 GiB free; the cache on top of it does not
    monkeypatch.setattr(mg, "_free_total_mib", lambda ordinal: (1024, 2048))
    router, replicas = _router(dit)
    copies = []
    real_walk = mg._walk_cache

    def walk(
        obj,
        move,
        found,
        depth = 0,
    ):
        if move is not None and depth == 0:
            copies.append(obj)
        return real_walk(obj, move, found, depth)

    monkeypatch.setattr(mg, "_walk_cache", walk)
    main = threading.current_thread()
    _PrefixKVDit.seen.clear()
    out, _caches = _kv_render(dit, latents.clone())
    assert torch.equal(out, reference)
    assert copies == [] and router.stats["kv_cache_copies"] == 0
    assert router.stats["kv_cache_declined"] == 1  # probed once per cache, not per step
    assert all(th is main for th, _mode, _c in _PrefixKVDit.seen)
    # declined for this render only: the replica stays built and armed
    assert replicas and router.replica is not None
    assert router.describe()["disabled_reason"] is None
    router.end_render()
    assert router.kv_declined == {}


def test_opaque_detection_sees_tensors_in_a_list_of_value_holders():
    cache = _KVCache(2)
    assert not mg._holds_opaque_tensors({"kv_cache": cache})  # empty before the prefill
    cache.layer_caches[1].k = torch.ones(2)
    assert mg._holds_opaque_tensors({"kv_cache": cache})
    # a filled cache in cached mode is copied for the replica, the rest of the call is checked as before
    kwargs = {"kv_cache": cache, "kv_cache_mode": "cached", "x": torch.ones(1)}
    assert mg._prefix_kv_mode(kwargs) == "cached"
    assert not mg._holds_opaque_tensors(mg._without_prefix_kv((), kwargs))
    assert mg._prefix_kv_mode({**kwargs, "kv_cache_mode": "extract"}) == "extract"
    assert mg._prefix_kv_mode({**kwargs, "kv_cache_mode": None}) is None
    # a prefill stays on the primary even when its cache could not be copied, or is returned (FLUX.2)
    assert mg._is_prefill({"kv_cache": object(), "kv_cache_mode": "extract"})
    assert mg._is_prefill({"kv_cache_mode": "extract"}) and not mg._is_prefill(kwargs)
    longer = _KVCache(2)
    longer.layer_caches[1].k = torch.ones(3)
    assert mg._call_signature((), kwargs) != mg._call_signature((), {**kwargs, "kv_cache": longer})


def test_replica_failure_is_reported_in_the_status():
    torch.manual_seed(0)
    dit = _Dit()

    def broken():
        raise RuntimeError("no card")

    router = mg.CfgParallelRouter(dit, primary = "cpu", secondary = "cpu", replica_factory = broken)
    dit.forward = router
    _cfg_loop(dit, torch.randn(2, 4), steps = 3)
    info = router.describe()
    assert info["armed"] is False and info["replica_failures"] == 1
    assert info["disabled_reason"].startswith("replica unavailable")


def test_split_video_request_is_guarded_on_every_card():
    from core.inference import video as vid

    class Pipe:
        def __init__(self):
            self.components = {
                "transformer": torch.nn.Linear(1024, 1024),
                "text_encoder": torch.nn.Linear(1024, 512),
                "vae": torch.nn.Linear(512, 512),
            }

    pipe = Pipe()
    mg._PLACEMENTS[pipe] = {
        "placement": {"transformer": "cuda:0", "text_encoder": "cuda:1", "vae": "cuda:1"},
        "moved": ["text_encoder", "vae"],
    }
    cards = vid._video_split_card_floors(pipe)
    assert cards == ((0, 4, False, True, True), (1, 3, True, False, False))
    big = dict(width = 1280, height = 720, frames = 121)
    clip = vid.estimate_video_runtime_mib(width = 1280, height = 720, num_frames = 121)
    roomy = {0: 1 << 20, 1: 1 << 20}
    assert (
        vid.video_split_shortfall_message(cards, family = "f", available_mib = roomy.get, **big) is None
    )
    # Decode card: the decoded clip; denoiser card: weights, base headroom, frames copied back.
    frames_mib = -(-1280 * 720 * 121 * 3 * 4 // (1 << 20))
    home_mib = max(frames_mib, vid._VIDEO_DENOISE_ACTIVATION_MIB)
    tight = {0: 4 + home_mib + dm.DEFAULT_BASE_OVERHEAD_MIB, 1: 3 + dm.DEFAULT_BASE_OVERHEAD_MIB}
    message = vid.video_split_shortfall_message(cards, family = "f", available_mib = tight.get, **big)
    assert clip > 0 and message is not None and "cuda:1" in message
    tight[1] = 1 << 20
    assert (
        vid.video_split_shortfall_message(cards, family = "f", available_mib = tight.get, **big) is None
    )
    tight[0] -= 1
    message = vid.video_split_shortfall_message(cards, family = "f", available_mib = tight.get, **big)
    assert "cuda:0" in message
    tight[0] = 4 + dm.DEFAULT_BASE_OVERHEAD_MIB
    assert vid.video_split_shortfall_message(cards, family = "f", available_mib = tight.get) is None
    del mg._PLACEMENTS[pipe]
    assert vid._video_split_card_floors(pipe) is None


def test_primary_tile_verdict_does_not_tile_a_vae_decoding_elsewhere(monkeypatch):
    import types as _types

    from core.inference import diffusion as D

    engaged = []
    monkeypatch.setattr(D, "diffusion_device_scope", lambda o: contextlib.nullcontext())
    monkeypatch.setattr(D, "reclaimable_snapshot_device_memory", lambda target: "card1-free")
    # the VAE's card decodes untiled; only the smaller primary asks for tiling
    monkeypatch.setattr(
        D,
        "raise_on_image_activation_shortfall",
        lambda **kw: _types.SimpleNamespace(action = "ok", overridden = False),
    )
    monkeypatch.setattr(
        D, "engage_vae_tiling", lambda pipe, logger = None: engaged.append(1) or ("restore", True)
    )
    pipe = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1))
    target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)
    monkeypatch.setattr(mg, "is_split", lambda p: True)
    primary_verdict = _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)
    kwargs = {"device_memory": "card0-free", "vae_tile_side": 512, "vae_sliced": False}
    monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))
    assert D._tile_for_verdict(pipe, primary_verdict, target, kwargs, 1, None, None) is None
    assert engaged == []
    monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))
    assert D._tile_for_verdict(pipe, primary_verdict, target, kwargs, 1, None, None) == "restore"
    assert engaged == [1]


def test_secondary_free_memory_reads_the_trusted_probe(monkeypatch):
    import utils.hardware as hw

    seen = []
    monkeypatch.setattr(torch.cuda, "mem_get_info", lambda *a: (80 << 30, 80 << 30))
    monkeypatch.setattr(
        hw,
        "trusted_mem_get_info",
        lambda device = None, module = None: seen.append(device) or (1 << 30, 80 << 30),
    )
    assert mg._free_total_mib(1) == (1024, 80 * 1024)
    assert seen == [1]


_GLOBAL_HOOK_CALLS = []


def _global_state_hook(module, inputs, output):
    _GLOBAL_HOOK_CALLS.append(1)
    return output * (1.0 + len(_GLOBAL_HOOK_CALLS) % 2)


def test_foreign_submodule_hook_declines_the_replica():
    dit = _Dit()
    dit.proj.register_forward_hook(_global_state_hook)
    assert _global_state_hook.__closure__ is None
    assert mg._hooks_present(dit) == "proj._forward_hooks"
    router, replicas = _router(dit)
    _cfg_loop(dit, torch.randn(2, 4), steps = 3)
    assert replicas == [] and router.stats["replica_calls"] == 0
    _global_state_hook.__module__, saved = mg.__name__, _global_state_hook.__module__
    try:
        assert mg._hooks_present(_Dit()) is None
        hooked = _Dit()
        hooked.proj.register_forward_hook(_global_state_hook)
        assert mg._hooks_present(hooked) is None
    finally:
        _global_state_hook.__module__ = saved


def test_force_mode_refines_a_resident_plan(monkeypatch):
    plan = dataclasses.replace(_plan(), offload_policy = dm.OFFLOAD_NONE)
    sizes = {
        "transformer": (9000, "dit"),
        "text_encoder": (8000, "text_encoder"),
        "vae": (300, "other"),
    }
    monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)
    monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None, **_: 1)
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        assert mg.refine_plan_for_multigpu(object(), plan) is plan
        monkeypatch.setenv(mg.MULTIGPU_ENV, "force")
        refined = mg.refine_plan_for_multigpu(object(), plan)
    assert refined.secondary_ordinal == 1 and refined.offload_policy == dm.OFFLOAD_NONE
    assert refined.secondary_components == ("text_encoder", "vae")
    assert refined.fallback_plan is plan


def _ltx25_split(single, **estimates):
    return dataclasses.replace(
        single,
        offload_policy = dm.OFFLOAD_NONE,
        secondary_ordinal = 1,
        fallback_plan = single,
        estimates = {**single.estimates, **estimates},
    )


def test_ltx25_sidecar_encoder_keeps_one_card_without_the_lever(monkeypatch):
    import types as _types

    from core.inference import video as vid

    single = _plan()
    split = _ltx25_split(single, multigpu_secondary_required_mib = 9000)
    ltx25 = _types.SimpleNamespace(name = "ltx-2.5")
    wan = _types.SimpleNamespace(name = "wan2.2")
    refined = []
    monkeypatch.setattr(
        vid,
        "refine_plan_for_multigpu",
        lambda pipe, plan, logger = None, **kw: refined.append(kw) or split,
    )
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)
    # no selection: one card, whatever the lever says
    assert vid._video_single_card_for_sidecar(ltx25, split) is single
    assert vid._video_single_card_for_sidecar(wan, split) is split
    assert vid._video_single_card_for_sidecar(ltx25, single) is single
    assert vid._video_refine_for_multigpu(ltx25, object(), single) is single
    assert vid._video_refine_for_multigpu(wan, object(), single) is split and refined == [{}]
    monkeypatch.setenv(vid.LTX25_MULTI_GPU_ENV, "0")
    refined.clear()
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        assert vid._video_single_card_for_sidecar(ltx25, split) is single
        assert vid._video_sidecar_worker_card(ltx25, split) is None
        assert vid._video_refine_for_multigpu(ltx25, object(), single) is single
    assert refined == []


def test_ltx25_split_keeps_the_worker_on_the_partner_when_it_fits(monkeypatch):
    import types as _types

    from core.inference import video as vid

    single = _plan()
    ltx25 = _types.SimpleNamespace(name = "ltx-2.5")
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)
    fits = _ltx25_split(single, multigpu_secondary_required_mib = 13600 - 1024)
    short = _ltx25_split(single, multigpu_secondary_required_mib = 13600 - 1023)
    unpriced = _ltx25_split(single)
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        assert vid._video_single_card_for_sidecar(ltx25, fits) is fits
        assert vid._video_sidecar_worker_card(ltx25, fits) == 1
        assert vid._video_single_card_for_sidecar(ltx25, short) is single
        assert vid._video_single_card_for_sidecar(ltx25, unpriced) is single
        assert vid._video_sidecar_worker_card(ltx25, single) is None
        monkeypatch.setattr(mg, "_rocm_torch", lambda: True)
        assert vid._video_single_card_for_sidecar(ltx25, fits) is single
        assert vid._video_sidecar_worker_card(ltx25, fits) is None


class _FakeEncoderClient:
    def __init__(self, device, info):
        self.config = {"device": device}
        self._info = info
        self.waits = 0

    def wait_ready(self, cancel = None):
        self.waits += 1
        if isinstance(self._info, Exception):
            raise self._info
        return self._info


def test_ltx25_worker_reserve_prices_its_card():
    from core.inference import video as vid

    def reserve(
        device,
        info,
        plan = None,
    ):
        pipe = _types_ns(_unsloth_ltx25_encoder = _FakeEncoderClient(device, info))
        return vid._video_sidecar_card_reserve_mib(pipe, plan)

    assert reserve("cuda:0", {"resident": True, "weights_mib": 13000}) == {0: 13000 + 1024}
    assert reserve(
        "cuda:1", {"resident": False, "weights_mib": 13000, "stream_need_mib": 6000}
    ) == {1: 6000 + 1024}
    assert reserve("cuda:0", {"resident": False, "weights_mib": 13000}) == {0: 13000 + 4096 + 1024}
    # on the primary a non-resident encode reuses the denoise headroom; resident weights and the partner never do
    plan = _types_ns(estimates = {"runtime_headroom_mib": 5000})
    streamed = {"resident": False, "weights_mib": 13000, "stream_need_mib": 6000}
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        assert reserve("cuda:0", streamed, plan) == {0: 1000 + 1024}
        assert reserve(
            "cuda:0", {"resident": False, "weights_mib": 16, "stream_need_mib": 4112}, plan
        ) == {0: 1024}
        assert reserve("cuda:0", {"resident": True, "weights_mib": 13000}, plan) == {
            0: 13000 + 1024
        }
        assert reserve("cuda:1", streamed, plan) == {1: 6000 + 1024}
    assert reserve("cuda:0", streamed, plan) == {0: 6000 + 1024}
    assert reserve("cpu", {"resident": True, "weights_mib": 13000}) is None
    assert reserve("cuda:0", {"resident": True}) is None
    with pytest.raises(RuntimeError, match = "worker died"):
        reserve("cuda:0", RuntimeError("worker died"))
    assert vid._video_sidecar_card_reserve_mib(_types_ns()) is None


def _types_ns(**kw):
    import types as _types
    return _types.SimpleNamespace(**kw)


def test_ltx25_refinement_is_charged_for_the_worker(monkeypatch):
    """A companion split that fits without the worker's charge must not fit with it."""
    from core.inference import diffusion_layer_split_plan as lsp
    from core.inference import video as vid

    plan = _plan()
    assert plan.offload_policy != dm.OFFLOAD_NONE
    sizes = {"transformer": (9000, "dit"), "connectors": (3000, "other"), "vae": (300, "other")}
    monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)
    monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None, **_: 1)
    monkeypatch.setattr(mg, "encoder_card_limit_mib", lambda o: 12750)
    limits = []

    def layer_split(pipe, plan, *, primary_limit_mib, secondary_limit_mib, **kw):
        limits.append((primary_limit_mib, secondary_limit_mib(1)))
        return plan

    monkeypatch.setattr(lsp, "refine_plan_for_layer_split", layer_split)
    ltx25 = _types_ns(name = "ltx-2.5")
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        free = mg.refine_plan_for_multigpu(object(), plan)
        assert free.secondary_ordinal == 1 and free.offload_policy == dm.OFFLOAD_NONE
        budget = int(plan.estimates["safe_device_budget_mib"])
        unreserved = (
            budget
            if dm.plan_keeps_transformer_resident(plan)
            else int(budget * mg._RESIDENT_FRACTION)
        )
        worker = _FakeEncoderClient("cuda:0", {"resident": True, "weights_mib": unreserved})
        charged = vid._video_refine_for_multigpu(
            ltx25, _types_ns(_unsloth_ltx25_encoder = worker), plan
        )
        assert charged is plan and worker.waits == 1
        assert limits[-1] == (unreserved - unreserved - 1024, 12750)
        partner = _FakeEncoderClient("cuda:1", {"resident": True, "weights_mib": 12000})
        assert (
            vid._video_refine_for_multigpu(ltx25, _types_ns(_unsloth_ltx25_encoder = partner), plan)
            is plan
        )
        assert limits[-1] == (unreserved, 12750 - 12000 - 1024)
        small = _FakeEncoderClient("cuda:1", {"resident": True, "weights_mib": 100})
        split = vid._video_refine_for_multigpu(ltx25, _types_ns(_unsloth_ltx25_encoder = small), plan)
        assert split.secondary_ordinal == 1
        unpriced = _FakeEncoderClient("cuda:0", {"resident": True})
        assert (
            vid._video_refine_for_multigpu(ltx25, _types_ns(_unsloth_ltx25_encoder = unpriced), plan)
            is plan
        )
        resident = dataclasses.replace(plan, offload_policy = dm.OFFLOAD_NONE)
        idle = _FakeEncoderClient("cuda:0", {"resident": True, "weights_mib": 1})
        kept = vid._video_refine_for_multigpu(
            ltx25, _types_ns(_unsloth_ltx25_encoder = idle), resident
        )
        assert kept is resident and idle.waits == 0


def test_refinement_without_a_reserve_is_unchanged(monkeypatch):
    plan = _plan()
    sizes = {
        "transformer": (9000, "dit"),
        "text_encoder": (8000, "text_encoder"),
        "vae": (300, "other"),
    }
    monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)
    monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None, **_: 1)
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        a = mg.refine_plan_for_multigpu(object(), plan)
        b = mg.refine_plan_for_multigpu(object(), plan, card_reserve_mib = {})
        c = mg.refine_plan_for_multigpu(object(), plan, card_reserve_mib = {1: 0, 5: 99999})
    assert a == b == c and a.secondary_components == ("text_encoder", "vae")


def _fake_cuda_host(
    monkeypatch,
    *,
    count,
    ranking_possible = True,
    current = 0,
):
    from core.inference import diffusion_device as dd

    monkeypatch.setattr(dd, "auto_cuda_ranking_possible", lambda: ranking_possible)
    monkeypatch.setattr(
        dd, "resolve_diffusion_device_target", lambda **kw: _Target(ordinal = kw.get("ordinal"))
    )
    monkeypatch.setattr(torch.cuda, "device_count", lambda: count)
    monkeypatch.setattr(torch.cuda, "current_device", lambda: current)
    monkeypatch.setattr(mg, "_capability", lambda o: (9, 0))
    from core.inference import gpu_residency

    monkeypatch.setattr(gpu_residency, "current_policy", lambda: "off")


def test_automatic_selection_uses_every_visible_card(monkeypatch):
    _fake_cuda_host(monkeypatch, count = 3)
    assert mg.automatic_selection(None) == mg.MultiGpuSelection(
        primary = 0, secondaries = (1, 2), automatic = True
    )
    assert mg.automatic_selection(2) == mg.MultiGpuSelection(
        primary = 2, secondaries = (0, 1), automatic = True
    )
    monkeypatch.setenv(mg.MULTIGPU_ENV, "0")
    assert mg.automatic_selection(None) is None


def test_automatic_selection_is_inert_on_ordinary_hosts(monkeypatch):
    # one GPU, ROCm, XPU, MPS and CPU all fail the ranking gate; a gated host never touches torch.cuda
    _fake_cuda_host(monkeypatch, count = 2, ranking_possible = False)
    monkeypatch.setattr(torch.cuda, "device_count", lambda: pytest.fail("probed a gated host"))
    assert mg.automatic_selection(None) is None
    _fake_cuda_host(monkeypatch, count = 1)
    assert mg.automatic_selection(None) is None


def test_with_load_selection_resolves_the_automatic_marker(monkeypatch):
    _fake_cuda_host(monkeypatch, count = 2)
    seen = {}

    class Backend:
        @mg.with_load_selection
        def _run_load(self, **kwargs):
            seen["kwargs"] = dict(kwargs)
            seen["selection"] = mg.current_selection()

    Backend()._run_load(gpu_ordinal = None, _gpu_ordinals = mg.AUTOMATIC_ORDINALS, repo_id = "x")
    assert seen["kwargs"] == {"gpu_ordinal": None, "repo_id": "x"}
    assert seen["selection"] == mg.MultiGpuSelection(primary = 0, secondaries = (1,), automatic = True)


def test_automatic_load_splits_only_when_one_card_would_offload(monkeypatch):
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)
    monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
    selection = mg.MultiGpuSelection(primary = 0, secondaries = (1,), automatic = True)
    target = _Target(ordinal = None)
    single = _plan(target = target)
    with mg.selection_scope(selection):
        plan = _plan(target = target)
        fits = _plan(
            target = target,
            model_dense_mib = 4000,
            companion_dense_mib = 2000,
            text_encoder_dense_mib = 1500,
        )
    assert single.offload_policy != dm.OFFLOAD_NONE and single.secondary_ordinal is None
    assert plan.offload_policy == dm.OFFLOAD_NONE and plan.secondary_ordinal == 1
    assert fits.secondary_ordinal is None


def test_automatic_placement_arms_cfg_parallel_only_with_the_auto_lever(monkeypatch):
    class Pipe:
        transformer = _Dit()

    automatic = mg.MultiGpuSelection(primary = 0, secondaries = (1,), automatic = True)
    # default: past the automatic gate (install_cfg_parallel then keeps only idle cards of the same model)
    assert "CUDA" in mg.cfg_parallel_decline(Pipe(), offload_policy = "none", selection = automatic)
    monkeypatch.setenv(mg.AUTO_CFG_PARALLEL_ENV, "0")
    assert "automatic" in mg.cfg_parallel_decline(
        Pipe(), offload_policy = "none", selection = automatic
    )
    monkeypatch.setenv(mg.CFG_PARALLEL_ENV, "force")
    assert "CUDA" in mg.cfg_parallel_decline(Pipe(), offload_policy = "none", selection = automatic)


@pytest.mark.parametrize("module_file", ["diffusion.py", "video.py"])
def test_begin_load_marks_an_unpicked_load_automatic(module_file):
    import ast
    from pathlib import Path

    path = Path(__file__).resolve().parents[1] / "core" / "inference" / module_file
    tree = ast.parse(path.read_text(encoding = "utf-8"))
    begin_loads = [
        node
        for node in ast.walk(tree)
        if isinstance(node, ast.FunctionDef) and node.name == "begin_load"
    ]
    assert len(begin_loads) == 1
    assigned = [
        node
        for node in ast.walk(begin_loads[0])
        if isinstance(node, ast.Assign)
        and any(getattr(t, "id", None) == "gpu_ordinals" for t in node.targets)
    ]
    assert len(assigned) == 1
    value = assigned[0].value
    assert isinstance(value, ast.IfExp) and isinstance(value.orelse, ast.IfExp)
    assert ast.unparse(value.orelse.body) == "AUTOMATIC_ORDINALS"
    assert ast.unparse(value.orelse.test) == "not gpu_ids"
    assert ast.unparse(value.orelse.orelse) == "[]"


def test_automatic_selection_leaves_module_compile_alone_without_the_auto_lever(monkeypatch):
    def original(self, *args, **kwargs):
        return self

    monkeypatch.setattr(torch.nn.Module, "compile", original)
    monkeypatch.setenv(mg.AUTO_CFG_PARALLEL_ENV, "0")
    automatic = mg.MultiGpuSelection(primary = 0, secondaries = (1,), automatic = True)
    with mg.selection_scope(automatic):
        assert torch.nn.Module.compile is original
    monkeypatch.delenv(mg.AUTO_CFG_PARALLEL_ENV)
    with mg.selection_scope(automatic):
        # Automatic may build a CFG replica, which re-applies the recorded compile arguments
        assert getattr(torch.nn.Module.compile, "_unsloth_recorder", False)
    monkeypatch.setattr(torch.nn.Module, "compile", original)
    with mg.selection_scope(mg.MultiGpuSelection(primary = 0, secondaries = (1,))):
        assert getattr(torch.nn.Module.compile, "_unsloth_recorder", False)


def test_automatic_secondaries_are_sized_without_a_cuda_context(monkeypatch):
    from core.inference import diffusion_device as dd

    monkeypatch.setattr(
        torch.cuda, "mem_get_info", lambda *a: pytest.fail("opened a context on a secondary")
    )
    monkeypatch.setattr(
        mg, "probe_devices", lambda *a, **k: pytest.fail("probed every card through torch")
    )
    readings = {1: (9000, 15360), 2: (14000, 15360)}
    monkeypatch.setattr(dd, "smi_free_total_mib", lambda o: readings.get(o, (None, None)))
    automatic = mg.MultiGpuSelection(primary = 0, secondaries = (1, 2), automatic = True)
    with mg.selection_scope(automatic):
        assert mg.best_secondary(automatic) == 2
        memory = mg.secondary_device_memory(2)
        assert (memory.free_mib, memory.total_mib) == (14000, 15360)
        assert mg.secondary_device_memory(3) is None


def test_smi_reading_keys_ordinals_through_the_mask(monkeypatch):
    import utils.hardware.hardware as hw
    from core.inference import diffusion_device as dd

    monkeypatch.setattr(hw, "IS_ROCM", False)
    monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: True)
    monkeypatch.setattr(hw, "get_parent_visible_gpu_ids", lambda: [4, 5])
    monkeypatch.setattr(hw, "_integrated_cuda_rows", lambda visible: ({}, ""))
    monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())
    monkeypatch.setattr(hw, "_get_parent_visible_gpu_spec", lambda: {"numeric_ids": [4, 5]})
    rows = [
        {"index": 4, "vram_total_gb": 24.0, "vram_used_gb": 20.0},
        {"index": 5, "vram_total_gb": 24.0, "vram_used_gb": 1.0},
    ]
    monkeypatch.setattr(hw, "_smi_visible_utilization", lambda spec: {"devices": rows})
    assert dd.smi_free_total_mib(1) == (23 * 1024, 24 * 1024)
    assert dd.smi_free_total_mib(2) == (None, None)
    monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: {5})
    assert dd.smi_free_total_mib(1) == (None, None)
    monkeypatch.setattr(hw, "_nvidia_mig_gpu_indices_or_none", lambda: set())
    monkeypatch.setattr(hw, "_cuda_order_matches_smi", lambda: False)
    assert dd.smi_free_total_mib(1) == (None, None)


def test_unsplit_automatic_load_has_no_other_vae_card(monkeypatch):
    import types as _types

    from core.inference import diffusion as D

    monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))
    monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
    pipe = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1))
    automatic = _types.SimpleNamespace(ordinal = None, is_cuda_torch_device = True)
    assert D._vae_on_other_card(pipe, automatic) is None
    monkeypatch.setattr(mg, "is_split", lambda p: True)
    assert D._vae_on_other_card(pipe, automatic) is None
    monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))
    assert D._vae_on_other_card(pipe, automatic) == torch.device("cuda", 1)


def test_secondary_vae_guard_refuses_a_batch_slicing_did_not_engage(monkeypatch):
    import types as _types

    from core.inference import diffusion as D

    calls = []

    def shortfall(**kw):
        calls.append(kw.get("vae_sliced"))
        if kw.get("vae_sliced") is False:
            raise ValueError("does not fit unsliced")
        return _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)

    restored = []
    monkeypatch.setattr(D, "diffusion_device_scope", lambda o: contextlib.nullcontext())
    monkeypatch.setattr(D, "reclaimable_snapshot_device_memory", lambda target: "card1-free")
    monkeypatch.setattr(D, "raise_on_image_activation_shortfall", shortfall)
    monkeypatch.setattr(
        D, "engage_vae_tiling", lambda pipe, logger = None: (lambda: restored.append(1), True)
    )
    monkeypatch.setattr(D, "vae_is_sliced", lambda vae: False)
    monkeypatch.setattr(mg, "is_split", lambda p: True)
    monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))
    pipe = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1))
    target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)
    with pytest.raises(ValueError):
        D._guard_vae_card(pipe, target, {"batch_size": 2, "vae_sliced": True}, None, None)
    assert restored == [1] and calls == [True, False]


@pytest.mark.parametrize("vae_card", [0, 1])
def test_a_broken_recheck_keeps_the_tiling_it_engaged(monkeypatch, vae_card):
    """A re-check that raises something other than the refusal fails open, as generate() always did: the decode
    stays tiled (the guard said only a tiled decode fits) and the handle comes back for the caller to restore."""
    import types as _types

    from core.inference import diffusion as D

    def shortfall(**kw):
        if kw.get("vae_sliced") is False:
            raise RuntimeError("probe broke")
        return _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)

    restored = []
    restore = lambda: restored.append(1)  # noqa: E731
    monkeypatch.setattr(D, "diffusion_device_scope", lambda o: contextlib.nullcontext())
    monkeypatch.setattr(D, "reclaimable_snapshot_device_memory", lambda target: "free")
    monkeypatch.setattr(D, "raise_on_image_activation_shortfall", shortfall)
    monkeypatch.setattr(D, "engage_vae_tiling", lambda pipe, logger = None: (restore, True))
    monkeypatch.setattr(D, "vae_is_sliced", lambda vae: False)
    monkeypatch.setattr(mg, "is_split", lambda p: True)
    monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", vae_card))
    pipe = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1))
    target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)
    kwargs = {"batch_size": 2, "vae_sliced": True, "vae_tile_side": 512, "device_memory": "free"}
    verdict = _types.SimpleNamespace(action = D.ACTIVATION_TILE, overridden = False)
    assert D._tile_for_verdict(pipe, verdict, target, kwargs, 2, None, None) is restore
    assert restored == []


def test_no_split_ever_placed_skips_the_component_walk(monkeypatch):
    reads = []

    class Pipe:
        @property
        def components(self):
            reads.append(1)
            return {}

    monkeypatch.setattr(mg, "_SPLIT_EVER_PLACED", False)
    assert mg._split_home(Pipe()) is None and reads == []
    monkeypatch.setattr(mg, "_SPLIT_EVER_PLACED", True)
    assert mg._split_home(Pipe()) is None and reads == [1]


def test_a_card_torch_cannot_address_is_not_a_secondary(monkeypatch):
    """nvidia-smi can list two cards while this process's torch addresses one: no second card, no selection."""
    monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
    monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
    monkeypatch.setattr(mg, "_capability", lambda o: (9, 0))
    assert mg.make_selection(0, [0, 1]) is None
    monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
    assert mg.make_selection(0, [0, 1]) == mg.MultiGpuSelection(primary = 0, secondaries = (1,))


def test_two_discrete_rocm_cards_split_like_cuda(monkeypatch):
    """2x 7900 XTX / MI300: a HIP torch with two addressable discrete cards plans the same split CUDA does."""
    monkeypatch.setattr(torch.version, "hip", "7.13.0", raising = False)
    monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
    monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
    monkeypatch.setattr(mg, "_capability", lambda o: (11, 0))  # gfx1100, as ROCm reports it
    monkeypatch.setattr(mg, "_rocm_torch", lambda: True)
    monkeypatch.setattr(mg, "_rocm_bf16", lambda o: True)
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 13600)
    selection = mg.make_selection(0, [0, 1])
    assert selection == mg.MultiGpuSelection(primary = 0, secondaries = (1,))
    with mg.selection_scope(selection):
        plan = _plan()
    assert plan.offload_policy == dm.OFFLOAD_NONE and plan.secondary_ordinal == 1


def test_refinement_keeps_an_explicit_cpu_offload(monkeypatch):
    sizes = {
        "transformer": (9000, "dit"),
        "text_encoder": (8000, "text_encoder"),
        "vae": (300, "other"),
    }
    monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: sizes)
    monkeypatch.setattr(mg, "secondary_resident_limit_mib", lambda o, mode: 12750)
    monkeypatch.setattr(mg, "best_secondary", lambda selection, probes = None, **_: 1)
    legacy = _plan(explicit_offload = True)
    assert legacy.estimates.get("explicit_cpu_offload")
    moded = _plan(explicit_offload = True, requested_mode = "auto")
    assert not moded.estimates.get("explicit_cpu_offload")
    with mg.selection_scope(mg.make_selection(0, [0, 1])):
        assert mg.refine_plan_for_multigpu(object(), legacy) is legacy
        assert mg.refine_plan_for_multigpu(object(), moded).secondary_ordinal == 1


def test_render_end_releases_the_branch_tensors():
    torch.manual_seed(0)
    dit = _Dit()
    latents = torch.randn(2, 4)
    router, replicas = _router(dit)

    class Pipe:
        transformer = dit

    pipe = Pipe()
    mg._CFG_ROUTERS[pipe] = router
    _cfg_loop(dit, latents.clone(), steps = 3)
    assert router.template is not None and router.prev_latent is not None
    mg.end_cfg_render(pipe)
    assert router.template is None and router.prev_latent is None and router.pending is None
    assert router.cfg_seen is False
    assert router.prefetched == {} and router.carry == [] and router.b0_leaves == {}
    assert router.replica is not None
    reference_router = dit.__dict__.pop("forward")
    reference = _cfg_loop(dit, latents.clone(), steps = 3)
    dit.forward = reference_router
    assert torch.equal(_cfg_loop(dit, latents.clone(), steps = 3), reference)
    assert len(replicas) == 1
    mg.end_cfg_render(object())


def test_workflow_pipeline_of_a_split_still_guards_the_vae_card(monkeypatch):
    monkeypatch.setattr(mg, "_SPLIT_EVER_PLACED", True)
    import types as _types

    from core.inference import diffusion as D

    dit = torch.nn.Linear(1, 1)
    dit._unsloth_split_home = torch.device("cuda", 0)
    sibling = _types.SimpleNamespace(vae = torch.nn.Linear(1, 1), components = {"transformer": dit})
    target = _types.SimpleNamespace(ordinal = 0, is_cuda_torch_device = True)
    monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 1))
    assert not mg.is_split(sibling)
    assert D._vae_on_other_card(sibling, target) == torch.device("cuda", 1)


def test_cfg_replica_avoids_the_card_a_split_decodes_on(monkeypatch):
    class Pipe:
        transformer = _Dit()

    probes = {
        o: mg.DeviceProbe(ordinal = o, name = "B200", total_mib = 180000, free_mib = 170000)
        for o in (0, 1, 2)
    }
    monkeypatch.setattr(mg, "cfg_parallel_decline", lambda pipe, **kw: None)
    monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))
    monkeypatch.setattr(
        mg, "probe_devices", lambda ordinals, **kw: {o: probes[o] for o in ordinals}
    )
    pipe = Pipe()
    mg._PLACEMENTS[pipe] = {"placement": {"transformer": "cuda:0", "vae": "cuda:1"}}
    try:
        with mg.selection_scope(mg.MultiGpuSelection(primary = 0, secondaries = (1,))):
            reason = mg.install_cfg_parallel(pipe, offload_policy = "none")
        assert reason is not None and "holds the split's components" in reason
        te_only = Pipe()
        mg._PLACEMENTS[te_only] = {
            "placement": {"transformer": "cuda:0", "vae": "cuda:0", "text_encoder": "cuda:1"}
        }
        with mg.selection_scope(mg.MultiGpuSelection(primary = 0, secondaries = (1,))):
            assert "holds the split's components" in mg.install_cfg_parallel(
                te_only, offload_policy = "none"
            )
        other = Pipe()
        mg._PLACEMENTS[other] = {"placement": {"transformer": "cuda:0", "vae": "cuda:1"}}
        with mg.selection_scope(mg.MultiGpuSelection(primary = 0, secondaries = (1, 2))):
            assert mg.install_cfg_parallel(other, offload_policy = "none") is None
        assert str(mg._CFG_ROUTERS[other].secondary) == "cuda:2"
    finally:
        mg.uninstall_cfg_parallel(pipe)
        mg.uninstall_cfg_parallel(other)


def test_background_compile_declines_cfg_parallel(monkeypatch):
    class Pipe:
        transformer = _Dit()

    sel = mg.MultiGpuSelection(primary = 0, secondaries = (1,))
    monkeypatch.setenv("UNSLOTH_DIFFUSION_BG_COMPILE", "1")
    assert "background compile" in mg.cfg_parallel_decline(
        Pipe(), offload_policy = "none", selection = sel
    )
    monkeypatch.delenv("UNSLOTH_DIFFUSION_BG_COMPILE")
    assert "background compile" not in mg.cfg_parallel_decline(
        Pipe(), offload_policy = "none", selection = sel
    )


def test_a_displaced_router_is_reported_declined():
    dit = _Dit()
    router, _replicas = _router(dit)

    class Pipe:
        transformer = dit

    pipe = Pipe()
    mg._CFG_ROUTERS[pipe] = router
    assert mg.drop_displaced_cfg_router(pipe, "graphs") is False
    graphed = lambda *a, **k: None  # noqa: E731
    dit.__dict__["forward"] = graphed
    assert mg.drop_displaced_cfg_router(pipe, "graphs") is True
    assert dit.__dict__["forward"] is graphed
    assert mg.placement_status(pipe) == {"cfg_parallel": {"armed": False, "declined": "graphs"}}
    assert mg.drop_displaced_cfg_router(pipe, "graphs") is False


def test_kept_models_keep_every_media_load_on_one_card(monkeypatch):
    from core.inference import gpu_residency

    _fake_cuda_host(monkeypatch, count = 2)
    seen = {}

    class Backend:
        @mg.with_load_selection
        def _run_load(self, **kwargs):
            seen["selection"] = mg.current_selection()

    Backend()._run_load(gpu_ordinal = None, _gpu_ordinals = mg.AUTOMATIC_ORDINALS)
    assert seen["selection"] is not None
    Backend()._run_load(gpu_ordinal = 0, _gpu_ordinals = [0, 1])
    assert seen["selection"] is not None
    for mode in ("fit", "always"):
        monkeypatch.setattr(gpu_residency, "current_policy", lambda mode = mode: mode)
        Backend()._run_load(gpu_ordinal = None, _gpu_ordinals = mg.AUTOMATIC_ORDINALS)
        assert seen["selection"] is None
        Backend()._run_load(gpu_ordinal = 0, _gpu_ordinals = [0, 1])
        assert seen["selection"] is None


def test_status_models_keep_the_multi_gpu_report():
    from models.inference import DiffusionStatusResponse, VideoStatusResponse

    report = {"primary": "cuda:0", "secondary": "cuda:1", "moved": ["text_encoder"]}
    assert (
        DiffusionStatusResponse(loaded = True, multi_gpu = report).model_dump()["multi_gpu"] == report
    )
    assert VideoStatusResponse(loaded = True, multi_gpu = report).model_dump()["multi_gpu"] == report


@pytest.mark.parametrize("module_file", ["video.py"])
def test_video_split_fallback_recomputes_the_floor(module_file):
    import ast
    from pathlib import Path

    source = (Path(__file__).resolve().parents[1] / "core" / "inference" / module_file).read_text(
        encoding = "utf-8"
    )
    tree = ast.parse(source)
    hits = [
        node
        for node in ast.walk(tree)
        if isinstance(node, ast.If)
        and "secondary_ordinal" in ast.unparse(node.test)
        and "plan is not planned" in ast.unparse(node.test)
        and any(
            isinstance(b, ast.Assign) and getattr(b.targets[0], "id", None) == "vram_floor_mib"
            for b in node.body
        )
    ]
    assert len(hits) == 1


def test_a_split_refuses_a_controlnet_it_cannot_stream():
    # The resident fallback would put the whole ControlNet beside a denoiser one card could not hold with it.
    import ast
    from pathlib import Path

    path = Path(__file__).resolve().parents[1] / "core" / "inference" / "diffusion.py"
    tree = ast.parse(path.read_text(encoding = "utf-8"))
    tests = [
        ast.unparse(node.test)
        for node in ast.walk(tree)
        if isinstance(node, ast.If) and "_offload_controlnet_module" in ast.unparse(node.test)
    ]
    assert len(tests) == 1
    branch = next(
        node
        for node in ast.walk(tree)
        if isinstance(node, ast.If) and "_offload_controlnet_module" in ast.unparse(node.test)
    )
    refusal = branch.orelse[0]
    assert (
        isinstance(refusal, ast.If)
        and ast.unparse(refusal.test) == "calibrated or is_split(state.pipe)"
    )
    assert any(isinstance(n, ast.Raise) for n in ast.walk(refusal))


def test_replica_rebuild_counts_the_allocator_cache_as_free(monkeypatch):
    import types as _types

    dit = _Dit()
    router, _replicas = _router(dit)
    router.secondary = torch.device("cuda", 1)
    monkeypatch.setattr(mg, "_free_total_mib", lambda o: (100, 15360))
    monkeypatch.setattr(torch.cuda, "memory_reserved", lambda d = None: 4096 << 20)
    monkeypatch.setattr(torch.cuda, "memory_allocated", lambda d = None: 0)
    monkeypatch.setattr(mg, "_device_context", lambda d: contextlib.nullcontext())
    monkeypatch.setattr(
        mg, "_ReplicaWorker", lambda device: _types.SimpleNamespace(close = lambda: None)
    )
    router._build_replica()
    assert router.replica is not None and router.disabled_reason is None
    router.replica = None
    monkeypatch.setattr(torch.cuda, "memory_reserved", lambda d = None: 0)
    router._build_replica()
    assert router.replica is None and "MiB free" in (router.disabled_reason or "")


def test_a_two_expert_video_split_prices_the_denoise_phase_on_both_cards():
    from core.inference import video as vid

    class Pipe:
        def __init__(self):
            self.components = {
                "transformer": torch.nn.Linear(1024, 1024),
                "transformer_2": torch.nn.Linear(1024, 1024),
                "vae": torch.nn.Linear(512, 512),
            }

    pipe = Pipe()
    mg._PLACEMENTS[pipe] = {
        "placement": {"transformer": "cuda:0", "vae": "cuda:0", "transformer_2": "cuda:1"},
        "moved": ["transformer_2"],
    }
    try:
        cards = vid._video_split_card_floors(pipe)
        assert cards == ((0, 5, True, True, True), (1, 4, False, False, True))
        shape = dict(width = 832, height = 480, frames = 25)
        roomy = {0: 1 << 20, 1: 1 << 20}
        assert (
            vid.video_split_shortfall_message(cards, family = "f", available_mib = roomy.get, **shape)
            is None
        )
        # the expert card has room for its weights but not the denoise activations
        expert_short = {0: 1 << 20, 1: 4 + dm.DEFAULT_BASE_OVERHEAD_MIB + 100}
        message = vid.video_split_shortfall_message(
            cards, family = "f", available_mib = expert_short.get, **shape
        )
        assert message is not None and "cuda:1" in message
        primary_short = {0: 5 + dm.DEFAULT_BASE_OVERHEAD_MIB + 100, 1: 1 << 20}
        message = vid.video_split_shortfall_message(
            cards, family = "f", available_mib = primary_short.get, **shape
        )
        assert message is not None and "cuda:0" in message
    finally:
        del mg._PLACEMENTS[pipe]


def test_untiled_decode_probes_the_card_the_vae_lives_on(monkeypatch):
    import types as _types

    from core.inference import video_vae_untiled as vu

    vae = torch.nn.Linear(1, 1)
    latent = _types.SimpleNamespace(device = torch.device("cuda", 0))
    monkeypatch.setattr(
        torch.nn.Linear,
        "parameters",
        lambda self, recurse = True: iter([_types.SimpleNamespace(device = torch.device("cuda", 1))]),
    )
    assert vu._decode_device(vae, latent) == torch.device("cuda", 1)
    # same card, or an offloaded VAE whose weights sit on the host: the latent's card, as before
    monkeypatch.setattr(
        torch.nn.Linear,
        "parameters",
        lambda self, recurse = True: iter([_types.SimpleNamespace(device = torch.device("cuda", 0))]),
    )
    assert vu._decode_device(vae, latent) == torch.device("cuda", 0)
    monkeypatch.setattr(
        torch.nn.Linear,
        "parameters",
        lambda self, recurse = True: iter([_types.SimpleNamespace(device = torch.device("cpu"))]),
    )
    assert vu._decode_device(vae, latent) == torch.device("cuda", 0)


def test_untiled_decode_sizes_the_card_the_vae_decodes_on(monkeypatch):
    import types as _types

    from core.inference import video_vae_untiled as vu

    family = next(iter({**vu._BYTES_PER_LATENT_PIXEL, **vu._BYTES_PER_FRAME_LATENT_PIXEL}))
    probed = []
    monkeypatch.setattr(vu, "_free_bytes", lambda device: probed.append(device))

    class Vae:
        use_tiling = True

        def decode(self, z, *args, **kwargs):
            return "tiled"

        def parameters(self):
            return iter([_types.SimpleNamespace(device = torch.device("cuda", 1))])

    pipe = _types.SimpleNamespace(vae = Vae())
    assert vu.install_untiled_decode(pipe, family)
    latent = _types.SimpleNamespace(device = torch.device("cuda", 0), shape = (1, 16, 3, 8, 8))
    assert pipe.vae.decode(latent) == "tiled"
    assert probed == [torch.device("cuda", 1)]


@pytest.mark.parametrize("module_name", ["diffusion", "video"])
def test_begin_load_hands_the_picked_cards_to_the_load_thread(monkeypatch, module_name):
    import importlib
    import types as _types

    backend_mod = importlib.import_module(f"core.inference.{module_name}")
    backend_cls = (
        backend_mod.DiffusionBackend if module_name == "diffusion" else backend_mod.VideoBackend
    )
    seen = {}

    def first_load_step(*args, **kwargs):
        seen["selection"] = mg.current_selection()
        raise RuntimeError("stop after the selection is read")

    class InlineThread:
        def __init__(
            self,
            target,
            kwargs,
            daemon = None,
        ):
            self.target, self.kwargs = target, kwargs

        def start(self):
            self.target(**self.kwargs)

    monkeypatch.setattr(backend_mod, "account_thread", InlineThread)
    monkeypatch.setattr(
        backend_mod,
        "resolve_diffusion_device_target",
        lambda *a, **k: _types.SimpleNamespace(device = "cuda"),
    )
    monkeypatch.setattr(backend_mod, "resolve_selected_cuda_ordinal", lambda ids: 0)
    monkeypatch.setattr(backend_mod, "resolve_selected_cuda_ordinals", lambda ids: [0, 1])
    monkeypatch.setattr(mg, "_capability", lambda ordinal: None)
    monkeypatch.setattr(mg, "models_kept_loaded", lambda: False)
    fam = _types.SimpleNamespace(name = "ltx-2", base_repo = "base/repo", single_file_is_pipeline = False)
    monkeypatch.setattr(backend_cls, "validate_load_request", lambda self, *a, **k: fam)
    if module_name == "diffusion":
        monkeypatch.setattr(backend_cls, "assert_precision_available", lambda self, *a, **k: None)
        monkeypatch.setattr(backend_mod, "detect_family_for_pick", first_load_step)
    else:
        from core.inference import video_minimax_h3

        monkeypatch.setattr(backend_mod, "assert_video_precision_available", lambda *a, **k: None)
        monkeypatch.setattr(backend_mod, "_ltx23_prequant_serves_on_card", lambda *a, **k: False)
        monkeypatch.setattr(video_minimax_h3, "is_h3_native", lambda *a, **k: False)
        monkeypatch.setattr(backend_mod, "_detect_load_family", first_load_step)

    backend_cls().begin_load("org/model", gpu_ids = [0, 1])
    assert seen["selection"] == mg.MultiGpuSelection(primary = 0, secondaries = (1,))
    # and only for that load
    assert mg.current_selection() is None

    # the kill switch: the extra cards are never even resolved (the single pick is, as before the multi-GPU layer)
    def never(ids):
        raise AssertionError("resolved several cards under UNSLOTH_DIFFUSION_MULTIGPU=0")

    monkeypatch.setenv(mg.MULTIGPU_ENV, "0")
    monkeypatch.setattr(backend_mod, "resolve_selected_cuda_ordinals", never)
    seen.clear()
    backend_cls().begin_load("org/model", gpu_ids = [0, 1])
    assert seen["selection"] is None


def test_fingerprint_sees_buffers_and_the_peft_adapter_selection():
    dit = _Dit()
    dit.proj.register_buffer("w_scale", torch.ones(4))
    base = mg.weights_fingerprint(dit)
    dit.proj.w_scale.mul_(2.0)
    edited = mg.weights_fingerprint(dit)
    assert edited != base
    dit.proj.w_scale = torch.ones(4)
    assert mg.weights_fingerprint(dit) not in (base, edited)
    before = mg.weights_fingerprint(dit)
    dit.proj._active_adapter = ["a"]
    with_a = mg.weights_fingerprint(dit)
    dit.proj._active_adapter = ["b"]
    assert len({before, with_a, mg.weights_fingerprint(dit)}) == 3


def test_replica_shares_the_nvfp4_step_controller():
    from core.inference.diffusion_nvfp4_protect import NVFP4StepController

    dit = _Dit()
    ctl = NVFP4StepController("0")
    dit.proj.protect = ctl
    replica = mg.build_replica(dit, torch.device("cpu"))
    assert replica.proj.protect is ctl


def test_boundary_and_router_removal_restore_an_instance_override():
    import types

    dit = _Dit()

    def own_forward(self, *args, **kwargs):
        return ("own",)

    override = types.MethodType(own_forward, dit)
    dit.forward = override
    mg.install_boundary(dit, "cpu")
    assert dit.__dict__["forward"] is not override
    mg.remove_boundary(dit)
    assert dit.__dict__["forward"] is override
    plain = _Dit()
    mg.install_boundary(plain, "cpu")
    mg.remove_boundary(plain)
    assert "forward" not in plain.__dict__

    class Pipe:
        pass

    pipe = Pipe()
    router = mg.CfgParallelRouter(dit, primary = "cpu", secondary = "cpu")
    dit.forward = router
    mg._CFG_ROUTERS[pipe] = router
    mg.uninstall_cfg_parallel(pipe)
    assert dit.__dict__["forward"] is override


def test_uninstalled_router_left_inside_a_wrapper_is_a_passthrough():
    torch.manual_seed(0)
    dit = _Dit()
    latents = torch.randn(2, 4)
    reference = _cfg_loop(dit, latents.clone(), steps = 4)
    router, replicas = _router(dit)

    class Skip:
        def __init__(self, inner):
            self.inner = inner

        def __call__(self, *args, **kwargs):
            return self.inner(*args, **kwargs)

    dit.forward = Skip(router)

    class Pipe:
        pass

    pipe = Pipe()
    mg._CFG_ROUTERS[pipe] = router
    assert mg.drop_displaced_cfg_router(pipe, "replaced")
    out = _cfg_loop(dit, latents.clone(), steps = 4)
    assert torch.equal(out, reference)
    assert replicas == [] and router.replica is None and router.prev_latent is None
    assert router.stats["replica_calls"] == 0


def test_router_stays_on_the_source_for_a_forced_eager_render_and_the_compile_thread():
    from core.inference import diffusion_bg_compile as bg

    torch.manual_seed(0)
    dit = _Dit()
    latents = torch.randn(2, 4)
    reference = _cfg_loop(dit, latents.clone(), steps = 4)
    router, replicas = _router(dit)
    for var in (bg._FORCE_EAGER, bg._NO_CAPTURE):
        token = var.set(True)
        try:
            out = _cfg_loop(dit, latents.clone(), steps = 4)
        finally:
            var.reset(token)
        assert torch.equal(out, reference)
        assert replicas == [] and router.stats["replica_calls"] == 0 and router.prev_latent is None
    out = _cfg_loop(dit, latents.clone(), steps = 4)
    assert torch.equal(out, reference) and router.stats["replica_calls"] == 3


def test_router_keeps_the_forward_signature_callers_filter_on():
    """FLUX.3's modular denoise step filters kwargs by forward's signature; a bare router dropped img_ids/txt_ids."""
    import inspect

    class Dit(torch.nn.Module):
        def __init__(self):
            super().__init__()
            self.lin = torch.nn.Linear(4, 4)

        def forward(
            self,
            hidden_states,
            encoder_hidden_states = None,
            timestep = None,
            img_ids = None,
            txt_ids = None,
            return_dict = True,
        ):
            assert img_ids is not None and txt_ids is not None, "the ids were filtered out"
            return (self.lin(hidden_states) + img_ids.sum() + txt_ids.sum(),)

    dit = Dit()
    router = mg.CfgParallelRouter(dit, primary = "cpu", secondary = "cpu")
    dit.forward = router
    fields = {"img_ids": torch.zeros(2, 4), "txt_ids": torch.zeros(3, 4), "guidance_hint": 1.0}
    # the call diffusers' modular_pipelines/flux3/denoise.py makes
    accepted = set(inspect.signature(dit.forward).parameters)
    kwargs = {k: v for k, v in fields.items() if k in accepted}
    assert sorted(kwargs) == ["img_ids", "txt_ids"]
    x = torch.randn(1, 4)
    out = dit(
        hidden_states = x,
        encoder_hidden_states = None,
        timestep = torch.ones(1),
        return_dict = False,
        **kwargs,
    )[0]
    assert torch.equal(out, dit.lin(x))


def test_replica_blocks_get_their_own_compile_guard():
    from core.inference.diffusion_speed import guard_compiled_blocks

    mg.install_compile_recorder()
    dit = _Dit()
    dit.proj.compile(backend = "eager")  # lazy: nothing is traced here
    assert guard_compiled_blocks(dit) == 1
    source_guard = dit._unsloth_compile_guard
    replica = mg.build_replica(dit, torch.device("cpu"))
    wrapped = replica.proj._compiled_call_impl
    guard = getattr(wrapped, "_unsloth_compile_guard", None)
    assert guard is not None and guard is not source_guard
    assert replica.__dict__.get("_unsloth_compile_guard") is guard


def test_replica_builds_inside_inference_mode_from_torchao_int8_weights():
    # Regression: building inside inference_mode raised "Cannot set version_counter" on torchao int8 weights.
    pytest.importorskip("torchao")
    from torchao.quantization import quantize_

    from core.inference.diffusion_transformer_quant import TQ_INT8, _make_quant_config

    torch.manual_seed(0)
    dit = torch.nn.Sequential(torch.nn.Linear(64, 64), torch.nn.GELU(), torch.nn.Linear(64, 64))
    with torch.no_grad():
        quantize_(dit, _make_quant_config(TQ_INT8))
    assert type(dit[0].weight).__module__.startswith("torchao")
    x = torch.randn(8, 64)
    with torch.inference_mode():
        replica = mg.build_replica(dit, torch.device("cpu"))
        assert torch.equal(replica(x), dit(x))
    assert replica[0].weight is not dit[0].weight


class _ScaleDit(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.register_buffer("scale", torch.ones(4))

    def forward(
        self,
        hidden_states,
        timestep,
        encoder_hidden_states,
        return_dict = False,
    ):
        return (hidden_states * self.scale * timestep + encoder_hidden_states.sum(),)


def test_probe_sees_an_in_place_edit_of_an_inference_tensor():
    with torch.inference_mode():
        dit = _ScaleDit()
        assert dit.scale.is_inference()
    fp, probe = mg.weights_fingerprint(dit), mg.weights_probe(dit)
    with torch.inference_mode():
        dit.scale.mul_(3.0)
    # no version counter on an inference tensor: the fingerprint cannot see the edit, the probe does
    assert mg.weights_fingerprint(dit) == fp
    assert probe is not None and mg.weights_probe(dit) != probe
    assert mg.weights_probe(_Dit()) is None


def test_probe_stays_inside_inference_weights_past_two_to_the_24_bytes():
    # 3072 x 3072 bf16 = 18,874,368 bytes: a float32 linspace puts its last sample at n, one past the end
    with torch.inference_mode():
        dit = torch.nn.Linear(3072, 3072, bias = False, dtype = torch.bfloat16)
        dit.weight.fill_(1.0)
        dit.weight.view(-1)[-1] = 2.0
    probe = mg.weights_probe(dit)
    assert probe is not None and len(probe) == mg._PROBE_SAMPLES
    with torch.inference_mode():
        dit.weight.view(-1)[-1] = -2.0  # flips the high (last) byte of the last element
    assert mg.weights_probe(dit) != probe  # the last byte is sampled


def test_an_in_place_edit_of_inference_weights_rebuilds_the_replica_on_the_next_render():
    with torch.inference_mode():
        dit = _ScaleDit()
        router, replicas = _router(dit)
        latents = torch.randn(2, 4)
        _cfg_loop(dit, latents.clone(), steps = 3)
        router.end_render()
        assert len(replicas) == 1 and router.stats["replica_rebuilds"] == 0
        _cfg_loop(dit, latents.clone(), steps = 3)
        router.end_render()
        assert len(replicas) == 1  # unchanged weights: the probe matches, no rebuild
        dit.scale.mul_(3.0)
        _cfg_loop(dit, latents.clone(), steps = 3)
    assert router.stats["replica_rebuilds"] == 1 and len(replicas) == 2
    assert router.disabled_reason is None


def test_invalidate_cfg_replica_drops_a_built_replica():
    class Pipe:
        pass

    pipe = Pipe()
    dit = _Dit()
    router, replicas = _router(dit)
    mg._CFG_ROUTERS[pipe] = router
    try:
        with torch.no_grad():
            _cfg_loop(dit, torch.randn(2, 4), steps = 2)
        assert router.replica is not None
        assert mg.invalidate_cfg_replica(pipe, "test") is True
        assert router.replica is None and router.stats["replica_rebuilds"] == 1
        with torch.no_grad():
            _cfg_loop(dit, torch.randn(2, 4), steps = 2)
        assert len(replicas) == 2
    finally:
        mg._CFG_ROUTERS.pop(pipe, None)


@pytest.mark.parametrize("dynamic", [None, False])
def test_a_warm_signature_never_compiles_on_a_changed_number(dynamic, monkeypatch, real_cuda):
    # A warm call runs beside the other side, so it must never compile (FX tracing is process-wide up to 2.13).
    # dynamic=False recompiles per number: past the limit Dynamo runs eager and "no compile" proves nothing.
    cfg = torch._dynamo.config
    for limit, fail in (
        ("recompile_limit", "fail_on_recompile_limit_hit"),
        ("cache_size_limit", None),
    ):
        if hasattr(cfg, limit):
            monkeypatch.setattr(cfg, limit, 64)
            if fail and hasattr(cfg, fail):
                monkeypatch.setattr(cfg, fail, True)
            break
    torch._dynamo.reset()
    compiles = []

    def backend(gm, _example_inputs):
        compiles.append(1)
        return gm.forward

    @torch.compile(backend = backend, dynamic = dynamic)
    def fn(x, scale, k, kw):
        return x * scale + k + kw["s"]

    x = torch.randn(4)
    warm: set = set()
    calls = [
        (1.0, 1, 0.5),
        (2.0, 1, 0.5),
        (3.0, 1, 0.5),
        (3.0, 2, 0.5),
        (3.0, 3, 0.5),
        (4.5, 7, 0.25),
        (4.5, 0, 0.25),
        (4.5, 1, 0.25),
        (4.5, 7, 0.25),
        (5.5, 1, 0.25),
    ]
    for scale, k, s in calls:
        sig = mg._call_signature((x, scale, k), {"kw": {"s": s}})
        before = len(compiles)
        fn(x, scale, k, {"s": s})
        if sig in warm:
            assert len(compiles) == before, (scale, k, s)
        mg._note_warm(warm, sig)
    torch._dynamo.reset()
    assert compiles


def test_warm_signature_keys_on_storage_offset_and_autocast():
    base = torch.zeros(8)
    a = mg._call_signature((base[:4],), {})
    assert mg._call_signature((base[4:],), {}) != a
    with torch.autocast("cpu", dtype = torch.bfloat16):
        assert mg._call_signature((base[:4],), {}) != a
    with torch.autocast("cpu", enabled = False, cache_enabled = False):
        assert mg._call_signature((base[:4],), {}) != a
    assert mg._call_signature((base[:4],), {}) == a


def test_automatic_partner_skips_a_card_holding_the_splits_components(monkeypatch):
    """Three cards, Automatic: the idlest card holds a small text encoder of the split, so it cannot take the replica;
    the next idle card does (the occupied card is excluded before the partner is chosen, not after)."""

    class Pipe:
        transformer = _Dit()

    probes = {
        0: mg.DeviceProbe(ordinal = 0, name = "B200", total_mib = 180000, free_mib = 170000, util_pct = 0.0),
        1: mg.DeviceProbe(ordinal = 1, name = "B200", total_mib = 180000, free_mib = 179900, util_pct = 0.0),
        2: mg.DeviceProbe(ordinal = 2, name = "B200", total_mib = 180000, free_mib = 179800, util_pct = 0.0),
    }
    monkeypatch.setattr(mg, "cfg_parallel_decline", lambda pipe, **kw: None)
    monkeypatch.setattr(mg, "_module_device", lambda m: torch.device("cuda", 0))
    monkeypatch.setattr(
        mg, "probe_devices", lambda ordinals, **kw: {o: probes[o] for o in ordinals}
    )
    pipe = Pipe()
    mg._PLACEMENTS[pipe] = {
        "placement": {"transformer": "cuda:0", "vae": "cuda:0", "text_encoder": "cuda:1"}
    }
    try:
        with mg.selection_scope(
            mg.MultiGpuSelection(primary = 0, secondaries = (1, 2), automatic = True)
        ):
            assert mg.install_cfg_parallel(pipe, offload_policy = "none") is None
        assert str(mg._CFG_ROUTERS[pipe].secondary) == "cuda:2"
    finally:
        mg.uninstall_cfg_parallel(pipe)
        mg._PLACEMENTS.pop(pipe, None)