Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

37.1 KB
Raw Blame
"""#837: fused Fast-LoRA SwiGLU MLP correctness tests."""

from __future__ import annotations

import copy
from types import SimpleNamespace

import pytest

pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning")


def _deps():
    torch = pytest.importorskip("torch")
    pytest.importorskip("peft")
    return torch


def _randomise_b(model):
    torch = _deps()
    with torch.no_grad():
        for name, param in model.named_parameters():
            if "lora_B" in name:
                torch.nn.init.normal_(param, std=0.2)


def _make_model(
    targets,
    *,
    dropout=0.0,
    act="silu",
    module_act=None,
    ranks=None,
    bias=True,
    modules_to_save=None,
):
    _deps()
    import torch.nn as nn
    from peft import LoraConfig, inject_adapter_in_model

    class TinyMLP(nn.Module):
        def __init__(self):
            super().__init__()
            self.gate_proj = nn.Linear(8, 12, bias=bias)
            self.up_proj = nn.Linear(8, 12, bias=bias)
            self.down_proj = nn.Linear(12, 8, bias=bias)
            activation = module_act or act
            self.act_fn = nn.SiLU() if activation in {"silu", "swish"} else nn.GELU()

        def forward(self, x):
            return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))

    class Model(nn.Module):
        def __init__(self):
            super().__init__()
            self.config = SimpleNamespace(hidden_act=act)
            self.mlp = TinyMLP()

        def forward(self, x):
            return self.mlp(x)

    model = Model()
    config = LoraConfig(
        r=2,
        lora_alpha=4,
        lora_dropout=dropout,
        bias="none",
        target_modules=list(targets),
        rank_pattern=ranks or {},
        modules_to_save=modules_to_save,
    )
    inject_adapter_in_model(config, model)
    _randomise_b(model)
    return model


def _adapter_grads(model):
    return {
        name: p.grad.detach().clone()
        for name, p in model.named_parameters()
        if "lora_" in name and p.grad is not None
    }


class TestMath:
    def test_float64_gradcheck_all_six_adapter_matrices_unequal_ranks(self):
        torch = _deps()
        from soup_cli.utils.fast_lora_mlp import _mlp_function

        torch.manual_seed(3)
        x = torch.randn(5, 8, dtype=torch.float64, requires_grad=True)
        wg = torch.randn(12, 8, dtype=torch.float64)
        wu = torch.randn(12, 8, dtype=torch.float64)
        wd = torch.randn(8, 12, dtype=torch.float64)
        bg = torch.randn(12, dtype=torch.float64)
        bu = torch.randn(12, dtype=torch.float64)
        bd = torch.randn(8, dtype=torch.float64)
        ag = torch.randn(2, 8, dtype=torch.float64, requires_grad=True)
        bgl = torch.randn(12, 2, dtype=torch.float64, requires_grad=True)
        au = torch.randn(3, 8, dtype=torch.float64, requires_grad=True)
        bul = torch.randn(12, 3, dtype=torch.float64, requires_grad=True)
        ad = torch.randn(4, 12, dtype=torch.float64, requires_grad=True)
        bdl = torch.randn(8, 4, dtype=torch.float64, requires_grad=True)
        fn = _mlp_function()

        def call(x_, ag_, bgl_, au_, bul_, ad_, bdl_):
            return fn.apply(
                x_, wg, bg, wu, bu, wd, bd,
                ag_, bgl_, au_, bul_, ad_, bdl_,
                1.7, 0.8, 2.1, None, None, None,
            )

        assert torch.autograd.gradcheck(
            call, (x, ag, bgl, au, bul, ad, bdl), eps=1e-6, atol=1e-5, rtol=1e-4
        )

    @pytest.mark.parametrize("missing", ["gate", "up", "down"])
    def test_float64_gradcheck_with_each_projection_unadapted(self, missing):
        torch = _deps()
        from soup_cli.utils.fast_lora_mlp import _mlp_function

        torch.manual_seed(4)
        x = torch.randn(3, 8, dtype=torch.float64, requires_grad=True)
        weights = (
            torch.randn(12, 8, dtype=torch.float64),
            torch.randn(12, 8, dtype=torch.float64),
            torch.randn(8, 12, dtype=torch.float64),
        )
        biases = (
            torch.randn(12, dtype=torch.float64),
            torch.randn(12, dtype=torch.float64),
            torch.randn(8, dtype=torch.float64),
        )
        shapes = {
            "gate": ((2, 8), (12, 2)),
            "up": ((3, 8), (12, 3)),
            "down": ((4, 12), (8, 4)),
        }
        active = []
        for name in ("gate", "up", "down"):
            if name == missing:
                continue
            for shape in shapes[name]:
                active.append(torch.randn(*shape, dtype=torch.float64, requires_grad=True))

        def call(x_, *adapter_values):
            values = iter(adapter_values)
            adapters = []
            for name in ("gate", "up", "down"):
                if name == missing:
                    adapters.extend((x_.new_empty(0), x_.new_empty(0)))
                else:
                    adapters.extend((next(values), next(values)))
            return _mlp_function().apply(
                x_,
                weights[0], biases[0], weights[1], biases[1], weights[2], biases[2],
                *adapters,
                1.7, 0.8, 2.1, None, None, None,
            )

        assert torch.autograd.gradcheck(
            call, (x, *active), eps=1e-6, atol=1e-5, rtol=1e-4
        )

    @pytest.mark.parametrize(
        "targets",
        [
            ("gate_proj", "up_proj", "down_proj"),
            ("gate_proj",),
            ("up_proj",),
            ("down_proj",),
            ("gate_proj", "down_proj"),
        ],
    )
    def test_fp32_forward_and_backward_match_peft_for_partial_adapters(self, targets):
        torch = _deps()
        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        torch.manual_seed(11)
        model = _make_model(targets)
        x_ref = torch.randn(2, 5, 8, requires_grad=True)
        ref = model(x_ref)
        loss_ref = ref.square().mean()
        loss_ref.backward()
        ref_x = x_ref.grad.detach().clone()

        ref_grads = _adapter_grads(model)

        for p in model.parameters():
            p.grad = None
        x_fast = x_ref.detach().clone().requires_grad_(True)
        assert patch_fast_lora_mlp(model) == 1
        out = model(x_fast)
        out.square().mean().backward()

        torch.testing.assert_close(out, ref)
        torch.testing.assert_close(x_fast.grad, ref_x)
        got_grads = _adapter_grads(model)
        assert got_grads.keys() == ref_grads.keys()
        for name in ref_grads:
            torch.testing.assert_close(got_grads[name], ref_grads[name], msg=name)

    def test_unequal_rank_pattern_matches_peft(self):
        torch = _deps()
        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        torch.manual_seed(17)
        model = _make_model(
            ("gate_proj", "up_proj", "down_proj"),
            ranks={"gate_proj": 2, "up_proj": 3, "down_proj": 4},
        )
        ranks = tuple(
            model.mlp.__getattr__(name).lora_A["default"].weight.shape[0]
            for name in ("gate_proj", "up_proj", "down_proj")
        )
        assert ranks == (2, 3, 4)

        x0 = torch.randn(7, 8, requires_grad=True)
        ref = model(x0)
        ref.sum().backward()
        ref_x = x0.grad.detach().clone()
        ref_grads = _adapter_grads(model)
        for p in model.parameters():
            p.grad = None

        x1 = x0.detach().clone().requires_grad_(True)
        assert patch_fast_lora_mlp(model) == 1
        got = model(x1)
        got.sum().backward()
        torch.testing.assert_close(got, ref)
        torch.testing.assert_close(x1.grad, ref_x)
        for name, grad in _adapter_grads(model).items():
            torch.testing.assert_close(grad, ref_grads[name], msg=name)


class TestPatching:
    def test_patch_is_idempotent_and_reversible(self):
        _deps()
        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp, unpatch_fast_lora_mlp

        model = _make_model(("gate_proj", "up_proj", "down_proj"))
        assert patch_fast_lora_mlp(model) == 1
        assert patch_fast_lora_mlp(model) == 0
        assert unpatch_fast_lora_mlp(model) == 1
        assert unpatch_fast_lora_mlp(model) == 0
        assert "forward" not in vars(model.mlp)

    def test_non_silu_activation_falls_back_without_patch(self):
        _deps()
        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        model = _make_model(("gate_proj",), act="gelu")
        assert patch_fast_lora_mlp(model) == 0

    def test_parent_config_hidden_act_gate_refuses_a_silu_module(self, caplog):
        _deps()
        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        model = _make_model(("gate_proj",), act="gelu", module_act="silu")
        with caplog.at_level("INFO", logger="soup_cli.utils.fast_lora_mlp"):
            assert patch_fast_lora_mlp(model) == 0
        assert "hidden_act='gelu'" in caplog.text

    def test_nonzero_dropout_delegates_to_original_mlp(self):
        torch = _deps()
        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        model = _make_model(("gate_proj", "up_proj", "down_proj"), dropout=0.25)
        model.train()
        calls = []
        original = model.mlp.forward

        def spy(x):
            calls.append(True)
            return original(x)

        model.mlp.forward = spy
        assert patch_fast_lora_mlp(model) == 1
        torch.manual_seed(9)
        model(torch.randn(2, 8))
        assert calls == [True]

    def test_modules_to_save_projection_keeps_peft_ownership(self):
        _deps()
        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        model = _make_model(
            ("gate_proj", "up_proj"), modules_to_save=["down_proj"]
        )

        assert hasattr(model.mlp.down_proj, "modules_to_save")
        assert patch_fast_lora_mlp(model) == 0

    def test_real_peft_8bit_layers_are_not_patched(self):
        _deps()
        bnb = pytest.importorskip("bitsandbytes")
        import torch.nn as nn
        from peft import LoraConfig, inject_adapter_in_model

        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        class Tiny8BitMLP(nn.Module):
            def __init__(self):
                super().__init__()
                self.gate_proj = bnb.nn.Linear8bitLt(8, 12, bias=False)
                self.up_proj = bnb.nn.Linear8bitLt(8, 12, bias=False)
                self.down_proj = bnb.nn.Linear8bitLt(12, 8, bias=False)
                self.act_fn = nn.SiLU()

            def forward(self, x):
                return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))

        class Model(nn.Module):
            def __init__(self):
                super().__init__()
                self.config = SimpleNamespace(hidden_act="silu")
                self.is_loaded_in_8bit = True
                self.mlp = Tiny8BitMLP()

        model = Model()
        inject_adapter_in_model(
            LoraConfig(
                r=2,
                lora_alpha=4,
                target_modules=["gate_proj", "up_proj", "down_proj"],
            ),
            model,
        )

        assert all(
            type(getattr(model.mlp, name)).__name__ == "Linear8bitLt"
            for name in ("gate_proj", "up_proj", "down_proj")
        )
        assert patch_fast_lora_mlp(model) == 0

    def test_saturation_is_finite_on_mps_bfloat16(self):
        torch = _deps()
        if not torch.backends.mps.is_available():
            pytest.skip("Apple MPS is required")
        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        model = _make_model(("gate_proj", "up_proj", "down_proj")).to(
            device="mps", dtype=torch.bfloat16
        )
        with torch.no_grad():
            model.mlp.gate_proj.get_base_layer().weight.fill_(2.5)
        assert patch_fast_lora_mlp(model) == 1
        x = torch.ones(3, 8, device="mps", dtype=torch.bfloat16, requires_grad=True)
        y = model(x)
        y.float().sum().backward()
        assert torch.isfinite(y).all()
        assert torch.isfinite(x.grad).all()


class TestFastPathAndScope:
    @pytest.mark.parametrize(
        ("targets", "ranks"),
        [
            pytest.param(("gate_proj",), None, id="gate"),
            pytest.param(("up_proj",), None, id="up"),
            pytest.param(("down_proj",), None, id="down"),
            pytest.param(("gate_proj", "down_proj"), None, id="gate-down"),
            pytest.param(
                ("gate_proj", "up_proj", "down_proj"), None, id="all-equal-rank"
            ),
            pytest.param(
                ("gate_proj", "up_proj", "down_proj"),
                {"gate_proj": 2, "up_proj": 3, "down_proj": 4},
                id="all-unequal-rank",
            ),
        ],
    )
    @pytest.mark.parametrize("bias", [True, False], ids=["bias", "no-bias"])
    @pytest.mark.parametrize("shape", [(4, 8), (2, 5, 8)], ids=["2d", "3d"])
    @pytest.mark.parametrize("dtype_name", ["float32", "bfloat16"], ids=["fp32", "bf16"])
    def test_kernel_is_taken_across_targets_bias_rank_and_dtype(
        self, targets, ranks, bias, shape, dtype_name
    ):
        torch = _deps()
        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        dtype = getattr(torch, dtype_name)
        model = _make_model(targets, bias=bias, ranks=ranks).to(dtype=dtype)
        x = torch.randn(*shape, dtype=dtype, requires_grad=True)
        before = model(x)
        assert type(before.grad_fn).__name__ != "_FastLoraSwiGLUBackward"

        assert patch_fast_lora_mlp(model) == 1
        out = model(x)
        assert type(out.grad_fn).__name__ == "_FastLoraSwiGLUBackward"

    def test_cpu_autocast_preserves_pefts_bfloat16_output(self):
        torch = _deps()
        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        torch.manual_seed(29)
        model = _make_model(("gate_proj", "up_proj", "down_proj"))
        x = torch.randn(2, 5, 8, requires_grad=True)

        with torch.autocast("cpu", dtype=torch.bfloat16):
            reference = model(x)
        assert reference.dtype == torch.bfloat16

        assert patch_fast_lora_mlp(model) == 1
        with torch.autocast("cpu", dtype=torch.bfloat16):
            output = model(x)

        assert output.dtype == reference.dtype
        assert type(output.grad_fn).__name__ == "_FastLoraSwiGLUBackward"

    def test_single_projection_keeps_pefts_bfloat16_output_under_cpu_autocast(self):
        torch = _deps()
        import torch.nn as nn
        from peft import LoraConfig, inject_adapter_in_model

        from soup_cli.utils.fast_lora import patch_fast_lora_single_projection

        class Model(nn.Module):
            def __init__(self):
                super().__init__()
                self.o_proj = nn.Linear(8, 8)

        torch.manual_seed(0)
        model = Model()
        inject_adapter_in_model(
            LoraConfig(
                r=2,
                lora_alpha=4,
                lora_dropout=0.0,
                target_modules=["o_proj"],
            ),
            model,
        )
        x = torch.randn(2, 5, 8, requires_grad=True)
        with torch.autocast("cpu", dtype=torch.bfloat16):
            reference = model.o_proj(x)
        assert reference.dtype == torch.bfloat16

        assert patch_fast_lora_single_projection(model) == 1
        with torch.autocast("cpu", dtype=torch.bfloat16):
            output = model.o_proj(x)

        assert output.dtype == reference.dtype
        assert type(output.grad_fn).__name__ == "_FastLoraSingleProjectionBackward"

    def test_module_gelu_is_refused_even_when_parent_config_says_silu(self):
        _deps()
        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        model = _make_model(("gate_proj",), act="silu", module_act="gelu")
        assert patch_fast_lora_mlp(model) == 0

    def test_missing_module_activation_fails_closed(self):
        _deps()
        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        model = _make_model(("gate_proj",), act="silu")
        del model.mlp.act_fn
        assert patch_fast_lora_mlp(model) == 0

    def test_unadapted_int8_sibling_refuses_the_whole_mlp(self):
        torch = _deps()
        bnb = pytest.importorskip("bitsandbytes")
        import torch.nn as nn
        from peft import LoraConfig, inject_adapter_in_model

        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        class TinyMLP(nn.Module):
            def __init__(self):
                super().__init__()
                self.gate_proj = nn.Linear(8, 12)
                self.up_proj = bnb.nn.Linear8bitLt(8, 12, has_fp16_weights=False)
                self.down_proj = bnb.nn.Linear8bitLt(12, 8, has_fp16_weights=False)
                self.act_fn = nn.SiLU()

            def forward(self, x):
                return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))

        class Model(nn.Module):
            def __init__(self):
                super().__init__()
                self.config = SimpleNamespace(hidden_act="silu")
                self.mlp = TinyMLP()

            def forward(self, x):
                return self.mlp(x)

        model = Model().to("cpu")
        inject_adapter_in_model(
            LoraConfig(
                r=2,
                lora_alpha=4,
                lora_dropout=0.0,
                target_modules=["gate_proj"],
            ),
            model,
        )
        x = torch.randn(2, 3, 8, requires_grad=True)
        reference = model(x)
        assert not model.mlp.up_proj.weight.is_floating_point()
        assert patch_fast_lora_mlp(model) == 1
        output = model(x)
        assert type(output.grad_fn).__name__ != "_FastLoraSwiGLUBackward"
        torch.testing.assert_close(output, reference)

    def test_moe_expert_path_is_out_of_scope(self):
        _deps()
        import torch.nn as nn
        from peft import LoraConfig, inject_adapter_in_model

        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        class Expert(nn.Module):
            def __init__(self):
                super().__init__()
                self.gate_proj = nn.Linear(8, 12)
                self.up_proj = nn.Linear(8, 12)
                self.down_proj = nn.Linear(12, 8)
                self.act_fn = nn.SiLU()

            def forward(self, x):
                return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))

        class Model(nn.Module):
            def __init__(self):
                super().__init__()
                self.config = SimpleNamespace(hidden_act="silu")
                self.experts = nn.ModuleList([Expert()])

        model = Model()
        inject_adapter_in_model(
            LoraConfig(
                r=2,
                lora_alpha=4,
                lora_dropout=0.0,
                bias="none",
                target_modules=["gate_proj", "up_proj", "down_proj"],
            ),
            model,
        )
        assert patch_fast_lora_mlp(model) == 0

    @pytest.mark.parametrize(
        "device", ["cpu", pytest.param("cuda", marks=pytest.mark.gpu)]
    )
    @pytest.mark.parametrize("seed", [3, 7, 11])
    def test_nf4_forward_backward_and_all_adapter_grads_match_peft(
        self, seed, device
    ):
        torch = _deps()
        bnb = pytest.importorskip("bitsandbytes")
        import torch.nn as nn
        from peft import LoraConfig, inject_adapter_in_model

        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        class TinyNF4MLP(nn.Module):
            def __init__(self):
                super().__init__()
                self.gate_proj = bnb.nn.Linear4bit(
                    8, 12, bias=False, compute_dtype=torch.bfloat16, quant_type="nf4"
                )
                self.up_proj = bnb.nn.Linear4bit(
                    8, 12, bias=False, compute_dtype=torch.bfloat16, quant_type="nf4"
                )
                self.down_proj = bnb.nn.Linear4bit(
                    12, 8, bias=False, compute_dtype=torch.bfloat16, quant_type="nf4"
                )
                self.act_fn = nn.SiLU()

            def forward(self, x):
                return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))

        class Model(nn.Module):
            def __init__(self):
                super().__init__()
                self.config = SimpleNamespace(hidden_act="silu")
                self.mlp = TinyNF4MLP()

            def forward(self, x):
                return self.mlp(x)

        torch.manual_seed(seed)
        model = Model()
        with torch.no_grad():
            model.mlp.up_proj.weight.mul_(4)
        model = model.to(device)
        inject_adapter_in_model(
            LoraConfig(
                r=2,
                lora_alpha=4,
                lora_dropout=0.0,
                bias="none",
                target_modules=["gate_proj", "up_proj", "down_proj"],
            ),
            model,
        )
        _randomise_b(model)
        for name in ("gate_proj", "up_proj", "down_proj"):
            assert getattr(model.mlp, name).get_base_layer().weight.quant_state is not None

        reference_model = copy.deepcopy(model)
        x_ref = torch.randn(
            2, 3, 8, device=device, dtype=torch.bfloat16, requires_grad=True
        )
        reference = reference_model(x_ref)
        assert type(reference.grad_fn).__name__ != "_FastLoraSwiGLUBackward"
        reference.float().square().mean().backward()
        reference_x_grad = x_ref.grad.detach().clone()
        reference_grads = _adapter_grads(reference_model)

        oracle_x = x_ref.detach().double().requires_grad_(True)
        oracle_adapters = {}

        def oracle_projection(layer, value, projection_name):
            base = layer.get_base_layer()
            dense = bnb.functional.dequantize_4bit(
                base.weight, base.weight.quant_state
            ).double()
            adapter = layer.active_adapters[0]
            lora_a = layer.lora_A[adapter].weight.detach().double().requires_grad_(True)
            lora_b = layer.lora_B[adapter].weight.detach().double().requires_grad_(True)
            oracle_adapters[
                f"mlp.{projection_name}.lora_A.{adapter}.weight"
            ] = lora_a
            oracle_adapters[
                f"mlp.{projection_name}.lora_B.{adapter}.weight"
            ] = lora_b
            base_out = torch.nn.functional.linear(value, dense)
            lora_out = torch.nn.functional.linear(
                torch.nn.functional.linear(value, lora_a), lora_b
            )
            return base_out + lora_out * float(layer.scaling[adapter])

        oracle_gate = oracle_projection(
            reference_model.mlp.gate_proj, oracle_x, "gate_proj"
        )
        oracle_up = oracle_projection(reference_model.mlp.up_proj, oracle_x, "up_proj")
        oracle_hidden = torch.nn.functional.silu(oracle_gate) * oracle_up
        oracle = oracle_projection(
            reference_model.mlp.down_proj, oracle_hidden, "down_proj"
        )
        oracle.square().mean().backward()
        oracle_x_grad = oracle_x.grad.detach()
        oracle_grads = {
            name: parameter.grad.detach()
            for name, parameter in oracle_adapters.items()
        }

        x = x_ref.detach().clone().requires_grad_(True)
        assert patch_fast_lora_mlp(model) == 1
        out = model(x)
        assert type(out.grad_fn).__name__ == "_FastLoraSwiGLUBackward"
        out.float().square().mean().backward()

        measured_ratios = []

        def assert_within_twice_peft_error(actual, peft_value, oracle_value, name):
            actual_error = (actual.double() - oracle_value).abs().max().item()
            peft_error = (peft_value.double() - oracle_value).abs().max().item()
            ratio = (
                actual_error / peft_error
                if peft_error
                else (float("inf") if actual_error else 0.0)
            )
            measured_ratios.append((name, ratio))
            assert actual_error <= 2.0 * peft_error + 1e-8, (
                f"{name}: kernel error {actual_error:.6g} exceeds twice "
                f"peft error {peft_error:.6g}"
            )

        assert_within_twice_peft_error(out, reference, oracle.detach(), "output")
        assert_within_twice_peft_error(
            x.grad, reference_x_grad, oracle_x_grad, "input gradient"
        )
        got_grads = _adapter_grads(model)
        assert got_grads.keys() == reference_grads.keys()
        for name, grad in got_grads.items():
            assert_within_twice_peft_error(
                grad, reference_grads[name], oracle_grads[name], name
            )
        print("#837 NF4 fast/PEFT error ratios: " + repr(measured_ratios))


@pytest.mark.parametrize("seed", range(12))
def test_bf16_mlp_stays_within_twice_pefts_error_against_float64(
    seed, aten_half_matmuls
):
    torch = _deps()
    import torch.nn as nn
    from peft import LoraConfig, inject_adapter_in_model

    from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

    class TinyMLP(nn.Module):
        def __init__(self):
            super().__init__()
            self.gate_proj = nn.Linear(32, 48, bias=False)
            self.up_proj = nn.Linear(32, 48, bias=False)
            self.down_proj = nn.Linear(48, 32, bias=False)
            self.act_fn = nn.SiLU()

        def forward(self, x):
            return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))

    class Model(nn.Module):
        def __init__(self):
            super().__init__()
            self.config = SimpleNamespace(hidden_act="silu")
            self.mlp = TinyMLP()

        def forward(self, x):
            return self.mlp(x)

    torch.manual_seed(seed)
    model = Model()
    with torch.no_grad():
        model.mlp.up_proj.weight.mul_(4)
    inject_adapter_in_model(
        LoraConfig(
            r=4,
            lora_alpha=8,
            lora_dropout=0.0,
            target_modules=["gate_proj", "up_proj", "down_proj"],
        ),
        model,
    )
    _randomise_b(model)
    for name, param in model.named_parameters():
        # Soup's layout since #429: frozen base in bf16, trainable LoRA in fp32.
        param.data = param.data.float() if "lora_" in name else param.data.to(torch.bfloat16)
    x0 = torch.randn(2, 5, 32, dtype=torch.bfloat16)

    def run(module, x):
        x = x.clone().requires_grad_(True)
        out = module(x)
        out.double().square().mean().backward()
        grads = [
            param.grad
            for name, param in sorted(module.named_parameters())
            if "lora_" in name
        ]
        result = [tensor.detach().double().clone() for tensor in (out, x.grad, *grads)]
        module.zero_grad(set_to_none=True)
        return result

    oracle = run(copy.deepcopy(model).double(), x0.double())
    reference = run(model, x0)
    assert patch_fast_lora_mlp(model) == 1
    fast = run(model, x0)
    for got, ref, exact in zip(fast, reference, oracle):
        assert (got - exact).abs().max() <= 2 * (ref - exact).abs().max() + 1e-8
    assert (fast[0] - oracle[0]).abs().max() <= 1.05 * (
        reference[0] - oracle[0]
    ).abs().max() + 1e-8


class TestStreamedModel:
    def test_mlp_kernel_matches_resident_peft_across_reused_stream_buffers(
        self, tmp_path
    ):
        torch = _deps()
        pytest.importorskip("transformers")
        pytest.importorskip("safetensors")
        from peft import LoraConfig, TaskType, get_peft_model
        from safetensors.torch import save_file
        from transformers import AutoModelForCausalLM, LlamaConfig, LlamaForCausalLM

        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp
        from soup_cli.utils.layer_shard import shard_checkpoint
        from soup_cli.utils.layer_stream_runtime import build_streamed_model

        def lora_config():
            return LoraConfig(
                r=2,
                lora_alpha=4,
                lora_dropout=0.0,
                bias="none",
                target_modules=["gate_proj", "up_proj", "down_proj"],
                task_type=TaskType.CAUSAL_LM,
            )

        torch.manual_seed(41)
        config = LlamaConfig(
            vocab_size=64,
            hidden_size=16,
            intermediate_size=32,
            num_hidden_layers=4,
            num_attention_heads=2,
            num_key_value_heads=2,
            tie_word_embeddings=False,
            max_position_embeddings=64,
        )
        weights = tmp_path / "model"
        weights.mkdir(parents=True, exist_ok=True)
        save_file(
            {
                key: value.contiguous()
                for key, value in LlamaForCausalLM(config).state_dict().items()
            },
            str(weights / "model.safetensors"),
        )
        config.save_pretrained(str(weights))

        shards = str(tmp_path / "shards")
        index = shard_checkpoint(str(weights), shards, dtype="float32", arch="llama")
        streamed, _runtime = build_streamed_model(
            model_id=str(weights),
            shard_dir=shards,
            index=index,
            lora_config=lora_config(),
            device="cpu",
            dtype="float32",
            buffers=2,
            pin=False,
            seed=5,
        )
        assert patch_fast_lora_mlp(streamed) == 4
        _randomise_b(streamed)

        resident = get_peft_model(
            AutoModelForCausalLM.from_pretrained(str(weights), dtype=torch.float32),
            lora_config(),
        )
        resident_params = dict(resident.named_parameters())
        copied = 0
        for name, parameter in streamed.named_parameters():
            if "lora_" not in name or parameter.is_meta:
                continue
            target = next(
                (
                    candidate
                    for candidate_name, candidate in resident_params.items()
                    if candidate_name.endswith(name)
                ),
                None,
            )
            assert target is not None, f"resident model has no parameter ending in {name!r}"
            with torch.no_grad():
                target.copy_(parameter)
            copied += 1
        assert copied == 24

        input_ids = torch.randint(0, 64, (1, 8))
        labels = torch.randint(0, 64, (1, 8))
        seen = []
        hooks = [
            module.register_forward_hook(
                lambda _module, _inputs, output: seen.append(type(output.grad_fn).__name__)
            )
            for module in streamed.modules()
            if getattr(module, "_soup_fast_lora_mlp", False)
        ]
        streamed_loss = streamed(input_ids=input_ids, labels=labels).loss
        for hook in hooks:
            hook.remove()
        assert seen == ["_FastLoraSwiGLUBackward"] * 4, seen
        streamed_loss.backward()
        resident_loss = resident(input_ids=input_ids, labels=labels).loss
        resident_loss.backward()

        torch.testing.assert_close(streamed_loss, resident_loss, rtol=1e-5, atol=1e-5)
        streamed_grads = {
            name: parameter.grad
            for name, parameter in streamed.named_parameters()
            if "lora_" in name and parameter.grad is not None
        }
        resident_grads = {
            name: parameter.grad
            for name, parameter in resident.named_parameters()
            if "lora_" in name and parameter.grad is not None
        }
        assert len(streamed_grads) == len(resident_grads) == 24
        for name, grad in streamed_grads.items():
            target = next(
                (
                    candidate
                    for candidate_name, candidate in resident_grads.items()
                    if candidate_name.endswith(name)
                ),
                None,
            )
            assert target is not None, f"no resident gradient for {name!r}"
            torch.testing.assert_close(grad, target, rtol=1e-4, atol=1e-5, msg=name)


class TestRealLlamaAndSavedBytes:
    def test_real_llama_mlp_matches_peft(self):
        torch = _deps()
        from peft import LoraConfig, inject_adapter_in_model
        from transformers import LlamaConfig
        from transformers.models.llama.modeling_llama import LlamaMLP

        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        class Wrapper(torch.nn.Module):
            def __init__(self):
                super().__init__()
                self.config = LlamaConfig(
                    hidden_size=8,
                    intermediate_size=12,
                    num_hidden_layers=1,
                    num_attention_heads=2,
                    hidden_act="silu",
                )
                self.mlp = LlamaMLP(self.config)

            def forward(self, x):
                return self.mlp(x)

        torch.manual_seed(31)
        model = Wrapper()
        inject_adapter_in_model(
            LoraConfig(
                r=2,
                lora_alpha=4,
                lora_dropout=0.0,
                bias="none",
                target_modules=["gate_proj", "up_proj", "down_proj"],
            ),
            model,
        )
        _randomise_b(model)
        x0 = torch.randn(2, 5, 8, requires_grad=True)
        ref = model(x0)
        ref.square().mean().backward()
        ref_x = x0.grad.detach().clone()
        ref_grads = _adapter_grads(model)
        for param in model.parameters():
            param.grad = None

        x1 = x0.detach().clone().requires_grad_(True)
        assert patch_fast_lora_mlp(model) == 1
        got = model(x1)
        got.square().mean().backward()
        torch.testing.assert_close(got, ref)
        torch.testing.assert_close(x1.grad, ref_x)
        for name, grad in _adapter_grads(model).items():
            torch.testing.assert_close(grad, ref_grads[name], msg=name)

    def test_saved_tensor_bytes_are_lower_than_peft(self):
        torch = _deps()
        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        def saved_bytes(model):
            seen = set()
            total = 0

            def pack(tensor):
                nonlocal total
                key = (
                    tensor.untyped_storage().data_ptr(),
                    tensor.storage_offset(),
                    tuple(tensor.shape),
                    tuple(tensor.stride()),
                )
                if key not in seen:
                    seen.add(key)
                    total += tensor.numel() * tensor.element_size()
                return tensor

            x = torch.randn(2, 64, 8, requires_grad=True)
            with torch.autograd.graph.saved_tensors_hooks(pack, lambda tensor: tensor):
                model(x).sum().backward()
            return total

        torch.manual_seed(0)
        model = _make_model(("gate_proj", "up_proj", "down_proj"))
        plain = saved_bytes(model)
        for param in model.parameters():
            param.grad = None
        assert patch_fast_lora_mlp(model) == 1
        fast = saved_bytes(model)

        assert 0 < fast < plain
        print(f"#837 saved bytes: peft={plain} fast={fast}")

    def test_saved_bytes_at_llama_31_8b_shape(self):
        torch = _deps()
        import torch.nn as nn
        from peft import LoraConfig, inject_adapter_in_model

        from soup_cli.utils.fast_lora_mlp import patch_fast_lora_mlp

        class Llama31MLP(nn.Module):
            def __init__(self):
                super().__init__()
                kwargs = {"device": "meta", "dtype": torch.bfloat16, "bias": False}
                self.gate_proj = nn.Linear(4096, 14336, **kwargs)
                self.up_proj = nn.Linear(4096, 14336, **kwargs)
                self.down_proj = nn.Linear(14336, 4096, **kwargs)
                self.act_fn = nn.SiLU()

            def forward(self, x):
                return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))

        class Model(nn.Module):
            def __init__(self):
                super().__init__()
                self.config = SimpleNamespace(hidden_act="silu")
                self.mlp = Llama31MLP()

            def forward(self, x):
                return self.mlp(x)

        model = Model()
        inject_adapter_in_model(
            LoraConfig(
                r=64,
                lora_alpha=128,
                lora_dropout=0.0,
                target_modules=["gate_proj", "up_proj", "down_proj"],
            ),
            model,
        )

        def saved_bytes():
            seen = set()
            total = 0

            def pack(tensor):
                nonlocal total
                key = (
                    tensor.untyped_storage()._cdata,
                    tensor.storage_offset(),
                    tuple(tensor.shape),
                    tuple(tensor.stride()),
                )
                if key not in seen:
                    seen.add(key)
                    total += tensor.numel() * tensor.element_size()
                return tensor

            x = torch.empty(
                1, 2048, 4096, device="meta", dtype=torch.bfloat16, requires_grad=True
            )
            with torch.autograd.graph.saved_tensors_hooks(pack, lambda tensor: tensor):
                model(x).sum().backward()
            model.zero_grad(set_to_none=True)
            return total

        peft_bytes = saved_bytes()
        assert patch_fast_lora_mlp(model) == 1
        fast_bytes = saved_bytes()
        saved = peft_bytes - fast_bytes
        n_f = 2048 * 14336 * 2

        # Shape-only meta execution makes the real 8B tensor sizes inspectable
        # without allocating the roughly 584 MiB of saved state. The fused path
        # must eliminate at least two, but fewer than three, [N, f] tensors.
        assert 2 * n_f <= saved < 3 * n_f
        print(
            "#837 Llama-3.1-8B saved bytes: "
            f"peft={peft_bytes} fast={fast_bytes} saved={saved}"
        )