Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

25.4 KB
Raw Blame
"""#792 — ``training.fast_lora``: the switch that puts the Fast-LoRA kernels
(``utils/fast_lora.py``, ``fast_lora_mlp.py``, ``fast_lora_qkv.py``) on a run.

What is checked here runs on CPU in float32 and float64: the settings the flag
refuses, the wiring on the resident and the streamed SFT paths, and parity with
peft's own modules through the switch. The bf16 / fp16, NF4 and streamed-CUDA
rows of the issue are ``gpu`` tests at the end of the file.
"""

from __future__ import annotations

import json
import types

import pytest
from rich.console import Console

torch = pytest.importorskip("torch")
pytest.importorskip("peft")
pytest.importorskip("transformers")

CONSOLE = Console()
TARGETS = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
ON = types.SimpleNamespace(fast_lora=True)
OFF = types.SimpleNamespace(fast_lora=False)


def _cfg(task="sft", data=None, backend="transformers", modality="text", **training):
    from soup_cli.config.schema import SoupConfig

    lora = {"dropout": 0.0, **training.pop("lora", {})}
    return SoupConfig(
        base="test/model",
        task=task,
        backend=backend,
        modality=modality,
        data={"train": "tests/fixtures/sample_train.jsonl", **(data or {})},
        training={"fast_lora": True, "lora": lora, **training},
    )


def _refusal(**kwargs) -> str:
    from pydantic import ValidationError

    with pytest.raises(ValidationError) as excinfo:
        _cfg(**kwargs)
    return str(excinfo.value)


# ─── The flag and what it refuses ────────────────────────────────────────────


class TestTheFlag:
    def test_it_is_off_unless_asked_for(self):
        from soup_cli.config.schema import TrainingConfig

        assert TrainingConfig().fast_lora is False

    @pytest.mark.parametrize("quantization", ["none", "4bit"])
    def test_the_first_slice_loads(self, quantization):
        assert _cfg(quantization=quantization).training.fast_lora is True

    def test_the_default_lora_dropout_is_refused_with_the_way_out(self):
        from pydantic import ValidationError

        from soup_cli.config.schema import SoupConfig

        with pytest.raises(ValidationError) as excinfo:
            SoupConfig(
                base="test/model",
                data={"train": "tests/fixtures/sample_train.jsonl"},
                training={"fast_lora": True},
            )
        message = str(excinfo.value)
        assert "training.fast_lora needs lora.dropout: 0" in message
        assert "Soup's default is 0.05" in message

    def test_another_backend_is_refused(self):
        assert "training.fast_lora needs backend='transformers'" in _refusal(backend="unsloth")

    def test_another_modality_is_refused(self):
        assert "training.fast_lora supports modality='text' only" in _refusal(
            modality="vision", data={"format": "llava"}
        )

    @pytest.mark.parametrize("task", ["dpo", "orpo", "kto"])
    def test_another_task_is_refused(self, task):
        data = {"format": "kto" if task == "kto" else "dpo"}
        assert "training.fast_lora supports task='sft' only" in _refusal(task=task, data=data)

    def test_no_lora_is_refused(self):
        assert "training.fast_lora needs a LoRA; lora.r=0" in _refusal(
            quantization="none", lora={"r": 0}
        )

    @pytest.mark.parametrize("adapter", ["use_dora", "use_vera"])
    def test_another_adapter_is_refused(self, adapter):
        assert f"training.fast_lora cannot be combined with lora.{adapter}" in _refusal(
            quantization="none", lora={adapter: True}
        )

    def test_target_parameters_are_refused(self):
        assert "cannot be combined with lora.target_parameters" in _refusal(
            quantization="none", lora={"target_parameters": ["mlp.experts.gate_up_proj"]}
        )

    @pytest.mark.parametrize("quantization", ["8bit", "gptq"])
    def test_another_quantization_is_refused(self, quantization):
        assert "training.fast_lora supports quantization 'none' or '4bit'" in _refusal(
            quantization=quantization
        )

    def test_use_liger_is_refused(self):
        assert "training.fast_lora cannot be combined with training.use_liger" in _refusal(
            use_liger=True
        )

    @pytest.mark.parametrize(
        ("field", "value"),
        [("unfrozen_parameters", ["model.layers.0.mlp"]), ("lisa_enabled", True)],
        ids=["spectrum", "lisa"],
    )
    def test_a_full_fine_tuning_mode_is_refused(self, field, value):
        """They train the model's own weights and skip the LoRA setup the
        kernels hook into, so the flag would be ignored. The default rank 64,
        as a user who sets only these would have it."""
        from pydantic import ValidationError

        from soup_cli.config.schema import SoupConfig

        kwargs = {
            "base": "test/model",
            "data": {"train": "tests/fixtures/sample_train.jsonl"},
        }
        training = {"quantization": "none", "lora": {"dropout": 0.0}, field: value}
        SoupConfig(**kwargs, training=training)  # the premise: it loads without the flag
        with pytest.raises(ValidationError) as excinfo:
            SoupConfig(**kwargs, training={**training, "fast_lora": True})
        assert f"training.fast_lora cannot be combined with training.{field}" in str(excinfo.value)

    def test_fp8_training_is_refused(self):
        assert (
            "training.fast_lora cannot be combined with training.quantization_aware='fp8'"
            in _refusal(quantization="none", quantization_aware="fp8")
        )


# ─── The switch on a model ───────────────────────────────────────────────────


def _model(seed=0, dtype=None, hidden=32):
    from peft import LoraConfig, get_peft_model
    from transformers import LlamaConfig, LlamaForCausalLM

    torch.manual_seed(seed)
    config = LlamaConfig(
        vocab_size=64,
        hidden_size=hidden,
        intermediate_size=2 * hidden,
        num_hidden_layers=2,
        num_attention_heads=4,
        num_key_value_heads=4,
        max_position_embeddings=64,
    )
    lora = LoraConfig(r=4, lora_alpha=8, lora_dropout=0.0, bias="none", target_modules=TARGETS)
    model = get_peft_model(LlamaForCausalLM(config), lora)
    with torch.no_grad():  # a trained adapter: lora_B at zero would hide a wrong path
        for name, param in model.named_parameters():
            if "lora_B" in name:
                param.normal_(std=0.1)
    return model.to(dtype) if dtype is not None else model


def _pair(**kwargs):
    """The same model twice; the second has the switch on."""
    from soup_cli.utils.peft_wiring import apply_fast_lora_patches

    plain, fast = _model(**kwargs), _model(**kwargs)
    counts = apply_fast_lora_patches(fast, ON)
    return plain, fast, counts


def _ids(seed=1):
    torch.manual_seed(seed)
    return torch.randint(0, 64, (2, 7))


def _lora_grads(model):
    return {n: p.grad.detach().clone() for n, p in model.named_parameters() if "lora_" in n}


class TestTheSwitch:
    def test_off_leaves_peft_alone(self):
        from soup_cli.utils.peft_wiring import apply_fast_lora_patches

        model = _model()
        assert apply_fast_lora_patches(model, OFF) is None
        assert not any("forward" in vars(m) for m in model.modules())

    def test_on_puts_every_kernel_on_its_modules(self):
        _, fast, counts = _pair()

        # Two layers: q/k/v and the MLP are each one group per layer, and o_proj
        # is the only projection left for the single-projection kernel.
        assert counts == {"q/k/v": 2, "MLP": 2, "single projection": 2}
        layer = fast.base_model.model.model.layers[0]
        x = torch.randn(1, 3, 32, requires_grad=True)
        assert "Fast" in type(layer.self_attn.o_proj(x).grad_fn).__name__
        assert "Fast" in type(layer.mlp(x).grad_fn).__name__

    def test_on_with_nothing_to_patch_is_refused(self):
        from soup_cli.utils.peft_wiring import apply_fast_lora_patches

        with pytest.raises(ValueError, match="no LoRA module was patched"):
            apply_fast_lora_patches(torch.nn.Sequential(torch.nn.Linear(4, 4)), ON)

    def test_a_conv1d_only_model_is_refused(self):
        """GPT-2's Conv1D LoRA (``fan_in_fan_out``) is patched but delegates to
        peft on every call: nothing the kernels can run, so the switch refuses
        and takes its patches off."""
        from peft import LoraConfig, get_peft_model
        from transformers import GPT2Config, GPT2LMHeadModel

        from soup_cli.utils.peft_wiring import apply_fast_lora_patches

        torch.manual_seed(0)
        config = GPT2Config(vocab_size=64, n_positions=32, n_embd=16, n_layer=2, n_head=2)
        lora = LoraConfig(r=4, lora_alpha=8, lora_dropout=0.0, target_modules=["c_attn"])
        model = get_peft_model(GPT2LMHeadModel(config), lora)

        with pytest.raises(ValueError, match="no LoRA module was patched"):
            apply_fast_lora_patches(model, ON)
        assert not any("forward" in vars(m) for m in model.modules())

    def test_the_setup_line_names_each_kernel(self):
        from soup_cli.utils.peft_wiring import fast_lora_summary

        line = fast_lora_summary({"q/k/v": 2, "MLP": 2, "single projection": 2})
        assert line == "Fast-LoRA kernels on: 2 q/k/v, 2 MLP, 2 single projection"


class TestParityThroughTheSwitch:
    """fp32 against peft's own modules; forward and backward reported apart."""

    def test_forward(self):
        plain, fast, _ = _pair()
        ids = _ids()

        torch.testing.assert_close(fast(input_ids=ids).logits, plain(input_ids=ids).logits)

    def test_backward(self):
        plain, fast, _ = _pair()
        ids = _ids()
        for model in (plain, fast):
            model(input_ids=ids, labels=ids).loss.backward()

        plain_grads, fast_grads = _lora_grads(plain), _lora_grads(fast)
        assert plain_grads.keys() == fast_grads.keys() and len(plain_grads) == 28
        for name in plain_grads:
            torch.testing.assert_close(fast_grads[name], plain_grads[name], msg=name)

    def test_gradients_are_the_same_with_non_reentrant_checkpointing(self):
        _, fast, _ = _pair()
        _, checkpointed, _ = _pair()
        checkpointed.base_model.model.gradient_checkpointing_enable(
            gradient_checkpointing_kwargs={"use_reentrant": False}
        )
        checkpointed.train()
        fast.train()
        ids = _ids()
        for model in (fast, checkpointed):
            model(input_ids=ids, labels=ids).loss.backward()

        for name, grad in _lora_grads(fast).items():
            torch.testing.assert_close(_lora_grads(checkpointed)[name], grad, msg=name)

    def test_with_the_adapter_disabled_the_output_is_the_base_models(self):
        plain, fast, _ = _pair()
        ids = _ids()
        with fast.disable_adapter():
            fast_base = fast(input_ids=ids).logits
        with plain.disable_adapter():
            plain_base = plain(input_ids=ids).logits

        torch.testing.assert_close(fast_base, plain_base)
        assert not torch.allclose(fast(input_ids=ids).logits, fast_base)  # the premise

    @pytest.mark.parametrize("path", ["o_proj", "mlp", "self_attn"])
    def test_gradcheck_in_float64(self, path):
        """Per kernel, w.r.t. its input and its LoRA A and B, with the base
        weight frozen. (Not through the whole model: Llama's RMSNorm runs in
        float32 whatever the model's dtype, so peft fails that check too.)"""
        from torch.func import functional_call

        _, fast, _ = _pair(dtype=torch.float64, hidden=16)
        decoder = fast.base_model.model.model
        layer = decoder.layers[0]
        module = layer.self_attn.o_proj if path == "o_proj" else getattr(layer, path)
        lora = {
            n: p.detach().clone().requires_grad_(True)
            for n, p in module.named_parameters()
            if "lora_" in n
        }
        names = list(lora)
        torch.manual_seed(2)
        x = torch.randn(1, 3, 16, dtype=torch.float64, requires_grad=True)
        kwargs = {}
        if path == "self_attn":
            positions = torch.arange(3).unsqueeze(0)
            kwargs = {
                "position_embeddings": decoder.rotary_emb(x, positions),
                "attention_mask": None,
            }

        def run(inputs, *params):
            out = functional_call(module, dict(zip(names, params)), (inputs,), kwargs)
            return out[0] if isinstance(out, tuple) else out

        assert len(names) >= 2
        assert torch.autograd.gradcheck(run, (x, *lora.values()), eps=1e-6, atol=1e-5)

    def test_fifty_steps_lose_the_same_to_three_decimals(self):
        """One batch, fifty AdamW steps at lr 1e-3. (At lr 1e-2 the two runs
        stay bit-identical for 33 steps and then part, as any two float32 runs
        whose last bits differ do under AdamW at that rate.)"""
        plain, fast, _ = _pair()
        torch.manual_seed(3)
        ids = torch.randint(0, 64, (2, 9))
        losses = {}
        for name, model in (("plain", plain), ("fast", fast)):
            params = [p for p in model.parameters() if p.requires_grad]
            optimizer = torch.optim.AdamW(params, lr=1e-3)
            losses[name] = []
            for _ in range(50):
                loss = model(input_ids=ids, labels=ids).loss
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()
                losses[name].append(loss.item())

        # Equal to 3 decimals: never 1e-3 apart (rounding each to 3 places would
        # call 4.1935 and 4.1934 different).
        gap = max(abs(f - p) for f, p in zip(losses["fast"], losses["plain"]))
        CONSOLE.print(json.dumps({"fifty_step_max_loss_gap": gap}))
        assert gap < 1e-3
        assert losses["plain"][-1] < losses["plain"][0] - 0.1  # it trained


# ─── Through the real SFT setup ──────────────────────────────────────────────


def _sft_rows(n=4):
    return [
        {
            "messages": [
                {"role": "user", "content": "hello"},
                {"role": "assistant", "content": "world"},
            ]
        }
    ] * n


def _resident_wrapper(tmp_path, monkeypatch, *, fast_lora):
    import yaml

    from soup_cli.config.loader import load_config_from_string
    from soup_cli.trainer.sft import SFTTrainerWrapper
    from tests._tiny_hf_models import tiny_model_dir

    monkeypatch.chdir(tmp_path)
    cfg = load_config_from_string(
        yaml.safe_dump(
            {
                "base": tiny_model_dir("llama"),
                "task": "sft",
                "backend": "transformers",
                "data": {"train": "unused.jsonl", "max_length": 64, "chat_template": "chatml"},
                "training": {
                    "epochs": 1,
                    "batch_size": 2,
                    "quantization": "none",
                    "fast_lora": fast_lora,
                    "lora": {"r": 4, "alpha": 8, "dropout": 0.0, "target_modules": TARGETS},
                },
                "output": str(tmp_path / "out"),
            }
        )
    )
    wrapper = SFTTrainerWrapper(cfg, device="cpu")
    wrapper.setup({"train": _sft_rows()})
    return wrapper


def _kernel_nodes(model):
    return {type(m).__name__ for m in model.modules() if "forward" in vars(m)}


class TestTheResidentSetup:
    def test_the_switch_reaches_the_model_the_trainer_trains(self, tmp_path, monkeypatch, capsys):
        wrapper = _resident_wrapper(tmp_path, monkeypatch, fast_lora=True)

        modules = dict(wrapper.trainer.model.named_modules())
        patched = [n for n, m in modules.items() if "forward" in vars(m)]
        assert any(
            hasattr(m, "_soup_fast_lora_qkv") for n, m in modules.items() if n.endswith("self_attn")
        )
        assert any(n.endswith("mlp") for n in patched)
        assert any(n.endswith("o_proj") for n in patched)
        assert "Fast-LoRA kernels on:" in capsys.readouterr().out

    def test_without_the_switch_nothing_is_patched(self, tmp_path, monkeypatch):
        wrapper = _resident_wrapper(tmp_path, monkeypatch, fast_lora=False)

        assert not any("forward" in vars(m) for m in wrapper.trainer.model.modules())


class TestAFailedPatchReleasesTheStreamedRuntime:
    """The switch runs after the runtime is built and before training's
    cleanup context starts: a failure there closes the runtime itself."""

    def _setup(self, tmp_path, monkeypatch):
        """The real streamed setup; returns, for each close, whether a runtime
        was there to close."""
        from soup_cli.trainer.stream_setup import StreamingSetupMixin
        from tests.test_v07204 import _build_streamed_wrapper

        closes = []
        real_close = StreamingSetupMixin._close_stream_runtime

        def counting_close(self):
            closes.append(getattr(self, "_stream_runtime", None) is not None)
            return real_close(self)

        monkeypatch.setattr(StreamingSetupMixin, "_close_stream_runtime", counting_close)
        with pytest.raises(Exception) as excinfo:
            _build_streamed_wrapper(
                tmp_path,
                monkeypatch,
                task="sft",
                n_layers=2,
                device="cpu",
                tie=False,
                fast_lora=True,
                lora={"r": 4, "alpha": 8, "dropout": 0.0, "target_modules": TARGETS},
            )
        return closes, excinfo.value

    def test_nothing_patched(self, tmp_path, monkeypatch):
        from soup_cli.utils import fast_lora, fast_lora_mlp, fast_lora_qkv

        monkeypatch.setattr(fast_lora_qkv, "patch_fast_lora_qkv", lambda model: 0)
        monkeypatch.setattr(fast_lora_mlp, "patch_fast_lora_mlp", lambda model: 0)
        monkeypatch.setattr(fast_lora, "patch_fast_lora_single_projection", lambda model: 0)
        monkeypatch.setattr(fast_lora, "eligible_single_projections", lambda model: 0)

        closes, error = self._setup(tmp_path, monkeypatch)

        assert "no LoRA module was patched" in str(error)
        assert closes == [True]

    def test_a_patcher_that_raises(self, tmp_path, monkeypatch):
        from soup_cli.utils import fast_lora_qkv

        def boom(model):
            raise RuntimeError("patcher failed")

        monkeypatch.setattr(fast_lora_qkv, "patch_fast_lora_qkv", boom)

        closes, error = self._setup(tmp_path, monkeypatch)

        assert str(error) == "patcher failed"
        assert closes == [True]


class TestTheStreamedSetup:
    """A streamed tiny Llama with more layers than ``stream_buffers``: the LoRA
    gradients on every layer equal those of the same run without the switch."""

    def _grads(self, tmp_path, monkeypatch, fast_lora):
        from tests.test_v07204 import _build_streamed_wrapper

        wrapper, _, _ = _build_streamed_wrapper(
            tmp_path,
            monkeypatch,
            task="sft",
            n_layers=4,
            device="cpu",
            tie=False,
            stream_buffers=2,
            fast_lora=fast_lora,
            lora={"r": 4, "alpha": 8, "dropout": 0.0, "target_modules": TARGETS},
        )
        try:
            with torch.no_grad():
                for name, param in wrapper.model.named_parameters():
                    if "lora_B" in name:
                        torch.manual_seed(abs(hash(name)) % 2**31)
                        param.normal_(std=0.1)
            ids = torch.tensor([[1, 5, 9, 13, 2, 7, 11]])
            out = wrapper.model(input_ids=ids, labels=ids)
            out.loss.backward()
            patched = sum("forward" in vars(m) for m in wrapper.model.modules())
            return _lora_grads(wrapper.model), patched
        finally:
            wrapper._close_stream_runtime()

    def test_every_layers_gradients_match_without_the_switch(self, tmp_path, monkeypatch):
        plain, plain_patched = self._grads(tmp_path / "plain", monkeypatch, False)
        fast, fast_patched = self._grads(tmp_path / "fast", monkeypatch, True)

        assert plain_patched < fast_patched  # the premise: the kernels are on
        layers = {name.split(".layers.")[1].split(".")[0] for name in fast if ".layers." in name}
        assert layers == {"0", "1", "2", "3"}
        assert plain.keys() == fast.keys()
        for name in plain:
            assert torch.count_nonzero(plain[name]) > 0, name
            torch.testing.assert_close(fast[name], plain[name], msg=name)


# ─── Rows that need CUDA (not run on CPU CI) ─────────────────────────────────


def _half_error(dtype):
    """Max-abs error of the fast and the peft path against float64, per output."""
    plain, fast, _ = _pair()
    reference = _model().double()
    ids = _ids().cuda()
    plain, fast, reference = plain.cuda().to(dtype), fast.cuda().to(dtype), reference.cuda()
    want = reference(input_ids=ids).logits
    plain_err = (plain(input_ids=ids).logits.double() - want).abs().max().item()
    fast_err = (fast(input_ids=ids).logits.double() - want).abs().max().item()
    return plain_err, fast_err


@pytest.mark.gpu
@pytest.mark.parametrize("dtype", ["bfloat16", "float16"])
def test_half_precision_forward_error_is_within_twice_peft(dtype):
    plain_err, fast_err = _half_error(getattr(torch, dtype))
    CONSOLE.print(json.dumps({"dtype": dtype, "peft_max_abs": plain_err, "fast_max_abs": fast_err}))
    assert fast_err <= 2 * plain_err


@pytest.mark.gpu
@pytest.mark.parametrize("dtype", ["bfloat16", "float16"])
def test_half_precision_backward_error_is_within_twice_peft(dtype):
    dtype = getattr(torch, dtype)
    plain, fast, _ = _pair()
    reference = _model().double().cuda()
    ids = _ids().cuda()
    reference(input_ids=ids, labels=ids).loss.backward()
    want = _lora_grads(reference)
    errors = {}
    for name, model in (("peft", plain), ("fast", fast)):
        model = model.cuda().to(dtype)
        model(input_ids=ids, labels=ids).loss.backward()
        got = _lora_grads(model)
        errors[name] = max((got[n].double() - want[n]).abs().max().item() for n in want)
    CONSOLE.print(json.dumps({"dtype": str(dtype), **errors}))
    assert errors["fast"] <= 2 * errors["peft"]


@pytest.mark.gpu
def test_nf4_matches_peft_over_the_same_linear4bit(tmp_path, monkeypatch):
    """Reference: unpatched peft over the same NF4 model, never bf16."""
    plain = _resident_wrapper_4bit(tmp_path / "plain", monkeypatch, fast_lora=False)
    fast = _resident_wrapper_4bit(tmp_path / "fast", monkeypatch, fast_lora=True)
    ids = torch.tensor([[1, 5, 9, 13, 2, 7, 11]], device="cuda")
    for model in (plain, fast):
        model(input_ids=ids, labels=ids).loss.backward()
    torch.testing.assert_close(fast(input_ids=ids).logits, plain(input_ids=ids).logits)
    for name, grad in _lora_grads(plain).items():
        torch.testing.assert_close(_lora_grads(fast)[name], grad, msg=name)


def _resident_wrapper_4bit(tmp_path, monkeypatch, *, fast_lora):
    import yaml

    from soup_cli.config.loader import load_config_from_string
    from soup_cli.trainer.sft import SFTTrainerWrapper
    from tests._tiny_hf_models import tiny_model_dir

    tmp_path.mkdir(parents=True, exist_ok=True)
    monkeypatch.chdir(tmp_path)
    cfg = load_config_from_string(
        yaml.safe_dump(
            {
                "base": tiny_model_dir("llama"),
                "task": "sft",
                "data": {"train": "unused.jsonl", "max_length": 64, "chat_template": "chatml"},
                "training": {
                    "epochs": 1,
                    "batch_size": 2,
                    "quantization": "4bit",
                    "fast_lora": fast_lora,
                    "lora": {"r": 4, "alpha": 8, "dropout": 0.0, "target_modules": TARGETS},
                },
                "output": str(tmp_path / "out"),
            }
        )
    )
    wrapper = SFTTrainerWrapper(cfg, device="cuda")
    wrapper.setup({"train": _sft_rows()})
    torch.manual_seed(4)
    with torch.no_grad():
        for name, param in wrapper.model.named_parameters():
            if "lora_B" in name:
                param.normal_(std=0.1)
    return wrapper.model


@pytest.mark.gpu
@pytest.mark.parametrize("quantization", ["none", "4bit"])
def test_streamed_on_cuda_every_layer_matches(tmp_path, monkeypatch, quantization):
    """bf16 and NF4 on the GPU, streamed on both sides (#776: streamed NF4 and
    resident NF4 are not bit-exact on sm_120)."""
    from tests.test_v07204 import _build_streamed_wrapper

    grads = {}
    for name, flag in (("plain", False), ("fast", True)):
        wrapper, _, _ = _build_streamed_wrapper(
            tmp_path / name,
            monkeypatch,
            task="sft",
            n_layers=4,
            device="cuda",
            tie=False,
            stream_buffers=2,
            quantization=quantization,
            fast_lora=flag,
            lora={"r": 4, "alpha": 8, "dropout": 0.0, "target_modules": TARGETS},
        )
        try:
            torch.manual_seed(5)
            with torch.no_grad():
                for pname, param in wrapper.model.named_parameters():
                    if "lora_B" in pname:
                        param.normal_(std=0.1)
            ids = torch.tensor([[1, 5, 9, 13, 2, 7, 11]], device="cuda")
            wrapper.model(input_ids=ids, labels=ids).loss.backward()
            grads[name] = _lora_grads(wrapper.model)
        finally:
            wrapper._close_stream_runtime()
    for pname, grad in grads["plain"].items():
        torch.testing.assert_close(grads["fast"][pname], grad, msg=pname)