Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""#839: the fast-LoRA single-projection kernel and its own tests (tracker #792).
``utils/fast_lora.py`` replaces peft's generic autograd for one projection at a
time: ``Y = X @ W^T + b + s * (X @ A^T) @ B^T``, with a hand-written backward
that saves only what the backward reads. Nothing here claims a speedup; the
tracker measures before any number is published, and the tests below assert
correctness only.
Asserted on CPU (no CUDA needed):
- ``torch.autograd.gradcheck`` in float64, with and without a base bias.
- fp32 forward, input-grad and adapter-grad parity against the unpatched peft
``lora.Linear`` at ``assert_close`` defaults. The parity is asserted at
tolerance, not bit-exactness: an earlier bit-exact readout was taken with
``lora_B`` at zero, which makes the adapter term vanish, so it measured far
less than it appeared to. With ``lora_B`` randomised the comparison is a real
one and the tolerance is what carries it.
- Delegation to peft's own forward: disabled adapters, merged adapters, a DoRA
variant, non-zero dropout without the opt-in, and the ``adapter_names`` kwarg.
- The dropout opt-in (``allow_dropout=True``) on the single-projection patcher:
forward and backward parity against an unpatched peft twin under an injected
fixed mask and under one seed for both paths, mask equality across a
``checkpoint`` recompute (both ``use_reentrant`` modes), eval-mode identity,
``p = 1`` delegation, and the real draw's keep-rate statistics.
- NF4 + dropout on CPU (bitsandbytes quantizes on CPU): the captured mask
replays into the unpatched path on the same module under checkpoint, which
is the arm that pins the ``ctx.qstart`` slice the backward rebuilds from.
- The patched forward reads the base weight at call time (the streaming
contract); patching is counted, idempotent and reversible.
- A 4-layer tiny Llama with 2 stream buffers: the patched streamed model's loss
and all 16 adapter grads match the resident twin (#331 shape).
- Saved-for-backward bytes are not worse than peft's graph at the unit shapes
(measured equal; the assertion is ``<=``).
- The kernel's own grad_fn is asserted on CPU across bias, rank, and dtypes
(fp32, bf16, fp16, mixed), ensuring non-fp32 or mixed-dtype execution paths
do not silently delegate.
Marked ``gpu`` (the protocol for a CUDA run; skipped in CI):
- NF4 forward/backward parity and kernel execution (asserting grad_fn is the
kernel's autograd node, not the delegate's) against unpatched
``lora.bnb.Linear4bit``. The tolerance is loose on purpose: the fused-vs-dequant
divergence #776 pins is part of whatever delta is measured there, and the honest
magnitude needs a card.
- A micro-benchmark at the Llama-3.1-8B ``o_proj`` shape. Report, do not
assert.
The gpu-marked tests need a card. Everything CPU above ran green on macOS with
torch 2.14.0 / peft 0.21.0 / Python 3.12 (the versions CI installs).
"""
from __future__ import annotations
import copy
import pytest
pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning")
def _requires_train_extra():
for mod in ("torch", "peft"):
pytest.importorskip(mod, reason=f"{mod} is only in the [train] extra")
def _spy(real_forward, calls, *args, **kwargs):
"""A stand-in forward that records that it was called, then behaves normally."""
calls.append("delegated")
return real_forward(*args, **kwargs)
def _randomise_lora_b(module):
"""Give ``lora_B`` a non-zero weight so the LoRA term is actually exercised.
peft initialises ``lora_B`` to zeros, which makes ``h @ B^T`` vanish: every
comparison then holds trivially, ``dA`` is compared zero against zero, and a
kernel that drops the dropout, ``disable_adapters``, ``merged`` or DoRA guard
still passes. Randomising B is what makes those tests mean anything.
"""
import torch
with torch.no_grad():
for _name, param in module.named_parameters():
if "lora_B" in _name:
torch.nn.init.normal_(param, std=0.5)
return module
def _make_adapted_linear(in_f=8, out_f=6, r=2, alpha=4, bias=True, dropout=0.0, dora=False):
"""A minimal peft-adapted module: one ``nn.Linear`` named ``linear``."""
import torch.nn as nn
from peft import LoraConfig, inject_adapter_in_model
class _Wrapper(nn.Module):
def __init__(self, layer):
super().__init__()
self.linear = layer
model = _Wrapper(nn.Linear(in_f, out_f, bias=bias))
config = LoraConfig(
r=r,
lora_alpha=alpha,
lora_dropout=dropout,
bias="none",
target_modules=["linear"],
use_dora=dora,
)
inject_adapter_in_model(config, model)
_randomise_lora_b(model)
return model, model.linear
def _saved_bytes(module, x):
"""Bytes held for backward during one forward+backward, deduped by storage."""
import torch
seen: dict[int, int] = {}
def pack(tensor):
storage = tensor.untyped_storage()
seen[storage.data_ptr()] = storage.nbytes()
return tensor
def unpack(tensor):
return tensor
with torch.autograd.graph.saved_tensors_hooks(pack, unpack):
out = module(x)
out.sum().backward()
return sum(seen.values())
class TestGradcheckFloat64:
"""#839: gradcheck in float64 on CPU, with and without a base bias."""
@pytest.mark.parametrize("use_bias", [False, True])
def test_gradcheck_wrt_x_a_b(self, use_bias):
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import _single_projection_function
fn = _single_projection_function()
torch.manual_seed(0)
n, in_f, out_f, r = 5, 8, 6, 2
w = torch.randn(out_f, in_f, dtype=torch.float64)
b = torch.randn(out_f, dtype=torch.float64) if use_bias else None
x = torch.randn(n, in_f, dtype=torch.float64, requires_grad=True)
a = torch.randn(r, in_f, dtype=torch.float64, requires_grad=True)
bb = torch.randn(out_f, r, dtype=torch.float64, requires_grad=True)
def f(x_, a_, b_):
return fn.apply(x_, w, b, a_, b_, 0.5, None, 0.0, False)
assert torch.autograd.gradcheck(f, (x, a, bb))
def test_double_backward_is_refused_rather_than_silently_wrong(self):
"""#792 marks ``backward`` once-differentiable, and the refusal must be real.
Not ``gradgradcheck``: that numerically differentiates and compares
Jacobians, and it reports a mismatch for this Function with or without
the decorator (measured on torch 2.14 both ways), so it cannot tell the
two apart. What a caller can actually observe is that a second
differentiation of the same graph raises instead of returning a number,
which is the behaviour ``@once_differentiable`` provides.
"""
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import _single_projection_function
fn = _single_projection_function()
torch.manual_seed(0)
n, in_f, out_f, r = 4, 6, 5, 2
w = torch.randn(out_f, in_f, dtype=torch.float64)
x = torch.randn(n, in_f, dtype=torch.float64, requires_grad=True)
a = torch.randn(r, in_f, dtype=torch.float64, requires_grad=True)
bb = torch.randn(out_f, r, dtype=torch.float64, requires_grad=True)
out = fn.apply(x, w, None, a, bb, 0.5, None, 0.0, False)
first = torch.autograd.grad(out.sum(), (x, a, bb), create_graph=True)
with pytest.raises(RuntimeError) as caught:
torch.autograd.grad(first[0].sum(), x)
# The refusal, rather than a silently wrong second derivative. torch
# reports it as the backward outputs carrying no graph, which is what
# ``once_differentiable`` does by running backward under no_grad.
assert "does not require grad" in str(caught.value) or "once_differentiable" in str(
caught.value
), f"unexpected refusal: {caught.value}"
def test_x_that_needs_no_grad_still_gets_correct_parameter_grads(self):
"""``needs_input_grad[0]`` false is the layer-0 case, and where the skip pays.
The dX GEMM is skipped there; dA and dB must still be right, because the
adapter is what is being trained.
"""
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear()
x = torch.randn(5, 8) # no requires_grad: this is the skip path
before = layer(x)
before.sum().backward()
expected = {
name: p.grad.detach().clone()
for name, p in layer.named_parameters()
if p.grad is not None
}
assert expected, "fixture produced no gradients at all"
for p in layer.parameters():
p.grad = None
assert patch_fast_lora_single_projection(model) == 1
after = layer(x)
torch.testing.assert_close(after, before)
after.sum().backward()
for name, p in layer.named_parameters():
if p.grad is None or name not in expected:
continue
torch.testing.assert_close(p.grad, expected[name])
class TestPeftParity:
"""#839: forward and backward parity against the unpatched peft forward."""
@pytest.mark.parametrize("bias", [True, False])
def test_fp32_forward_and_backward_match_unpatched_peft(self, bias):
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear(bias=bias)
x = torch.randn(5, 8, requires_grad=True)
ref = layer(x)
ref.sum().backward()
ref_x = x.grad.detach().clone()
ref_p = {
name: p.grad.detach().clone()
for name, p in layer.named_parameters()
if p.grad is not None
}
x.grad = None
for p in layer.parameters():
p.grad = None
assert patch_fast_lora_single_projection(model) == 1
out = layer(x)
out.sum().backward()
torch.testing.assert_close(out, ref)
torch.testing.assert_close(x.grad, ref_x)
for name, p in layer.named_parameters():
if name in ref_p:
torch.testing.assert_close(p.grad, ref_p[name], msg=name)
def test_n_dimensional_input_matches_unpatched_peft(self):
"""transformers calls projections with ``[B, S, H]``; the kernel has to
keep every leading dimension."""
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear()
x = torch.randn(2, 3, 8, requires_grad=True)
ref = layer(x)
ref.sum().backward()
ref_x = x.grad.detach().clone()
ref_p = {
name: p.grad.detach().clone()
for name, p in layer.named_parameters()
if p.grad is not None
}
x.grad = None
for p in layer.parameters():
p.grad = None
assert patch_fast_lora_single_projection(model) == 1
out = layer(x)
out.sum().backward()
torch.testing.assert_close(out, ref)
torch.testing.assert_close(x.grad, ref_x)
for name, p in layer.named_parameters():
if name in ref_p:
torch.testing.assert_close(p.grad, ref_p[name], msg=name)
class TestDelegation:
"""#839: peft's own forward must run whenever it owns the call."""
def test_disabled_adapters_delegate(self):
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear()
x = torch.randn(5, 8)
layer.enable_adapters(False)
expected = layer(x)
layer.enable_adapters(True)
assert patch_fast_lora_single_projection(model) == 1
layer.enable_adapters(False)
got = layer(x)
layer.enable_adapters(True)
# With adapters disabled peft returns the base projection; a kernel
# call would still add the LoRA term, so equality means delegation.
torch.testing.assert_close(got, expected)
torch.testing.assert_close(got, layer.base_layer(x))
def test_merged_adapters_delegate(self):
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear()
x = torch.randn(5, 8)
layer.merged_adapters = ["default"]
expected = layer(x)
layer.merged_adapters = []
assert patch_fast_lora_single_projection(model) == 1
layer.merged_adapters = ["default"]
got = layer(x)
layer.merged_adapters = []
torch.testing.assert_close(got, expected)
def test_fused_variant_delegates(self):
"""DoRA and friends produce tuple results the hand-written backward
does not model, so they must run through peft."""
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear(dora=True)
x = torch.randn(5, 8)
expected = layer(x)
assert patch_fast_lora_single_projection(model) == 1
got = layer(x)
torch.testing.assert_close(got, expected)
def test_dropout_delegates(self, monkeypatch):
"""Non-zero dropout is refused without the opt-in; a direct caller gets
peft's own (stochastic) path, not silently unregularised math.
The kernel must not run, and on CPU its draw follows the same RNG order
as ``nn.Dropout``, so seeded output equality alone cannot tell the two
paths apart: a recorder on the draw plus the grad_fn pin it.
"""
_requires_train_extra()
import torch
from soup_cli.utils import fast_lora
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear(dropout=0.5)
twin = copy.deepcopy(model)
draws = []
real = fast_lora._dropout_mask
def recorder(x_, p):
draws.append(True)
return real(x_, p)
monkeypatch.setattr(fast_lora, "_dropout_mask", recorder)
assert patch_fast_lora_single_projection(model) == 1
layer.train()
twin.linear.train()
x = torch.randn(5, 8)
torch.manual_seed(7)
got = layer(x)
torch.manual_seed(7)
expected = twin.linear(x)
torch.testing.assert_close(got, expected)
assert draws == [], (
"the patched forward drew a mask on a delegated call, so the kernel ran "
"without the opt-in"
)
assert type(got.grad_fn).__name__ != "_FastLoraSingleProjectionBackward", (
"off-by-default is not pinned: the kernel ran on a call that must delegate"
)
def test_adapter_names_kwarg_goes_to_peft(self):
"""The mixed-batch forward is peft's; the patched and unpatched calls
must produce the same outcome, value or exception."""
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear()
x = torch.randn(5, 8)
def run():
try:
return ("value", layer(x, adapter_names=["default"]))
except Exception as exc: # noqa: BLE001
return ("raise", type(exc), str(exc))
before = run()
assert patch_fast_lora_single_projection(model) == 1
after = run()
assert before[0] == after[0]
if before[0] == "raise":
assert before[1:] == after[1:]
else:
torch.testing.assert_close(before[1], after[1])
def test_fan_in_fan_out_base_delegates(self):
"""A ``Conv1D``-shaped base is peft's, not the kernel's.
``fan_in_fan_out`` stores the weight transposed, so the kernel's
``x @ W^T`` and ``dY @ W`` are computed against the wrong layout: on a
square projection that is silently wrong and on a non-square one it
raises. #839 scopes Conv1D out, so the patched forward must return
peft's own result.
"""
_requires_train_extra()
import pytest as _pytest
import torch
import torch.nn as nn
from peft import LoraConfig, inject_adapter_in_model
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
conv1d_cls = _pytest.importorskip(
"transformers.pytorch_utils", reason="the Conv1D fixture is transformers'"
).Conv1D
class _Wrapper(nn.Module):
def __init__(self, layer):
super().__init__()
self.proj = layer
def _build(in_f, out_f):
torch.manual_seed(0)
model = _Wrapper(conv1d_cls(in_f, out_f))
inject_adapter_in_model(
LoraConfig(
r=2, lora_alpha=4, lora_dropout=0.0, bias="none", target_modules=["proj"]
),
model,
)
layer = model.proj
assert getattr(layer, "fan_in_fan_out", False) is True
_randomise_lora_b(model)
return model, layer
# Both shapes, because they fail differently. A ``Conv1D`` stores
# ``[in, out]``, so a "non-square" one is ``Conv1D(out_f, in_f)``: on the
# square case the kernel's wrong layout happens to agree, and only the
# non-square case can tell "delegated" from "computed the other way
# round" - which is the one that raises rather than drifts.
for in_f, out_f in ((8, 8), (8, 6)):
model, layer = _build(out_f, in_f)
torch.manual_seed(5)
x = torch.randn(3, in_f, requires_grad=True)
torch.manual_seed(9)
expected = layer(x)
expected.sum().backward()
expected_grads = {
name: p.grad.detach().clone()
for name, p in layer.named_parameters()
if p.grad is not None
}
expected_x_grad = x.grad.detach().clone()
for p in layer.parameters():
p.grad = None
x.grad = None
assert patch_fast_lora_single_projection(model) == 1
torch.manual_seed(9)
got = layer(x)
torch.testing.assert_close(got, expected)
got.sum().backward()
# Backward too, where ``dY @ W`` is the term whose layout a
# transposed weight would get wrong.
torch.testing.assert_close(x.grad, expected_x_grad)
for name, p in layer.named_parameters():
if p.grad is None or name not in expected_grads:
continue
torch.testing.assert_close(p.grad, expected_grads[name])
def test_delegation_is_what_happens_not_merely_what_agrees(self):
"""Assert the guard ran, not just that the output happened to match.
Output alone cannot prove the guard ran: a square ``Conv1D`` gives the
right answer whether the kernel handled it or peft did. So the original
forward is captured and spied on, and the test asserts the delegate was
the thing that ran.
"""
_requires_train_extra()
import functools
import pytest as _pytest
import torch
import torch.nn as nn
from peft import LoraConfig, inject_adapter_in_model
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
conv1d_cls = _pytest.importorskip(
"transformers.pytorch_utils", reason="the Conv1D fixture is transformers'"
).Conv1D
class _Wrapper(nn.Module):
def __init__(self, layer):
super().__init__()
self.proj = layer
# peft wraps a ``Conv1D`` base in its own ``lora.Linear``, which this
# patch does match, and sets ``fan_in_fan_out`` for exactly that case.
# ``torch.nn.Conv1d`` is wrapped in ``lora.Conv1d``, a type the patch
# deliberately does not match, so it is not a case the guard sees.
cases = (("fan_in_fan_out", lambda: conv1d_cls(8, 8), True),)
for label, make_layer, expected_fifo in cases:
torch.manual_seed(0)
model = _Wrapper(make_layer())
inject_adapter_in_model(
LoraConfig(
r=2, lora_alpha=4, lora_dropout=0.0, bias="none", target_modules=["proj"]
),
model,
)
layer = model.proj
assert getattr(layer, "fan_in_fan_out", None) is expected_fifo, label
_randomise_lora_b(model)
# Capture-and-spy: the patch takes ``child.forward`` at patch time,
# so the spy becomes the delegate the guard falls back to.
real_forward = layer.forward
calls = []
layer.forward = functools.partial(_spy, real_forward, calls)
assert patch_fast_lora_single_projection(model) == 1
torch.manual_seed(5)
x = torch.randn(3, 8)
layer(x)
assert calls == ["delegated"], (
f"{label}: the patched forward should have delegated to peft's own "
f"path, but the kernel ran instead (delegations seen: {calls})"
)
def test_lora_bias_delegates(self):
"""``lora_bias=True`` puts a bias on ``lora_B`` the kernel does not carry."""
_requires_train_extra()
import torch
import torch.nn as nn
from peft import LoraConfig, inject_adapter_in_model
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
class _Wrapper(nn.Module):
def __init__(self, layer):
super().__init__()
self.linear = layer
torch.manual_seed(0)
model = _Wrapper(nn.Linear(8, 6))
inject_adapter_in_model(
LoraConfig(
r=2,
lora_alpha=4,
lora_dropout=0.0,
bias="none",
lora_bias=True,
target_modules=["linear"],
),
model,
)
layer = model.linear
assert layer.lora_B["default"].bias is not None, "fixture needs lora_bias=True to hold"
_randomise_lora_b(model)
with torch.no_grad():
layer.lora_B["default"].bias.copy_(torch.randn(6) * 0.1)
torch.manual_seed(5)
x = torch.randn(4, 8)
torch.manual_seed(9)
expected = layer(x)
assert patch_fast_lora_single_projection(model) == 1
torch.manual_seed(9)
got = layer(x)
# The bias would change the output if the kernel ran instead of peft.
torch.testing.assert_close(got, expected)
class TestPatchPlumbing:
"""#839: instance-level patching, counting, reversibility, call-time reads."""
def test_counts_only_adapted_linears_and_is_idempotent(self):
_requires_train_extra()
import torch.nn as nn
from soup_cli.utils.fast_lora import (
patch_fast_lora_single_projection,
unpatch_fast_lora_single_projection,
)
model, _layer = _make_adapted_linear()
model.extra = nn.Linear(8, 6)
assert patch_fast_lora_single_projection(model) == 1
assert patch_fast_lora_single_projection(model) == 0
assert unpatch_fast_lora_single_projection(model) == 1
assert unpatch_fast_lora_single_projection(model) == 0
assert patch_fast_lora_single_projection(model) == 1
def test_unpatch_restores_peft_forward(self):
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import (
patch_fast_lora_single_projection,
unpatch_fast_lora_single_projection,
)
torch.manual_seed(0)
model, layer = _make_adapted_linear()
x = torch.randn(5, 8)
expected = layer(x)
assert patch_fast_lora_single_projection(model) == 1
assert unpatch_fast_lora_single_projection(model) == 1
assert not hasattr(layer, "_soup_fast_lora_single_projection")
torch.testing.assert_close(layer(x), expected)
def test_patched_forward_reads_the_base_weight_at_call_time(self):
"""The streaming contract: ``functional_call`` substitutes weights only
for the duration of a call, so a reference captured at patch time is
worthless. Mutating the base weight after patching must change the
output."""
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear()
x = torch.randn(5, 8)
assert patch_fast_lora_single_projection(model) == 1
first = layer(x)
with torch.no_grad():
layer.base_layer.weight.mul_(2.0)
second = layer(x)
assert not torch.equal(first, second)
with torch.no_grad():
w = layer.base_layer.weight.detach().clone()
b = layer.base_layer.bias
a = layer.lora_A["default"].weight.detach()
bb = layer.lora_B["default"].weight.detach()
s = layer.scaling["default"]
expected = x @ w.T + b + s * ((x @ a.T) @ bb.T)
torch.testing.assert_close(second, expected)
class TestDropoutOptIn:
"""#839: dropout is modelled only when the caller asks for it.
The default stays peft's call (``TestDelegation`` pins that the kernel
stays out); with ``allow_dropout=True`` the kernel draws the mask itself
and has to be held to the same standards as the deterministic path: parity
two ways (under an injected fixed mask, and under one seed for both paths,
which on CPU share the draw's RNG order), a mask that survives a checkpoint
recompute in both checkpoint modes, eval identity, ``p = 1`` delegation,
and the draw's own statistics. NF4 + dropout covers the quant-state slice
on CPU.
"""
def test_shared_check_still_refuses_dropout_for_other_callers(self):
"""The MLP and QKV kernels never pass the opt-in, so the guard stays.
A non-zero dropout layer must hand them ``None`` (peft keeps those
calls); only the single-projection opt-in turns the state into one
carrying the dropout knobs.
"""
_requires_train_extra()
import torch
from soup_cli.utils import fast_lora
torch.manual_seed(0)
model, layer = _make_adapted_linear(dropout=0.5)
layer.train()
x = torch.randn(5, 8)
assert fast_lora._projection_state(layer, x) is None
state = fast_lora._projection_state(layer, x, allow_dropout=True)
assert state is not None
assert state.dropout_p == 0.5
assert state.dropout_training is True
def test_dropout_parity_with_peft_under_an_injected_mask(self, monkeypatch):
"""Forward and backward match peft when both sides apply one fixed mask."""
_requires_train_extra()
import torch
from soup_cli.utils import fast_lora
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
class _FixedMaskDropout(torch.nn.Module):
"""peft's dropout slot, pinned to one mask for a deterministic twin."""
def __init__(self, keep, p):
super().__init__()
self.register_buffer("keep", keep)
self.p = p
def forward(self, x):
return x * self.keep / (1.0 - self.p)
torch.manual_seed(0)
model, layer = _make_adapted_linear(dropout=0.5)
ref_model = copy.deepcopy(model)
ref_layer = ref_model.linear
keep = torch.rand(5, 8) > 0.5
ref_layer.lora_dropout["default"] = _FixedMaskDropout(keep, 0.5)
def fixed_mask(x_, p_):
return keep, x_ * keep / (1.0 - p_)
monkeypatch.setattr(fast_lora, "_dropout_mask", fixed_mask)
assert patch_fast_lora_single_projection(model, allow_dropout=True) == 1
layer.train()
ref_layer.train()
x = torch.randn(5, 8, requires_grad=True)
ref = ref_layer(x)
ref.sum().backward()
ref_x = x.grad.detach().clone()
ref_p = {
name: p.grad.detach().clone()
for name, p in ref_layer.named_parameters()
if p.grad is not None
}
x.grad = None
for p in layer.parameters():
p.grad = None
out = layer(x)
assert type(out.grad_fn).__name__ == "_FastLoraSingleProjectionBackward", (
f"the kernel did not run: grad_fn is {type(out.grad_fn).__name__}"
)
out.sum().backward()
torch.testing.assert_close(out, ref)
torch.testing.assert_close(x.grad, ref_x)
for name, p in layer.named_parameters():
if name in ref_p:
torch.testing.assert_close(p.grad, ref_p[name], msg=name)
@pytest.mark.parametrize("use_reentrant", [False, True])
def test_mask_matches_under_checkpoint_recompute(self, monkeypatch, use_reentrant):
"""The recompute draws the same mask, and the grads land where the plain
run puts them.
``checkpoint`` runs the forward twice; with ``preserve_rng_state`` the
second pass starts from the same RNG state, so the recorded masks must
be identical. The saved-mask design is what makes this the equality to
assert: a recompute that re-drew fresh dropout would show up as a
different mask and different grads.
"""
_requires_train_extra()
import torch
from soup_cli.utils import fast_lora
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear(dropout=0.5)
draws = []
real = fast_lora._dropout_mask
def recorder(x_, p):
keep_, x_d = real(x_, p)
draws.append(keep_)
return keep_, x_d
monkeypatch.setattr(fast_lora, "_dropout_mask", recorder)
assert patch_fast_lora_single_projection(model, allow_dropout=True) == 1
layer.train()
x = torch.randn(5, 8, requires_grad=True)
torch.manual_seed(7)
ref = layer(x)
ref.sum().backward()
ref_x = x.grad.detach().clone()
ref_p = {
name: p.grad.detach().clone()
for name, p in layer.named_parameters()
if p.grad is not None
}
assert len(draws) == 1, f"the plain run should draw once, drew {len(draws)}"
ref_keep = draws[0]
x.grad = None
for p in layer.parameters():
p.grad = None
draws.clear()
torch.manual_seed(7)
out = torch.utils.checkpoint.checkpoint(
lambda t: layer(t), x, use_reentrant=use_reentrant
)
out.sum().backward()
assert len(draws) == 2, (
f"checkpoint should run the forward twice (original + recompute); saw {len(draws)}"
)
assert torch.equal(draws[0], ref_keep), "the same seed drew a different mask"
assert torch.equal(draws[0], draws[1]), "the recompute drew a different mask"
torch.testing.assert_close(out, ref)
torch.testing.assert_close(x.grad, ref_x)
for name, p in layer.named_parameters():
if name in ref_p:
torch.testing.assert_close(p.grad, ref_p[name], msg=name)
def test_eval_mode_skips_the_mask(self, monkeypatch):
"""nn.Dropout is identity in eval, and a validation pass must match.
The kernel reads the dropout module's training flag; in eval no mask
may be drawn and the output must equal the unpatched twin's.
"""
_requires_train_extra()
import torch
from soup_cli.utils import fast_lora
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear(dropout=0.5)
twin = copy.deepcopy(model)
draws = []
real = fast_lora._dropout_mask
def recorder(x_, p):
draws.append(True)
return real(x_, p)
monkeypatch.setattr(fast_lora, "_dropout_mask", recorder)
assert patch_fast_lora_single_projection(model, allow_dropout=True) == 1
layer.eval()
twin.linear.eval()
x = torch.randn(5, 8)
got = layer(x)
expected = twin.linear(x)
assert draws == [], "a mask was drawn in eval"
torch.testing.assert_close(got, expected)
def test_the_real_draw_keeps_at_about_one_minus_p(self):
"""The draw itself: boolean mask, one byte per element, keep rate ~1-p.
The parity arm injects its mask, so this is the only place a wrong
probability or an inverted keep/drop convention can be caught.
"""
_requires_train_extra()
import torch
from soup_cli.utils import fast_lora
torch.manual_seed(0)
x = torch.randn(1000, 1000)
keep, x_d = fast_lora._dropout_mask(x, 0.25)
assert keep.shape == x.shape
assert keep.dtype == torch.bool
assert keep.element_size() == 1
rate = keep.float().mean().item()
assert abs(rate - 0.75) < 0.003, f"keep rate {rate:.4f} is not ~0.75"
torch.testing.assert_close(x_d, x * keep / 0.75)
def test_dropout_parity_with_peft_under_the_same_seed(self):
"""Seeded, both paths draw the same masks on CPU, so the twin comparison
needs no injected mask. This is the stronger witness: it goes red if the
kernel's draw diverges from ``nn.Dropout``'s RNG order."""
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear(dropout=0.5)
twin = copy.deepcopy(model)
assert patch_fast_lora_single_projection(model, allow_dropout=True) == 1
layer.train()
twin.linear.train()
x = torch.randn(5, 8, requires_grad=True)
torch.manual_seed(7)
got = layer(x)
assert type(got.grad_fn).__name__ == "_FastLoraSingleProjectionBackward", (
f"the kernel did not run: grad_fn is {type(got.grad_fn).__name__}"
)
got.sum().backward()
got_x = x.grad.detach().clone()
got_p = {
name: p.grad.detach().clone()
for name, p in layer.named_parameters()
if p.grad is not None
}
x.grad = None
for p in layer.parameters():
p.grad = None
torch.manual_seed(7)
expected = twin.linear(x)
expected.sum().backward()
torch.testing.assert_close(got, expected)
torch.testing.assert_close(got_x, x.grad)
for name, p in twin.linear.named_parameters():
if name in got_p:
torch.testing.assert_close(p.grad, got_p[name], msg=name)
def test_lora_dropout_p_one_delegates_and_stays_finite(self, monkeypatch):
"""``lora_dropout=1.0`` delegates: ``1 / (1 - p)`` is undefined at one
and peft's native dropout handles the edge. The recorder proves the
kernel stayed out and nothing produced NaN."""
_requires_train_extra()
import torch
from soup_cli.utils import fast_lora
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear(dropout=1.0)
twin = copy.deepcopy(model)
draws = []
real = fast_lora._dropout_mask
def recorder(x_, p):
draws.append(True)
return real(x_, p)
monkeypatch.setattr(fast_lora, "_dropout_mask", recorder)
assert patch_fast_lora_single_projection(model, allow_dropout=True) == 1
layer.train()
twin.linear.train()
x = torch.randn(5, 8)
torch.manual_seed(7)
got = layer(x)
torch.manual_seed(7)
expected = twin.linear(x)
assert draws == [], "the kernel drew for p = 1.0 instead of delegating"
assert bool(torch.isfinite(got).all()), "p = 1.0 produced non-finite output"
torch.testing.assert_close(got, expected)
def test_nf4_dropout_under_checkpoint_rebuilds_the_quant_state(self, monkeypatch):
"""NF4 + dropout: the backward's ``saved[ctx.qstart:]`` slice must skip
the mask before rebuilding the ``QuantState``.
bitsandbytes quantizes on CPU (0.50.2), so this arm needs no card: the
captured mask replays into the unpatched path on the same module under
the same checkpoint mode. ``ctx.qstart = 5``, which survives every other
test, turns this one red.
"""
_requires_train_extra()
pytest.importorskip("bitsandbytes", reason="4-bit tests need bitsandbytes")
import bitsandbytes as bnb
import torch
import torch.nn as nn
from peft import LoraConfig, inject_adapter_in_model
from soup_cli.utils import fast_lora
from soup_cli.utils.fast_lora import (
patch_fast_lora_single_projection,
unpatch_fast_lora_single_projection,
)
class _Wrapper(nn.Module):
def __init__(self, layer):
super().__init__()
self.linear = layer
class _FixedMaskDropout(torch.nn.Module):
"""peft's dropout slot, pinned to one mask for the unpatched twin."""
def __init__(self, keep, p):
super().__init__()
self.register_buffer("keep", keep)
self.p = p
def forward(self, x):
return x * self.keep / (1.0 - self.p)
torch.manual_seed(0)
model = _Wrapper(
bnb.nn.Linear4bit(16, 16, bias=False, compute_dtype=torch.float32, quant_type="nf4")
)
inject_adapter_in_model(
LoraConfig(r=4, lora_alpha=8, lora_dropout=0.5, bias="none", target_modules=["linear"]),
model,
)
_randomise_lora_b(model)
layer = model.linear
# Quantize explicitly: bnb's lazy path wants a ``.cuda()`` move, and
# this arm must exercise the quant branch, not the dense fallback.
packed, quant_state = bnb.functional.quantize_4bit(layer.weight.data, quant_type="nf4")
layer.weight.data = packed
layer.weight.quant_state = quant_state
draws = []
real = fast_lora._dropout_mask
def recorder(x_, p):
keep_, x_d = real(x_, p)
draws.append(keep_)
return keep_, x_d
monkeypatch.setattr(fast_lora, "_dropout_mask", recorder)
assert patch_fast_lora_single_projection(model, allow_dropout=True) == 1
layer.train()
x = torch.randn(5, 16, requires_grad=True)
torch.manual_seed(7)
out = torch.utils.checkpoint.checkpoint(lambda t: layer(t), x, use_reentrant=False)
out.sum().backward()
assert len(draws) == 2, f"checkpoint should draw twice; saw {len(draws)}"
assert torch.equal(draws[0], draws[1]), "the recompute drew a different mask"
got_x = x.grad.detach().clone()
got_p = {
name: p.grad.detach().clone()
for name, p in layer.named_parameters()
if p.grad is not None
}
assert got_p, "no parameter grads reached the backward"
# The unpatched path on the same module, fed the captured mask.
assert unpatch_fast_lora_single_projection(model) == 1
layer.lora_dropout["default"] = _FixedMaskDropout(draws[0], 0.5)
x.grad = None
for p in layer.parameters():
p.grad = None
ref = layer(x)
ref.sum().backward()
# Loose on purpose, like the CUDA NF4 arm: the fused-vs-dequant
# divergence (#776) is part of this delta.
atol = 1e-2
rtol = 1e-2
torch.testing.assert_close(out, ref, rtol=rtol, atol=atol)
torch.testing.assert_close(got_x, x.grad, rtol=rtol, atol=atol)
for name, p in layer.named_parameters():
if name in got_p:
torch.testing.assert_close(p.grad, got_p[name], rtol=rtol, atol=atol, msg=name)
class TestSavedBytes:
"""#839: bytes held for backward, reported; no figure asserted in advance."""
def test_not_worse_than_pefts_graph(self):
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear(in_f=8, out_f=6, r=2, alpha=4)
plain = _saved_bytes(layer, torch.randn(64, 8, requires_grad=True))
for p in layer.parameters():
p.grad = None
assert patch_fast_lora_single_projection(model) == 1
fast = _saved_bytes(layer, torch.randn(64, 8, requires_grad=True))
assert 0 < fast <= plain
print(f"#839 saved bytes: peft={plain} fast={fast}")
class TestStreamedModel:
"""#331 shape: more decoder layers than stream buffers, CPU."""
def test_fast_lora_matches_resident_on_a_streamed_tiny_llama(self, tmp_path):
_requires_train_extra()
for mod in ("transformers", "safetensors"):
pytest.importorskip(mod, reason=f"{mod} is only in the [train] extra")
import torch
from peft import LoraConfig, TaskType, get_peft_model
from safetensors.torch import save_file
from transformers import AutoModelForCausalLM, LlamaConfig, LlamaForCausalLM
from soup_cli.utils.fast_lora import (
patch_fast_lora_single_projection,
unpatch_fast_lora_single_projection,
)
from soup_cli.utils.layer_shard import shard_checkpoint
from soup_cli.utils.layer_stream_runtime import build_streamed_model
def lora_config():
return LoraConfig(
r=4,
lora_alpha=8,
lora_dropout=0.0,
bias="none",
target_modules=["q_proj", "v_proj"],
task_type=TaskType.CAUSAL_LM,
)
torch.manual_seed(7)
config = LlamaConfig(
vocab_size=64,
hidden_size=32,
intermediate_size=64,
num_hidden_layers=4,
num_attention_heads=4,
num_key_value_heads=2,
tie_word_embeddings=False,
max_position_embeddings=128,
)
weights = tmp_path / "model"
weights.mkdir(parents=True, exist_ok=True)
save_file(
{k: v.contiguous() for k, v in LlamaForCausalLM(config).state_dict().items()},
str(weights / "model.safetensors"),
)
config.save_pretrained(str(weights))
shards = str(tmp_path / "shards")
index = shard_checkpoint(str(weights), shards, dtype="float32", arch="llama")
model, _runtime = build_streamed_model(
model_id=str(weights),
shard_dir=shards,
index=index,
lora_config=lora_config(),
device="cpu",
dtype="float32",
buffers=2,
pin=False,
seed=3,
)
# q_proj + v_proj on each of the 4 layers, all single projections.
assert patch_fast_lora_single_projection(model) == 8
# peft leaves lora_B at zeros, so the resident twin would be compared
# LoRA-term-against-LoRA-term with the term equal to zero. Randomise
# before the copy so both twins carry a live adapter.
_randomise_lora_b(model)
resident = get_peft_model(
AutoModelForCausalLM.from_pretrained(str(weights), dtype=torch.float32),
lora_config(),
)
src = dict(model.named_parameters())
dst = dict(resident.named_parameters())
copied = 0
for name, param in src.items():
if "lora_" not in name or param.is_meta:
continue
assert name in dst, f"resident model has no parameter {name!r}"
with torch.no_grad():
dst[name].copy_(param)
copied += 1
assert copied == 16
# The resident twin is a matchable target, but it is left UNPATCHED so
# the comparison below is kernel against peft rather than kernel against
# kernel. The kernel is the streamed side on purpose: #331 is a claim
# about the kernel's saved references surviving the streamed pools, so
# the kernel is what has to run there. patch+unpatch is the control that
# this twin was patchable in the first place.
assert patch_fast_lora_single_projection(resident) == 8
assert unpatch_fast_lora_single_projection(resident) == 8
torch.manual_seed(11)
input_ids = torch.randint(0, 64, (1, 8))
labels = torch.randint(0, 64, (1, 8))
streamed_loss = model(input_ids=input_ids, labels=labels).loss
streamed_loss.backward()
resident_loss = resident(input_ids=input_ids, labels=labels).loss
resident_loss.backward()
torch.testing.assert_close(streamed_loss, resident_loss, rtol=1e-5, atol=1e-5)
streamed_grads = {
n: p.grad for n, p in model.named_parameters() if "lora_" in n and p.grad is not None
}
resident_grads = {
n: p.grad for n, p in resident.named_parameters() if "lora_" in n and p.grad is not None
}
assert len(streamed_grads) == 16
assert len(resident_grads) == 16
matched = 0
for name, grad in sorted(streamed_grads.items()):
target = next((g for n, g in resident_grads.items() if n.endswith(name)), None)
assert target is not None, f"no resident grad for {name!r}"
# Every layer is checked, not just the first buffer's worth.
torch.testing.assert_close(grad, target, rtol=1e-4, atol=1e-5, msg=name)
matched += 1
assert matched == 16
@pytest.mark.gpu
class TestNf4Parity:
"""NF4: parity against unpatched ``lora.bnb.Linear4bit``. Needs a card."""
def test_nf4_forward_and_backward_match_unpatched_peft(self):
_requires_train_extra()
pytest.importorskip("bitsandbytes", reason="4-bit tests need bitsandbytes")
import bitsandbytes as bnb
import torch
import torch.nn as nn
from peft import LoraConfig, inject_adapter_in_model
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
class _Wrapper(nn.Module):
def __init__(self, layer):
super().__init__()
self.linear = layer
torch.manual_seed(0)
base = bnb.nn.Linear4bit(32, 32, bias=False, compute_dtype=torch.bfloat16, quant_type="nf4")
model = _Wrapper(base).to("cuda")
inject_adapter_in_model(
LoraConfig(r=4, lora_alpha=8, lora_dropout=0.0, bias="none", target_modules=["linear"]),
model,
)
# As elsewhere: zeros in lora_B would make the parity comparison vacuous.
_randomise_lora_b(model)
layer = model.linear
x = torch.randn(5, 32, device="cuda", dtype=torch.bfloat16, requires_grad=True)
ref = layer(x)
assert type(ref.grad_fn).__name__ != "_FastLoraSingleProjectionBackward", (
"the fixture must not start patched, or the assertion below proves nothing"
)
ref.sum().backward()
ref_x = x.grad.detach().clone()
ref_p = {
name: p.grad.detach().clone()
for name, p in layer.named_parameters()
if p.grad is not None
}
x.grad = None
for p in layer.parameters():
p.grad = None
assert patch_fast_lora_single_projection(model) == 1
out = layer(x)
assert type(out.grad_fn).__name__ == "_FastLoraSingleProjectionBackward", (
f"the kernel did not run on NF4: grad_fn is {type(out.grad_fn).__name__}, "
"which is the delegate's node"
)
out.sum().backward()
# Loose on purpose: the fused-vs-dequant divergence pinned by #776 is
# part of this delta, and its magnitude on this shape is unmeasured
# without a card. The PR asks for the readout from the first CUDA run.
atol = 1e-2
rtol = 1e-2
torch.testing.assert_close(out, ref, rtol=rtol, atol=atol)
torch.testing.assert_close(x.grad, ref_x, rtol=rtol, atol=atol)
for name, p in layer.named_parameters():
if name in ref_p:
torch.testing.assert_close(p.grad, ref_p[name], rtol=rtol, atol=atol, msg=name)
@pytest.mark.gpu
class TestMicroBenchmark:
"""Llama-3.1-8B ``o_proj`` shape: report, do not assert (tracker #792)."""
def _time_fwd_bwd(self, layer, x, iters=5):
import torch
def step():
out = layer(x)
out.sum().backward()
step()
torch.cuda.synchronize()
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start.record()
for _ in range(iters):
step()
end.record()
torch.cuda.synchronize()
return start.elapsed_time(end) / iters
def test_report_only(self):
_requires_train_extra()
pytest.importorskip("bitsandbytes", reason="NF4 timing needs bitsandbytes")
import bitsandbytes as bnb
import torch
import torch.nn as nn
from peft import LoraConfig, inject_adapter_in_model
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
class _Wrapper(nn.Module):
def __init__(self, layer):
super().__init__()
self.linear = layer
config = LoraConfig(
r=16, lora_alpha=32, lora_dropout=0.0, bias="none", target_modules=["linear"]
)
x = torch.randn(1, 4096, 4096, device="cuda", dtype=torch.bfloat16)
dense = _Wrapper(nn.Linear(4096, 4096, bias=False, dtype=torch.bfloat16)).to("cuda")
inject_adapter_in_model(config, dense)
plain_ms = self._time_fwd_bwd(dense.linear, x)
assert patch_fast_lora_single_projection(dense) == 1
fast_ms = self._time_fwd_bwd(dense.linear, x)
assert torch.isfinite(dense.linear(x)).all()
print(f"#839 o_proj bf16 dense: peft={plain_ms:.2f}ms fast={fast_ms:.2f}ms")
nf4 = _Wrapper(
bnb.nn.Linear4bit(
4096, 4096, bias=False, compute_dtype=torch.bfloat16, quant_type="nf4"
)
).to("cuda")
inject_adapter_in_model(config, nf4)
nf4_plain_ms = self._time_fwd_bwd(nf4.linear, x)
assert patch_fast_lora_single_projection(nf4) == 1
nf4_fast_ms = self._time_fwd_bwd(nf4.linear, x)
assert torch.isfinite(nf4.linear(x)).all()
print(f"#839 o_proj bf16 nf4: peft={nf4_plain_ms:.2f}ms fast={nf4_fast_ms:.2f}ms")
class TestTheFastPathIsActuallyTaken:
"""#839 review: nothing pinned that the kernel runs at all.
Every other parity test is a "matches unpatched peft" assertion, which is
trivially true when the patched forward delegates to peft. The suite could
not tell "correct" from "absent": make ``_fast_lora_single_forward``
delegate unconditionally and the whole file stays green.
These two tests assert the kernel was TAKEN, from the two sides that only
the kernel can satisfy.
"""
@pytest.mark.parametrize("bias", [True, False], ids=["bias", "no-bias"])
@pytest.mark.parametrize("shape", [(4, 8), (2, 3, 8)], ids=["2d", "3d"]) # [N, in] / [B, S, in]
@pytest.mark.parametrize(
"dtype", ["fp32", "bf16", "fp16", "mixed"], ids=["fp32", "bf16", "fp16", "mixed"]
)
def test_the_output_carries_the_kernels_own_grad_fn(self, bias, shape, dtype):
"""The kernel's Function node, not peft's, is what built this output.
``grad_fn`` is the positive edge: the name is produced by the autograd
Function that ran, so it cannot hold when the kernel was skipped. The
unpatched forward yields ``AddBackward0`` for the same input.
Parametrised over bias and rank because the pin has to see a narrowed
delegation guard, not only an unconditional one. Measured at review
time: with both arms at defaults (bias, 2-D), three mutations at
``fast_lora.py:278`` all SURVIVED the suite - delegating on a 3-D
activation, delegating for a bias-less base, and both together. The
combination is what the kernel exists for: ``_flatten``'s own docstring
names ``[B, S, in]`` as the transformers rank, and every ``o_proj`` call
in a Llama is 3-D on a bias-less base. The parity tests cannot catch it
because agreement is trivially true under delegation.
Additionally, parametrised over dtype (fp32, bf16, fp16, mixed) because
delegating on non-fp32, fp16, or mixed-dtype (fp32 adapters on a bf16 base,
as left by get_peft_model with autocast_adapter_dtype=True) would otherwise
survive both parity and gradcheck.
"""
_requires_train_extra()
import torch
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
torch_dtype = {
"fp32": torch.float32,
"bf16": torch.bfloat16,
"fp16": torch.float16,
"mixed": torch.bfloat16,
}[dtype]
model, layer = _make_adapted_linear(bias=bias)
model.to(torch_dtype)
if dtype == "mixed": # what get_peft_model leaves by default: fp32 adapters on a bf16 base
for name, param in model.named_parameters():
if "lora_" in name:
param.data = param.data.float()
x = torch.randn(*shape, dtype=torch_dtype, requires_grad=True)
before = layer(x)
assert type(before.grad_fn).__name__ != "_FastLoraSingleProjectionBackward", (
"the fixture must not start patched, or the assertion below proves nothing"
)
assert patch_fast_lora_single_projection(model) == 1
after = layer(x)
assert type(after.grad_fn).__name__ == "_FastLoraSingleProjectionBackward", (
f"the kernel did not run: grad_fn is {type(after.grad_fn).__name__}, "
"which is the delegate's node"
)
# and the delegate really was not the thing that ran
assert layer.forward.__func__.__name__ == "_fast_lora_single_forward"
def test_the_delegate_is_not_called_when_the_kernel_applies(self):
"""The spy inverts: an eligible layer must reach the kernel with no delegation.
``test_delegation_is_what_happens_not_merely_what_agrees`` proves the
guard fires for the cases that must bail. This is its other half for the
happy path, and it is the arm that dies when the kernel is made to
delegate unconditionally.
"""
_requires_train_extra()
import functools
import torch
from soup_cli.utils.fast_lora import patch_fast_lora_single_projection
torch.manual_seed(0)
model, layer = _make_adapted_linear()
real_forward = layer.forward
calls = []
layer.forward = functools.partial(_spy, real_forward, calls)
assert patch_fast_lora_single_projection(model) == 1
x = torch.randn(4, 8, requires_grad=True)
out = layer(x)
out.sum().backward()
assert calls == [], (
"the patched forward delegated on an eligible layer, so the kernel "
f"did not run (delegations seen: {calls})"
)
# both directions of the backward are live, so this is a real step
assert x.grad is not None
assert layer.lora_A["default"].weight.grad is not None
assert layer.lora_B["default"].weight.grad is not None