Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Fast-LoRA single-projection autograd (correctness half of tracker #792).
Scope: the single-projection path from #839, which is ``o_proj`` plus any
adapted ``nn.Linear`` / ``bnb.nn.Linear4bit`` the MLP (#837) and QKV (#838)
paths do not cover. The kernel computes the same math as peft's generic
autograd with fewer intermediates; **no speedup is claimed here** -- the
tracker measures before any number is published.
What this module establishes for the sibling paths:
- instance-level forward patching (``types.MethodType``, as the streamed-layer
dequant patch already does in ``layer_stream_runtime``);
- ``save_for_backward`` for every tensor the backward reads (#331: streamed
pools rewrite their slots, so a saved reference is only safe when it went
through the pool-aware save);
- NF4: the packed weight and its quant-state tensors are saved and the weight
is re-dequantised in the backward; the dequantised matrix is never saved;
- delegation back to the unpatched peft forward whenever ``disable_adapters``,
``merged`` or ``adapter_names`` is set, the adapter is a fused variant
(DoRA/VeRA and friends produce tuple results the hand-written backward does
not model), dropout is non-zero without the explicit opt-in, or the call
shape is not a plain ``(x)``. Dropout is modelled only when a caller asks
for it (``patch_fast_lora_single_projection(..., allow_dropout=True)``): the
keep-mask is drawn and saved (one byte per element) and ``X_d`` is rebuilt
in the backward.
The patch covers only the single-projection path. Which kernel a projection
gets when the QKV or MLP patcher also wants it is written once, in the comment
above ``_GROUP_PATCH_OWNER_MARKER`` below.
The patched forward reads the base weight at call time on purpose: streamed
layers substitute weights via ``functional_call`` only for the duration of a
call, and a reference captured at patch time under streaming is a meta
placeholder.
"""
from __future__ import annotations
import logging
from typing import Any, NamedTuple
logger = logging.getLogger(__name__)
_PATCH_MARKER = "_soup_fast_lora_single_projection"
_ORIGINAL_FORWARD_MARKER = "_soup_fast_lora_original_forward"
_HAD_INSTANCE_FORWARD_MARKER = "_soup_fast_lora_had_instance_forward"
# When two patchers want one projection, the group kernel takes it, whichever
# was patched first: shared-X Q/K/V owns q/k/v and the fused SwiGLU MLP owns
# gate/up/down. A group patcher releases this kernel from the projections it
# takes and marks them with _GROUP_PATCH_OWNER_MARKER, which this one skips.
_GROUP_PATCH_OWNER_MARKER = "_soup_fast_lora_group_owner"
_FORWARD_OWNER_MARKER = "_soup_fast_lora_forward_owner"
_OWNER = "single"
__all__ = [
"patch_fast_lora_single_projection",
"unpatch_fast_lora_single_projection",
]
_FUNCTION: Any = None
def _as_dtype(tensor: Any, dtype: Any) -> Any:
"""Cast only when the dtype differs, so the uniform case stays op-for-op."""
return tensor if tensor.dtype == dtype else tensor.to(dtype)
def _scaled_lora_add(out: Any, hidden: Any, lora_b: Any, scaling: float) -> Any:
"""Add a LoRA update with PEFT's promote, add, then cast semantics."""
import torch
lora_term = torch.matmul(hidden, lora_b.t())
promoted = torch.promote_types(out.dtype, lora_term.dtype)
return torch.add(
out.to(promoted), lora_term.to(promoted), alpha=float(scaling)
).to(out.dtype)
def _flatten(tensor: Any) -> Any:
"""Collapse the leading dimensions into one, for weight-shaped gradients.
``dB`` and ``dA`` sum over every leading dimension regardless of the input
rank (``[N, in]`` for the unit tests, ``[B, S, in]`` from transformers).
``reshape`` is a view for the contiguous activation layouts that reach a
projection; a non-contiguous grad pays a copy, which is the correct trade
against a silent shape assumption.
"""
return tensor.reshape(-1, tensor.shape[-1])
def _dropout_mask(x: Any, p: float) -> tuple[Any, Any]:
"""Draw LoRA dropout's keep-mask and apply it to the adapter input.
Returns ``(keep, x_d)`` where ``keep`` is boolean, one byte per element.
The mask is what the backward saves: ``X_d`` is rebuilt from it, so the
full-precision dropped copy peft's ``lora_A`` retains does not have to be
kept (#839). The draw uses the global torch RNG, which is what lets a
checkpoint recompute reproduce the same mask when ``preserve_rng_state``
is on.
"""
import torch
keep = torch.empty_like(x, dtype=torch.bool).bernoulli_(1.0 - p)
return keep, x * keep / (1.0 - p)
def _quant_state_parts(quant_state: Any) -> tuple[list[Any], dict[str, Any]]:
"""Split a ``QuantState`` into ``save_for_backward`` tensors and metadata.
Roles are recorded in save order so the backward can rebuild the state
without guessing which optional tensor is which (``offset`` and the nested
``state2`` are only present for some checkpoints).
"""
roles = ["absmax", "code"]
tensors = [quant_state.absmax, quant_state.code]
meta: dict[str, Any] = {
"shape": tuple(quant_state.shape),
"dtype": quant_state.dtype,
"blocksize": quant_state.blocksize,
"quant_type": quant_state.quant_type,
}
if quant_state.offset is not None:
roles.append("offset")
tensors.append(quant_state.offset)
if quant_state.state2 is not None:
roles += ["state2_absmax", "state2_code"]
tensors += [quant_state.state2.absmax, quant_state.state2.code]
meta["state2_blocksize"] = quant_state.state2.blocksize
if quant_state.state2.offset is not None:
roles.append("state2_offset")
tensors.append(quant_state.state2.offset)
meta["roles"] = roles
return tensors, meta
def _rebuild_quant_state(meta: dict[str, Any], tensors: list[Any]) -> Any:
"""Reassemble a ``QuantState`` from saved tensors (``layer_stream_runtime``
precedent, including the nested ``state2`` dtype)."""
import torch
from bitsandbytes.functional import QuantState
parts = dict(zip(meta["roles"], tensors))
state2 = None
if "state2_absmax" in parts:
state2 = QuantState(
absmax=parts["state2_absmax"],
code=parts["state2_code"],
blocksize=meta["state2_blocksize"],
dtype=torch.float32,
offset=parts.get("state2_offset"),
)
return QuantState(
absmax=parts["absmax"],
shape=torch.Size(meta["shape"]),
dtype=meta["dtype"],
blocksize=meta["blocksize"],
code=parts["code"],
quant_type=meta["quant_type"],
offset=parts.get("offset"),
state2=state2,
)
class _ProjectionState(NamedTuple):
weight: Any
bias: Any
lora_a: Any
lora_b: Any
scaling: float
qmeta: dict[str, Any] | None
qparts: list[Any]
compute_dtype: Any
dropout_p: float
dropout_training: bool
def _projection_state(
proj: Any, x: Any, *, allow_dropout: bool = False, allow_unadapted: bool = False
) -> _ProjectionState | None:
"""Return one PEFT/base projection state or None when PEFT must run.
This is shared by the single, MLP and QKV kernels so the seven delegation
guards, streamed QuantState repair and bitsandbytes compute-dtype handling
cannot drift between patchers.
``allow_dropout`` is the single-projection kernel's explicit opt-in: with
it, a non-zero ``lora_dropout`` returns a state carrying ``dropout_p`` and
``dropout_training`` instead of delegating; ``p >= 1`` still delegates,
since the mask rescale is undefined at 1. The MLP and QKV kernels never
pass it, so for them the shared check keeps refusing exactly as before.
"""
lora_a_map = getattr(proj, "lora_A", None)
if lora_a_map is not None and not _is_supported_lora_projection(proj):
return None
base = proj.get_base_layer() if hasattr(proj, "get_base_layer") else proj
if not hasattr(base, "weight"):
return None
adapter = None
dropout_p = 0.0
dropout_training = False
if lora_a_map is None:
if not allow_unadapted:
return None
else:
if getattr(proj, "disable_adapters", False) or getattr(proj, "merged", False):
return None
active = getattr(proj, "active_adapters", None) or []
if len(active) != 1:
return None
adapter = active[0]
if adapter not in proj.lora_A:
if not allow_unadapted:
return None
adapter = None
else:
if adapter in getattr(proj, "lora_variant", {}):
return None
# Dropout changes the adapter computation and mask lifetime. The
# single-projection kernel models the mask when it is asked to
# (``allow_dropout``); every other caller keeps handing dropout
# calls back to peft.
dropout = proj.lora_dropout[adapter]
dropout_p = float(getattr(dropout, "p", 0.0))
dropout_training = bool(getattr(dropout, "training", False))
if dropout_p != 0.0 and not allow_dropout:
return None
if allow_dropout and dropout_p >= 1.0:
# 1 / (1 - p) is undefined at p = 1; peft's native dropout
# handles that edge, so the opt-in delegates it too.
return None
# Conv1D-style/transposed storage needs the PEFT fan-in/fan-out
# path rather than the Linear weight orientation used below.
if getattr(proj, "fan_in_fan_out", False):
return None
# ``lora_bias`` adds another trained term that these Functions do
# not accept or differentiate.
if getattr(proj.lora_B[adapter], "bias", None) is not None:
return None
weight = base.weight
qstate = getattr(weight, "quant_state", None)
if qstate is None and getattr(base, "quant_state", None) is not None:
# Streamed Params4bit views can temporarily carry the state on the
# module rather than on the view itself. Repair the same way as the
# single-projection path before any sibling reads it.
try:
weight.quant_state = base.quant_state
except AttributeError:
pass
qstate = getattr(weight, "quant_state", None) or base.quant_state
# An unadapted sibling may still be an 8-bit/custom projection. Without a
# 4-bit QuantState it is not a dense floating weight and cannot use F.linear.
if qstate is None and not weight.is_floating_point():
return None
compute_dtype = None
qmeta = None
qparts: list[Any] = []
if qstate is not None:
if not getattr(base, "compute_type_is_set", True) and hasattr(base, "set_compute_type"):
base.set_compute_type(x)
base.compute_type_is_set = True
compute_dtype = getattr(base, "compute_dtype", None)
qparts, qmeta = _quant_state_parts(qstate)
qmeta = dict(qmeta)
qmeta["_count"] = len(qparts)
if adapter is None:
lora_a = x.new_empty(0)
lora_b = x.new_empty(0)
scaling = 0.0
else:
lora_a = proj.lora_A[adapter].weight
lora_b = proj.lora_B[adapter].weight
scaling = float(proj.scaling[adapter])
return _ProjectionState(
weight,
getattr(base, "bias", None),
lora_a,
lora_b,
scaling,
qmeta,
qparts,
compute_dtype,
dropout_p,
dropout_training,
)
def _supported_lora_projection_types() -> tuple[type, ...]:
"""Return PEFT projection types whose weight contracts the kernels model."""
from peft.tuners.lora import Linear as LoraLinear
supported: tuple[type, ...] = (LoraLinear,)
try:
from peft.tuners.lora import bnb as lora_bnb
supported = (LoraLinear, lora_bnb.Linear4bit)
except Exception: # noqa: BLE001 - bitsandbytes absence is a normal install
pass
return supported
def _is_supported_lora_projection(proj: Any) -> bool:
"""Exclude PEFT 8-bit/custom layers whose storage math differs."""
return isinstance(proj, _supported_lora_projection_types())
def _dense_weight(
weight: Any,
qmeta: dict[str, Any] | None,
qparts: list[Any],
dtype: Any,
) -> Any:
"""Return a dense view for one base projection without retaining it."""
if qmeta is None:
return _as_dtype(weight, dtype)
from bitsandbytes.functional import dequantize_4bit
state = _rebuild_quant_state(qmeta, qparts)
return dequantize_4bit(weight, state).to(dtype)
def _single_projection_function() -> Any:
"""Build (once per process) the ``autograd.Function`` class.
The class has to be built inside a function because its base class is
``torch.autograd.Function`` and torch must stay out of the light CLI import
path (tests/test_cli_startup_is_light.py).
"""
global _FUNCTION
if _FUNCTION is not None:
return _FUNCTION
import torch
class _FastLoraSingleProjection(torch.autograd.Function):
"""``Y = X @ W^T + b + s * (X_d @ A^T) @ B^T``, hand-written backward.
``X_d`` is ``X`` unless dropout is modelled (the opt-in path); there it
is ``X * keep / (1 - p)`` with ``keep`` the saved boolean mask, and the
base term above still reads the undropped ``X``.
``W`` and ``b`` are frozen during LoRA training, so the backward returns
``None`` for both, and the input gradient still carries the base term
``dY @ W``. ``H`` (``[N, r]``) is saved rather than recomputed: the
backward needs it twice (``dA`` and ``dB``) and it is rank-sized; the
saved-bytes test reports the actual delta against peft's graph.
NF4: ``weight`` is the packed tensor and ``quant_state`` carries the
per-block tensors. The forward dequantises for the base matmul only;
the backward rebuilds the ``QuantState`` from the saved tensors and
dequantises again for ``dX``.
"""
@staticmethod
def forward(
ctx, x, weight, bias, lora_a, lora_b, scaling, quant_state, dropout_p, dropout_training
):
import torch.nn.functional as functional
keep = None
x_lora = x
if dropout_p > 0.0 and dropout_training:
keep, x_lora = _dropout_mask(x, dropout_p)
h = functional.linear(_as_dtype(x_lora, lora_a.dtype), lora_a) # [N, r]
if quant_state is None:
out = functional.linear(x, weight, bias)
ctx.save_for_backward(
x, weight, lora_a, lora_b, h, *((keep,) if keep is not None else ())
)
ctx.qmeta = None
else:
from bitsandbytes.functional import dequantize_4bit
# Transient on purpose: under streaming this dense matrix would
# alias a pool slot, so it must not outlive the call.
dense = dequantize_4bit(weight, quant_state).to(x.dtype)
out = functional.linear(
x, dense, _as_dtype(bias, x.dtype) if bias is not None else None
)
del dense
qs_tensors, qs_meta = _quant_state_parts(quant_state)
ctx.save_for_backward(
x,
weight,
lora_a,
lora_b,
h,
*((keep,) if keep is not None else ()),
*qs_tensors,
)
ctx.qmeta = qs_meta
ctx.scaling = float(scaling)
ctx.dropout_p = float(dropout_p)
ctx.dropout_active = keep is not None
ctx.qstart = 5 + (1 if keep is not None else 0)
# ``torch.addmm(out, h, lora_b.t(), alpha=s)`` is the 2-D form of
# this sum. matmul + add keeps one code path that also covers the
# ``[B, S, H]`` inputs transformers hands to real projections, and
# the add's ``alpha`` still folds the scaling into the term.
out = _scaled_lora_add(out, h, lora_b, ctx.scaling)
return out
@staticmethod
@torch.autograd.function.once_differentiable
def backward(ctx, grad_out):
scaling = ctx.scaling
# One access only: under a non-reentrant checkpoint the saved
# tensors go through an unpack hook that refuses a second read.
saved = ctx.saved_tensors
x, weight, lora_a, lora_b, h = saved[:5]
keep = saved[5] if ctx.dropout_active else None
# dA contracts against the DROPPED input; it is rebuilt from the
# saved mask rather than retaining peft's full-precision copy.
x_lora = x if keep is None else x * keep / (1.0 - ctx.dropout_p)
# dB sums over every leading dimension, so flatten it; dB and dA
# are weight-shaped regardless of the input's rank.
grad_b = _as_dtype(_flatten(grad_out), h.dtype).t() @ _flatten(h) * scaling
# dH keeps the input's leading dimensions; dA = dH^T @ X_d.
grad_h = torch.matmul(_as_dtype(grad_out, lora_b.dtype), lora_b) * scaling
grad_a = _flatten(grad_h).t() @ _as_dtype(_flatten(x_lora), grad_h.dtype)
grad_x = None
if ctx.needs_input_grad[0]:
if ctx.qmeta is None:
dense = weight
else:
from bitsandbytes.functional import dequantize_4bit
quant_state = _rebuild_quant_state(ctx.qmeta, list(saved[ctx.qstart :]))
dense = dequantize_4bit(weight, quant_state).to(x.dtype)
# dX = dY @ W, plus the LoRA term accumulated into the same
# result: one add instead of separate dH @ A and sum
# allocations. Skipped entirely when X needs no grad, which
# recovers the 2x the unskipped dX GEMM costs on a layer-0
# projection.
grad_x = torch.matmul(grad_out, _as_dtype(dense, grad_out.dtype))
grad_lora = torch.matmul(
_as_dtype(grad_h, grad_x.dtype), _as_dtype(lora_a, grad_x.dtype)
)
# The LoRA branch read the dropped input, so its gradient takes
# the mask and the 1/(1-p) rescale on the way back to X.
if keep is not None:
grad_lora = grad_lora * keep / (1.0 - ctx.dropout_p)
grad_x = torch.add(grad_x, grad_lora)
return grad_x, None, None, grad_a, grad_b, None, None, None, None
_FUNCTION = _FastLoraSingleProjection
return _FUNCTION
def _make_patched_forward(original_forward: Any, *, allow_dropout: bool = False) -> Any:
"""Wrap a bound peft forward; delegate whenever peft's own path must run.
``allow_dropout`` is the explicit opt-in for this kernel: with it, a
non-zero LoRA dropout is modelled (the mask is drawn, saved and applied in
the backward) instead of delegating to peft.
"""
fast = _single_projection_function()
def _fast_lora_single_forward(self, x, *args, **kwargs):
if args or kwargs:
return original_forward(x, *args, **kwargs)
state = _projection_state(self, x, allow_dropout=allow_dropout)
if state is None:
return original_forward(x)
input_dtype = x.dtype
work_x = x
if state.compute_dtype is not None and work_x.dtype != state.compute_dtype:
work_x = work_x.to(state.compute_dtype)
out = fast.apply(
work_x,
state.weight,
state.bias,
state.lora_a,
state.lora_b,
state.scaling,
None if state.qmeta is None else _rebuild_quant_state(state.qmeta, state.qparts),
state.dropout_p,
state.dropout_training,
)
return out if work_x is x else out.to(input_dtype)
setattr(_fast_lora_single_forward, _FORWARD_OWNER_MARKER, _OWNER)
return _fast_lora_single_forward
def patch_fast_lora_single_projection(model: Any, *, allow_dropout: bool = False) -> int:
"""Replace the forward of every single-projection LoRA linear in ``model``.
Returns the number of patched modules so a caller can assert it patched
something (``install_dequant_forward`` precedent). Already-patched modules
are skipped, which makes the call idempotent.
``allow_dropout`` opts this kernel into modelling non-zero LoRA dropout;
without it, dropout calls keep delegating to peft, which is the default
the MLP and QKV kernels and the tracker's validators assume.
"""
import types
types_to_match = _supported_lora_projection_types()
targets = []
for child in model.modules():
if getattr(child, _PATCH_MARKER, False):
continue
if getattr(child, _GROUP_PATCH_OWNER_MARKER, None) is not None:
continue
if isinstance(child, types_to_match):
targets.append(child)
for child in targets:
setattr(child, _ORIGINAL_FORWARD_MARKER, child.forward)
setattr(child, _HAD_INSTANCE_FORWARD_MARKER, "forward" in vars(child))
setattr(child, _PATCH_MARKER, True)
child.forward = types.MethodType(
_make_patched_forward(child.forward, allow_dropout=allow_dropout), child
)
return len(targets)
def eligible_single_projections(model: Any) -> int:
"""Patched single projections the kernel can run (#792).
A ``fan_in_fan_out`` LoRA (GPT-2's Conv1D layout) is patched like any other
but delegates to peft on every call, so it is not counted.
"""
return sum(
1
for child in model.modules()
if getattr(child, _PATCH_MARKER, False) and not getattr(child, "fan_in_fan_out", False)
)
def _release_single_projection(child: Any) -> bool:
"""Give ``child`` back the forward it had before this kernel; ``True`` if it had it.
A forward someone else installed over this kernel's is left in place.
"""
if not getattr(child, _PATCH_MARKER, False):
return False
if getattr(child.forward, _FORWARD_OWNER_MARKER, None) == _OWNER:
if getattr(child, _HAD_INSTANCE_FORWARD_MARKER, False):
child.forward = getattr(child, _ORIGINAL_FORWARD_MARKER)
else:
del child.forward
for attr in (_ORIGINAL_FORWARD_MARKER, _HAD_INSTANCE_FORWARD_MARKER, _PATCH_MARKER):
if attr in vars(child):
delattr(child, attr)
return True
def unpatch_fast_lora_single_projection(model: Any) -> int:
"""Restore the original peft forwards. Returns the number restored."""
return sum(_release_single_projection(child) for child in list(model.modules()))