Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

4.3 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Frozen base linears in fp8 e4m3, quantized once; backward returns only grad_x, so x is not saved."""

import torch
import torch.nn as nn

_E4M3 = torch.float8_e4m3fn
_E4M3_MAX = float(torch.finfo(_E4M3).max)
_EPS = 1e-12


def _rowwise_quant(x2d):
    xf = x2d.float()
    amax = xf.abs().amax(dim = -1, keepdim = True).clamp_min(_EPS)
    q = (xf * (_E4M3_MAX / amax)).clamp(-_E4M3_MAX, _E4M3_MAX).to(_E4M3)
    return q, amax / _E4M3_MAX


@torch.no_grad()
def _quantize_weight(w, rows = 2048):
    q = torch.empty(w.shape, dtype = _E4M3, device = w.device)
    s = torch.empty((w.shape[0], 1), dtype = torch.float32, device = w.device)
    for i in range(0, w.shape[0], rows):
        q[i : i + rows], s[i : i + rows] = _rowwise_quant(w[i : i + rows])
    return q, s


def _unit_scaled_mm(a, b):
    one = torch.ones(
        (), device = a.device, dtype = torch.float32
    )  # a fresh scalar: a buffer view fails to lower
    out = torch._scaled_mm(a, b, scale_a = one, scale_b = one, out_dtype = torch.bfloat16)
    return out[0] if isinstance(out, tuple) else out  # torch < 2.5 also returned amax


@torch._dynamo.allow_in_graph  # as torchao's float8 matmul: inlining the Function trips a dynamo freevar assert
class _FrozenFp8MM(torch.autograd.Function):
    @staticmethod
    def forward(ctx, x2d, q, s_row, out_dtype):
        xq, sx = _rowwise_quant(x2d)
        y = (_unit_scaled_mm(xq, q.t()).float() * (sx * s_row)).to(out_dtype)
        ctx.save_for_backward(q, s_row)
        ctx.x_dtype = x2d.dtype
        return y

    @staticmethod
    def backward(ctx, gy):
        if not ctx.needs_input_grad[0]:
            return None, None, None, None
        q, s_row = ctx.saved_tensors
        # The weight scale sits on the reduction dim of grad_y @ W, so it folds into grad_y; mat2 must be column-major.
        gq, sg = _rowwise_quant(gy.float() * s_row)
        gx = _unit_scaled_mm(gq, q.t().contiguous().t()).float() * sg
        return gx.to(ctx.x_dtype), None, None, None


class FrozenFp8Linear(nn.Module):
    """A frozen nn.Linear (a PEFT LoRA base_layer) with an fp8 weight; in / out features must be 16-aligned."""

    def __init__(self, linear: nn.Linear):
        super().__init__()
        out_f, in_f = linear.weight.shape
        if in_f % 16 or out_f % 16:
            raise ValueError(f"fp8 needs 16-aligned dims, got {out_f}x{in_f}")
        self.in_features, self.out_features = in_f, out_f
        # A Linear loaded from a hosted fp8 checkpoint carries its per-row codes already (diffusion_fp8_prequant).
        pre = getattr(linear, "_studio_fp8_prequant", None)
        q, s = pre if pre is not None else _quantize_weight(linear.weight.detach())
        self.register_buffer("weight", q)
        self.register_buffer("weight_scale", s.reshape(1, out_f).contiguous())
        self.bias = (
            None if linear.bias is None else nn.Parameter(linear.bias.detach(), requires_grad = False)
        )

    def forward(self, x):
        if torch.is_autocast_enabled(x.device.type):
            x = x.to(torch.get_autocast_dtype(x.device.type))
        shape = x.shape
        y = _FrozenFp8MM.apply(x.reshape(-1, shape[-1]), self.weight, self.weight_scale, x.dtype)
        y = y.reshape(*shape[:-1], self.out_features)
        return y if self.bias is None else y + self.bias.to(y.dtype)

    def extra_repr(self):
        return f"in_features={self.in_features}, out_features={self.out_features}, fp8"


def convert_frozen_fp8(model: nn.Module, names) -> tuple[int, int]:
    """Swap each frozen nn.Linear in ``names`` for FrozenFp8Linear one at a time (bounds peak memory); a layer that
    fails stays bf16 rather than raising mid-conversion. Returns (converted, failed)."""
    done = failed = 0
    for fqn in sorted(names):
        parent_name, _, child = fqn.rpartition(".")
        parent = model.get_submodule(parent_name) if parent_name else model
        old = getattr(parent, child)
        if type(old) is not nn.Linear or old.weight.requires_grad:
            continue
        try:
            new = FrozenFp8Linear(old)
        except (RuntimeError, ValueError):
            failed += 1
            continue
        new._studio_lowp = "fp8"
        setattr(parent, child, new)
        done += 1
    return done, failed