Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

16.4 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Per-layer NVFP4 policies for image DiTs: 4-bit layer sets over fp8, a memory lever not a speed
win, so the image auto order puts nvfp4 below fp8; fails closed"""

from __future__ import annotations

from collections import Counter
from dataclasses import dataclass, field
from typing import Any, Mapping, Optional

PRECISION_NVFP4 = "nvfp4"
PRECISION_FP8 = "fp8"
PRECISION_BF16 = "bf16"

NVFP4_POLICY_KEY = "nvfp4_policy"
NVFP4_POLICY_KIND = "unsloth_nvfp4_layer_policy_v1"


class PolicyMismatch(ValueError):
    """The model this policy was applied to is not the one it was written for."""


@dataclass(frozen = True)
class Rule:
    """``suffix`` -> ``precision``; ``expect`` is an asserted count, ``prefix`` narrows to one subtree."""

    suffix: str
    precision: str
    expect: int
    prefix: str = ""

    def matches(self, fqn: str) -> bool:
        """Exact dotted-suffix match: ``norm.linear`` does not select ``norm1.linear``."""
        if self.prefix and not fqn.startswith(self.prefix):
            return False
        return fqn == self.suffix or fqn.endswith("." + self.suffix)


@dataclass(frozen = True)
class Admit:
    """A filter-rejected linear this policy quantises anyway; ``shape`` is exact so ``t_embedder.mlp.*`` stays bf16."""

    suffix: str
    shape: tuple
    expect: int

    def matches(self, fqn: str) -> bool:
        return fqn == self.suffix or fqn.endswith("." + self.suffix)


@dataclass(frozen = True)
class NVFP4Policy:
    """One measured per-layer precision assignment, keyed on lowercased ``base_repos`` (never a sibling's)."""

    policy_id: str
    version: int
    family: str
    base_repos: tuple
    default: str = PRECISION_FP8
    rules: tuple = ()
    admit: tuple = ()
    expected_counts: Mapping = field(default_factory = dict)


QWEN_P02 = NVFP4Policy(
    policy_id = "qwen_p02_v1",
    version = 1,
    family = "qwen-image",
    base_repos = ("qwen/qwen-image",),
    rules = (
        Rule(suffix = "img_mod.1", precision = PRECISION_NVFP4, expect = 60),
        Rule(suffix = "txt_mod.1", precision = PRECISION_NVFP4, expect = 60),
    ),
    expected_counts = {PRECISION_NVFP4: 120, PRECISION_FP8: 723, PRECISION_BF16: 3},
)


def _per_block(suffix: str, stem: str, blocks: tuple) -> tuple:
    """One exact rule per listed block: the storage/quality solve ranks layers per block, not per class."""
    return tuple(
        Rule(suffix = suffix, precision = PRECISION_NVFP4, expect = 1, prefix = f"{stem}.{block}.")
        for block in blocks
    )


# Paired LPIPS(vgg) gap to fp8 < 0.05 at the upper 95% bound; artifacts at replaced policies are refused.
ZIMG_RG76 = NVFP4Policy(
    policy_id = "zimg_rg76_v1",
    version = 1,
    family = "z-image",
    base_repos = ("tongyi-mai/z-image-turbo",),
    rules = (
        Rule(suffix = "attention.to_q", precision = PRECISION_NVFP4, expect = 34),
        Rule(suffix = "attention.to_k", precision = PRECISION_NVFP4, expect = 34),
        *_per_block("feed_forward.w1", "layers", (0, 1, 2, 3, 4, 5)),
        Rule(
            suffix = "feed_forward.w1", precision = PRECISION_NVFP4, expect = 2, prefix = "noise_refiner."
        ),
    ),
    admit = (Admit(suffix = "adaLN_modulation.0", shape = (256, 15360), expect = 32),),
    expected_counts = {PRECISION_NVFP4: 76, PRECISION_FP8: 195, PRECISION_BF16: 5},
)

FLUX_R420 = NVFP4Policy(
    policy_id = "flux_r420_v1",
    version = 1,
    family = "flux.1",
    # schnell only: dev and Krea-dev are separate weights and need their own gate runs.
    base_repos = ("black-forest-labs/flux.1-schnell",),
    rules = (
        Rule(suffix = "norm_out.linear", precision = PRECISION_NVFP4, expect = 1),
        *_per_block("attn.to_k", "single_transformer_blocks", tuple(range(15, 38))),
        *_per_block("attn.to_q", "single_transformer_blocks", tuple(range(15, 38))),
        *_per_block(
            "attn.to_v", "single_transformer_blocks", (19, 21, 24, 26) + tuple(range(28, 38))
        ),
        *_per_block("norm.linear", "single_transformer_blocks", tuple(range(11, 38))),
        *_per_block("proj_mlp", "single_transformer_blocks", tuple(range(16, 38))),
        *_per_block("proj_out", "single_transformer_blocks", tuple(range(18, 38))),
        *_per_block("attn.add_k_proj", "transformer_blocks", (3, 4, 6, 15)),
        *_per_block("attn.add_q_proj", "transformer_blocks", (0, 4, 7, 9)),
        *_per_block("attn.to_add_out", "transformer_blocks", (2,)),
        *_per_block("attn.to_k", "transformer_blocks", (0, 4, 9)),
        *_per_block("attn.to_q", "transformer_blocks", (0, 1, 5, 6)),
        *_per_block("attn.to_v", "transformer_blocks", (6,)),
        *_per_block("ff.net.0.proj", "transformer_blocks", (0, 1, 3, 4, 5)),
        *_per_block("ff.net.2", "transformer_blocks", (0,) + tuple(range(2, 8))),
        *_per_block("ff_context.net.0.proj", "transformer_blocks", (12, 13, 14)),
        *_per_block(
            "ff_context.net.2",
            "transformer_blocks",
            (0, 1, 2, 5, 6, 7, 10, 11, 12, 13, 14, 16, 18),
        ),
        *_per_block("norm1.linear", "transformer_blocks", (5,)),
        *_per_block(
            "norm1_context.linear",
            "transformer_blocks",
            (3, 4, 7, 8, 9, 12, 13, 14, 15, 16, 18),
        ),
    ),
    expected_counts = {PRECISION_NVFP4: 187, PRECISION_FP8: 312, PRECISION_BF16: 3},
)

QWEN21_R020 = NVFP4Policy(
    policy_id = "qwen21_r020_v1",
    version = 1,
    family = "qwen-image-2.1",
    # Turbo is a distill of the same 32-block DiT (identical linear names and shapes) and quantizes its own weights at
    # load (no hosted artifact). Without a policy every linear went nvfp4: LPIPS 0.10 against its bf16 render on a G4,
    # where this assignment (ComfyUI arm, same layers) scored 0.011.
    base_repos = ("qwen/qwen-image-2.1", "qwen/qwen-image-2.1-turbo"),
    rules = (
        Rule(suffix = "norm_out.linear", precision = PRECISION_NVFP4, expect = 1),
        *_per_block("attn.to_k", "transformer_blocks", (5, 7, 13) + tuple(range(15, 28))),
        *_per_block("attn.to_q", "transformer_blocks", (1, 4, 18) + tuple(range(20, 28)) + (29,)),
        *_per_block(
            "img_mlp.gate_layer", "transformer_blocks", (8, 13, 17, 18, 19) + tuple(range(21, 29))
        ),
        *_per_block("img_mlp.proj", "transformer_blocks", (0, 1, 12, 13, 17, 18)),
    ),
    expected_counts = {PRECISION_NVFP4: 48, PRECISION_FP8: 181, PRECISION_BF16: 3},
)

# 2512 is its own weights under the qwen-image family: qwen_p02_v1 stays the Qwen/Qwen-Image policy.
QWEN2512_M120_ATTN8 = NVFP4Policy(
    policy_id = "qwen2512_m120_attn8_v1",
    version = 1,
    family = "qwen-image",
    base_repos = ("qwen/qwen-image-2512",),
    rules = tuple(
        Rule(suffix = suffix, precision = PRECISION_NVFP4, expect = 60)
        for suffix in (
            "attn.to_k",
            "img_mod.1",
            "txt_mod.1",
            "attn.to_add_out",
            "attn.add_q_proj",
            "attn.to_v",
            "attn.add_k_proj",
            "attn.to_q",
            "attn.to_out.0",
            "attn.add_v_proj",
        )
    ),
    expected_counts = {PRECISION_NVFP4: 600, PRECISION_FP8: 243, PRECISION_BF16: 3},
)

NVFP4_POLICIES: tuple = (ZIMG_RG76, FLUX_R420, QWEN21_R020, QWEN2512_M120_ATTN8, QWEN_P02)


def policy_by_id(policy_id: Any) -> Optional[NVFP4Policy]:
    wanted = str(policy_id or "").strip()
    for policy in NVFP4_POLICIES:
        if policy.policy_id == wanted:
            return policy
    return None


def policy_expected_counts(policy: NVFP4Policy) -> dict:
    return {
        str(key): int(value) for key, value in dict(policy.expected_counts).items() if int(value)
    }


def resolve_policy(family: Any, base_repo: Any = None) -> Optional[NVFP4Policy]:
    """The policy for ``(family, base_repo)``, or None; an unnamed base is None even if the family has one."""
    fam = str(family or "").strip().lower()
    if not fam:
        return None
    candidates = [policy for policy in NVFP4_POLICIES if policy.family == fam]
    if not candidates or not base_repo:
        return None
    from .diffusion_families import canonical_base

    base = canonical_base(str(base_repo).strip()).strip().lower()
    for policy in candidates:
        if base in policy.base_repos:
            return policy
    return None


def assign_precisions(
    transformer: Any,
    policy: NVFP4Policy,
    *,
    min_features: Optional[int] = None,
    require_divisible: Optional[int] = None,
) -> dict:
    """``{fqn: precision}`` for EVERY Linear, or raise ``PolicyMismatch``; rules apply in order, first claim wins."""
    import torch

    from .diffusion_transformer_quant import (
        DEFAULT_MIN_LINEAR_FEATURES,
        TQ_NVFP4,
        divisible_for_scheme,
        make_filter_fn,
    )

    if min_features is None:
        min_features = DEFAULT_MIN_LINEAR_FEATURES
    if require_divisible is None:
        require_divisible = divisible_for_scheme(TQ_NVFP4)
    base_filter = make_filter_fn(
        min_features,
        ("lora_",),
        require_bf16 = True,
        require_divisible = require_divisible,
    )
    linears = [
        (fqn, module)
        for fqn, module in transformer.named_modules()
        if isinstance(module, torch.nn.Linear)
    ]
    modules = dict(linears)
    admitted = {fqn for fqn, module in linears if base_filter(module, fqn)}
    for admit in policy.admit:
        hits = sorted(fqn for fqn, _ in linears if admit.matches(fqn))
        if len(hits) != admit.expect:
            raise PolicyMismatch(
                f"policy {policy.policy_id!r} admits {admit.suffix!r} expecting "
                f"{admit.expect} layers, found {len(hits)}"
            )
        for fqn in hits:
            module = modules[fqn]
            shape = (int(module.in_features), int(module.out_features))
            if shape != tuple(admit.shape):
                raise PolicyMismatch(
                    f"policy {policy.policy_id!r} admits {fqn!r} at {tuple(admit.shape)}, this "
                    f"model has {shape}"
                )
        admitted.update(hits)
    assignment: dict = {}
    for rule in policy.rules:
        hits = sorted(fqn for fqn in admitted if fqn not in assignment and rule.matches(fqn))
        if len(hits) != rule.expect:
            raise PolicyMismatch(
                f"policy {policy.policy_id!r} rule {rule.prefix}{rule.suffix!r} -> "
                f"{rule.precision} expects {rule.expect} layers, found {len(hits)}"
            )
        for fqn in hits:
            assignment[fqn] = rule.precision
    for fqn in admitted:
        assignment.setdefault(fqn, policy.default)
    for fqn, _ in linears:
        assignment.setdefault(fqn, PRECISION_BF16)
    counts = dict(Counter(assignment.values()))
    expected = policy_expected_counts(policy)
    if counts != expected:
        raise PolicyMismatch(
            f"policy {policy.policy_id!r} assigned {counts} over {len(linears)} linears, expected "
            f"{expected}"
        )
    return assignment


def policy_metadata(
    policy: NVFP4Policy,
    assignment: Mapping,
    *,
    activation_scales_baked: bool = False,
    refined: bool = False,
) -> dict:
    from .diffusion_nvfp4_gate import REFINED_FIELD
    counts = Counter(assignment.values())
    return {
        NVFP4_POLICY_KEY: {
            "kind": NVFP4_POLICY_KIND,
            "policy_id": policy.policy_id,
            "policy_version": int(policy.version),
            "counts": {str(key): int(counts[key]) for key in sorted(counts)},
            "nvfp4_fqns": sorted(
                fqn for fqn, precision in assignment.items() if precision == PRECISION_NVFP4
            ),
            "activation_scales_baked": bool(activation_scales_baked),
            REFINED_FIELD: bool(refined),
        }
    }


def declares_policy(metadata: Any) -> bool:
    """True when ``metadata`` has the policy KEY; an unreadable block must refuse, not read as "no policy"."""
    return isinstance(metadata, dict) and metadata.get(NVFP4_POLICY_KEY) not in (None, "")


def policy_metadata_error(metadata: Any) -> Optional[str]:
    """Why ``metadata``'s declared policy breaks the contract, or None. Torch-free."""
    if not declares_policy(metadata):
        return None
    block = metadata.get(NVFP4_POLICY_KEY)
    if not isinstance(block, dict):
        return f"nvfp4 policy block is {type(block).__name__}, not a dict"
    kind = block.get("kind")
    if kind != NVFP4_POLICY_KIND:
        return f"unsupported nvfp4 policy {kind!r} (this build implements {NVFP4_POLICY_KIND!r})"
    policy_id = block.get("policy_id")
    if not isinstance(policy_id, str) or not policy_id.strip():
        return f"nvfp4 policy records no policy_id ({policy_id!r})"
    version = block.get("policy_version")
    if not isinstance(version, int) or isinstance(version, bool) or version < 1:
        return f"nvfp4 policy {policy_id!r} records policy_version {version!r}"
    counts = block.get("counts")
    if not isinstance(counts, dict) or not counts:
        return f"nvfp4 policy {policy_id!r} records no counts ({counts!r})"
    for key, value in counts.items():
        if not isinstance(key, str) or not isinstance(value, int) or isinstance(value, bool):
            return f"nvfp4 policy {policy_id!r} has a malformed count entry {key!r}: {value!r}"
    fqns = block.get("nvfp4_fqns")
    if not isinstance(fqns, (list, tuple)) or not fqns:
        return f"nvfp4 policy {policy_id!r} records no nvfp4_fqns ({fqns!r})"
    if not all(isinstance(fqn, str) and fqn for fqn in fqns):
        return f"nvfp4 policy {policy_id!r} nvfp4_fqns has non-string entries"
    if len(set(fqns)) != len(fqns):
        return f"nvfp4 policy {policy_id!r} nvfp4_fqns has duplicates"
    declared = counts.get(PRECISION_NVFP4)
    if declared is not None and declared != len(fqns):
        return (
            f"nvfp4 policy {policy_id!r} counts {declared} nvfp4 layers but lists {len(fqns)} "
            "nvfp4_fqns"
        )
    return None


# By class NAME: torchao moves these between module paths across releases.
_EXPECTED_WEIGHT_CLASS = {
    PRECISION_NVFP4: "NVFP4Tensor",
    PRECISION_FP8: "Float8Tensor",
    PRECISION_BF16: "Parameter",
}


def _verify_weight_types(transformer: Any, assignment: Mapping) -> None:
    """Raise unless every assigned layer holds its precision's weight class (torchao can skip silently)."""
    wrong: list = []
    for fqn, module in transformer.named_modules():
        precision = assignment.get(fqn)
        if precision is None:
            continue
        want = _EXPECTED_WEIGHT_CLASS.get(precision)
        if want is None:
            continue
        got = type(getattr(module, "weight", None)).__name__
        if got != want:
            wrong.append(f"{fqn}: {precision} wanted {want}, got {got}")
    if wrong:
        raise PolicyMismatch(
            f"{len(wrong)} layers did not take the precision the policy assigned:\n  "
            + "\n  ".join(wrong[:20])
        )


def quantize_with_policy(
    transformer: Any,
    policy: NVFP4Policy,
    *,
    min_features: Optional[int] = None,
    fast_accum: Optional[bool] = None,
    logger: Any = None,
) -> dict:
    """Apply ``policy`` in place via two disjoint ``quantize_`` passes, NVFP4 first; discard on a raise."""
    import torch
    from torchao.quantization import quantize_

    from .diffusion_transformer_quant import TQ_FP8, TQ_NVFP4, _make_quant_config

    assignment = assign_precisions(transformer, policy, min_features = min_features)
    nvfp4_fqns = {fqn for fqn, precision in assignment.items() if precision == PRECISION_NVFP4}
    fp8_fqns = {fqn for fqn, precision in assignment.items() if precision == PRECISION_FP8}

    def nvfp4_filter(module: Any, fqn: str = "") -> bool:
        return fqn in nvfp4_fqns

    def fp8_filter(module: Any, fqn: str = "") -> bool:
        if fqn not in fp8_fqns:
            return False
        weight = getattr(module, "weight", None)
        return type(weight) is torch.nn.Parameter and weight.dtype == torch.bfloat16

    quantize_(transformer, _make_quant_config(TQ_NVFP4), filter_fn = nvfp4_filter)
    quantize_(
        transformer,
        _make_quant_config(TQ_FP8, fast_accum = fast_accum),
        filter_fn = fp8_filter,
    )
    _verify_weight_types(transformer, assignment)
    if logger is not None:
        counts = Counter(assignment.values())
        logger.info(
            "diffusion.nvfp4_policy: %s v%d applied (%s)",
            policy.policy_id,
            policy.version,
            ", ".join(f"{key} {counts[key]}" for key in sorted(counts)),
        )
    return assignment