Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""ConvRot: block-Hadamard activation rotation for a pre-quantized DiT transformer.
A dynamic-activation INT8 Linear quantizes BOTH sides of its GEMM to 8 bits, so its error is set
by whichever side has the heaviest outliers. A DiT's activations are outlier-heavy by
construction: a handful of channels carry most of the magnitude, the per-row amax is pinned by
them, and every other channel spends its 8 bits on a fraction of the range it could have used.
Rotating the input axis by an orthogonal matrix spreads that magnitude across the group. With a
normalized regular Hadamard ``H`` of size ``g``, applied blockwise along the input axis:
offline: W[o, k, :] <- W[o, k, :] @ H.T (baked into the checkpoint)
online: x[..., k, :] <- x[..., k, :] @ H (this module, at every forward)
``H`` is symmetric and orthogonal, so ``(x @ H) @ (W @ H.T).T == x @ W.T``: in float the pair is
an exact identity and the model is unchanged. What changes is only that the INT8 quantizer sees a
flatter distribution on both sides.
The arithmetic is the SAME ConvRot the hosted Qwen3-VL conditioner already runs, so
``build_convrot_hadamard`` / ``rotate_convrot_activation`` live here and
``video_minimax_h3_te`` imports them. One Hadamard in the tree, not two: the conditioner's and
the denoiser's rotations have to agree on the same ConvRot definition, and two copies of
a matrix nobody re-derives at review time is how they stop agreeing.
What this module adds on top of those primitives is the DENOISER half: the offline weight
rotation, the ``nn.Linear`` subclass that applies the online half, and the metadata contract that
keeps the two in step.
Why the fqn list is RECORDED rather than recomputed
---------------------------------------------------
The two halves of the identity live in different places: one in the checkpoint's weights, one in
this process. If they ever disagree about WHICH Linears are rotated, the mismatched ones compute
``x @ H @ W.T`` or ``x @ (W @ H.T).T`` -- finite, plausible-looking, and completely wrong. There
is no exception to catch and no NaN to notice; the render is just quietly worse. So the
checkpoint carries the exact list of rotated fqns and the loader rotates that list, rather than
whatever a re-evaluated filter rule selects today. A rule can drift with a code change (the set
is "quantized Linears whose in_features the group divides", and both halves of that have moved
before); a list cannot.
Everything here fails CLOSED for the same reason. ``apply_activation_rotation`` RAISES on an fqn
it cannot find, on a module that is not a Linear, on an ``in_features`` the group does not
divide, and on a rotation kind it does not implement. Its caller (the prequant loader) turns that
into a refused checkpoint and a dense fallback, which is slow but correct. Rotated artifacts also
carry their own format tag, so an Unsloth old enough to predate this module rejects them outright
instead of running them unrotated.
torch is imported inside the functions, matching the other lazily-loaded inference helpers, so
reading the metadata contract costs no import.
"""
from __future__ import annotations
from functools import lru_cache
from typing import Any, Iterable, Optional
# Unknown rotation kinds are refused so a future scheme is never misread as this one.
CONVROT_KIND = "convrot_hadamard_v1"
# Key naming mirrors the adaLN curve contract next door (``adaln_form`` / ``curve_dim`` / ``curve_grid``): a form tag
# plus the parameters needed to reproduce the form.
ROTATION_KEY = "activation_rotation"
ROTATION_GROUP_KEY = "activation_rotation_group"
ROTATION_FQNS_KEY = "activation_rotation_fqns"
# 256: lower quant error than 64 on H3, largest power of 4 dividing every quantized H3 axis.
DEFAULT_CONVROT_GROUPSIZE = 256
# Torch enumerates the parent-visible list in order, so an id's ordinal is its position in the mask; unmasked, this is the identity mapping
CONVROT_ATTR = "_unsloth_activation_rotation"
def is_power_of_four(size: Any) -> bool:
"""Only a real int: a string size in a checkpoint means a foreign artifact, not one to coerce."""
if not isinstance(size, int) or isinstance(size, bool):
return False
n = size
if n < 4 or n & (n - 1):
return False
# a power of two is a power of four exactly when its single set bit sits at an even index
return (n.bit_length() - 1) % 2 == 0
# Must stay bit-identical to the kron(H4, ...) construction hosted checkpoints were rotated with.
_HADAMARD_CACHE: dict = {}
def _convrot_sign_bits(size: int) -> Any:
"""Bool ``[size, size]`` tensor, True where the unnormalized kron power of H4 is -1."""
import torch
idx = torch.arange(size, dtype = torch.int64)
both = idx[:, None] ^ idx[None, :]
threes = both & (both >> 1) & 0x5555555555555555
parity = torch.zeros_like(threes)
# fixed Python loop count keeps this traceable under fullgraph
for _ in range((size.bit_length() - 1) // 2):
parity ^= threes & 1
threes = threes >> 2
return parity.bool()
def build_convrot_hadamard(
size: int,
device: Any = "cpu",
dtype: Any = None,
) -> Any:
"""Normalized ConvRot Hadamard, cached per (size, device, dtype). Symmetric, so ``H.T == H``."""
import torch
if dtype is None:
dtype = torch.float32
key = (size, str(device), dtype)
cached = _HADAMARD_CACHE.get(key)
if cached is not None:
return cached
if not is_power_of_four(size):
raise ValueError(f"ConvRot group size must be a power of 4, got {size}")
scale = 2.0 ** -((size.bit_length() - 1) // 2)
negative = _convrot_sign_bits(size).to(device)
h = torch.full((size, size), scale, dtype = dtype, device = device)
h = h.masked_fill(negative, -scale)
_HADAMARD_CACHE[key] = h
return h
def rotate_convrot_activation(x: Any, h: Any, group_size: int) -> Any:
"""``x @ blockdiag(H)`` over the last dimension."""
features = x.shape[-1]
if features % group_size != 0:
raise ValueError(f"features {features} not divisible by ConvRot group {group_size}")
blocks = x.reshape(-1, features // group_size, group_size)
return (blocks @ h.to(device = x.device, dtype = x.dtype)).reshape(x.shape)
def rotate_convrot_weight_(module: Any, group_size: int) -> None:
"""Accumulates in float32: a group-sized dot product in bfloat16 would spend the error rotation
saves."""
import torch
weight = module.weight.data
out_features, in_features = weight.shape
if in_features % group_size:
raise ValueError(
f"in_features {in_features} is not divisible by the ConvRot group {group_size}"
)
h = build_convrot_hadamard(group_size, device = weight.device, dtype = torch.float32)
blocks = weight.float().reshape(out_features, in_features // group_size, group_size)
module.weight.data = (blocks @ h.T).reshape(out_features, in_features).to(weight.dtype)
def quantize_convrot_int8_weight(weight: Any, group_size: int) -> tuple[Any, Any]:
"""``(int8 codes, float32 per-row scale)`` of ``W @ blockdiag(H).T``, matching comfy-kitchen
``quantize_int8_convrot_weight``. Float32 with TF32 off on purpose: bf16 rotation changes ~9% of the codes."""
import torch
out_features, in_features = weight.shape
if in_features % group_size:
raise ValueError(
f"in_features {in_features} is not divisible by the ConvRot group {group_size}"
)
h = build_convrot_hadamard(group_size, device = weight.device, dtype = torch.float32)
blocks = weight.float().reshape(out_features, in_features // group_size, group_size)
tf32 = torch.backends.cuda.matmul.allow_tf32
torch.backends.cuda.matmul.allow_tf32 = False
try:
rotated = (blocks @ h.T).reshape(out_features, in_features)
finally:
torch.backends.cuda.matmul.allow_tf32 = tf32
del blocks
scale = (rotated.abs().amax(dim = -1, keepdim = True) / 127.0).clamp(min = 1e-30)
codes = rotated.div_(scale).round_().clamp_(-128.0, 127.0).to(torch.int8)
return codes, scale
@lru_cache(maxsize = None)
def convrot_linear_class() -> Any:
"""Built once and cached: a fresh class per call makes torch.compile retrace every rotated block."""
import torch
import torch.nn.functional as F
from torch import nn
class ConvRotLinear(nn.Linear):
"""Set per instance by _install_rotation; the class default stops a half-built one rotating silently"""
convrot_groupsize: int = DEFAULT_CONVROT_GROUPSIZE
def forward(self, x: torch.Tensor) -> torch.Tensor:
group = self.convrot_groupsize
h = build_convrot_hadamard(group, device = x.device, dtype = x.dtype)
return F.linear(rotate_convrot_activation(x, h, group), self.weight, self.bias)
def extra_repr(self) -> str: # pragma: no cover - debug aid
return f"{super().extra_repr()}, convrot(group={self.convrot_groupsize})"
return ConvRotLinear
def is_rotated_linear(module: Any) -> bool:
"""True when ``module`` already applies the online rotation to its own input."""
return type(module).__name__ == "ConvRotLinear" and hasattr(module, "convrot_groupsize")
def _install_rotation(module: Any, group_size: int) -> None:
"""Swap ``module`` onto the rotating subclass, in place."""
module.convrot_groupsize = int(group_size)
module.__class__ = convrot_linear_class()
def warm_rotation_cache(transformer: Any, device: Any, dtype: Any) -> int:
"""Prefill ``_HADAMARD_CACHE``: dynamo guards on key absence, so a cold first compile recompiles on call 2."""
import torch
device = torch.device(device)
if device.type == "cuda" and device.index is None and torch.cuda.is_available():
device = torch.device("cuda", torch.cuda.current_device())
groups = {int(m.convrot_groupsize) for m in transformer.modules() if is_rotated_linear(m)}
for group in groups:
build_convrot_hadamard(group, device = device, dtype = dtype)
return len(groups)
def declares_rotation(metadata: Any) -> bool:
"""Keyed on the key being present, so an unknown rotation kind is refused rather than run unrotated."""
return isinstance(metadata, dict) and metadata.get(ROTATION_KEY) not in (None, "")
def rotation_metadata_error(metadata: Any) -> Optional[str]:
"""Checks the metadata contract only; fqn existence on the model is apply_activation_rotation's
check."""
if not declares_rotation(metadata):
return None
kind = metadata.get(ROTATION_KEY)
if kind != CONVROT_KIND:
return f"unsupported activation rotation {kind!r} (this build implements {CONVROT_KIND!r})"
group = metadata.get(ROTATION_GROUP_KEY)
if not is_power_of_four(group):
return (
f"activation rotation group {group!r} is not a power of 4; the Hadamard is built as a "
"kron power of H4"
)
fqns = metadata.get(ROTATION_FQNS_KEY)
if not isinstance(fqns, (list, tuple)) or not fqns:
# Empty list refused: it would load clean and render garbage.
return f"activation rotation records no fqns ({ROTATION_FQNS_KEY} is {fqns!r})"
if not all(isinstance(fqn, str) and fqn for fqn in fqns):
return f"activation rotation {ROTATION_FQNS_KEY} has non-string entries"
if len(set(fqns)) != len(fqns):
return f"activation rotation {ROTATION_FQNS_KEY} has duplicates"
return None
def rotation_metadata(group_size: int, fqns: Iterable[str]) -> dict:
"""Sorts the fqns so two builds of one model produce identical metadata and can be diffed."""
return {
ROTATION_KEY: CONVROT_KIND,
ROTATION_GROUP_KEY: int(group_size),
ROTATION_FQNS_KEY: sorted(fqns),
}
def rotatable_fqns(
transformer: Any,
filter_fn: Any,
group_size: int = DEFAULT_CONVROT_GROUPSIZE,
) -> tuple[tuple[str, ...], tuple[str, ...]]:
"""Linears the group cannot tile are left out, so the shipped set is recorded rather than re-derived."""
from torch import nn
rotatable: list[str] = []
not_divisible: list[str] = []
for fqn, module in transformer.named_modules():
if not isinstance(module, nn.Linear) or not filter_fn(module, fqn):
continue
(rotatable if module.in_features % group_size == 0 else not_divisible).append(fqn)
return tuple(rotatable), tuple(not_divisible)
def rotate_linears_(
transformer: Any,
fqns: Iterable[str],
group_size: int = DEFAULT_CONVROT_GROUPSIZE,
) -> tuple[str, ...]:
"""Offline half: call before quantize_ on a dense model, so the quantizer sees the flatter
distribution."""
from torch import nn
if not is_power_of_four(group_size):
raise ValueError(f"ConvRot group size must be a power of 4, got {group_size!r}")
modules = dict(transformer.named_modules())
rotated: list[str] = []
for fqn in fqns:
module = modules.get(fqn)
if not isinstance(module, nn.Linear):
raise ValueError(f"cannot rotate {fqn!r}: not an nn.Linear on this model")
rotate_convrot_weight_(module, group_size)
_install_rotation(module, group_size)
rotated.append(fqn)
return tuple(rotated)
def apply_activation_rotation(
transformer: Any,
metadata: Any,
*,
logger: Any = None,
) -> tuple[str, ...]:
"""Must run after load_state_dict and before apply_small_m_padding; raises rather than skip a
bad fqn."""
if not declares_rotation(metadata):
return ()
problem = rotation_metadata_error(metadata)
if problem:
raise ValueError(problem)
from torch import nn
group_size = int(metadata[ROTATION_GROUP_KEY])
fqns = list(metadata[ROTATION_FQNS_KEY])
modules = dict(transformer.named_modules())
missing = [fqn for fqn in fqns if fqn not in modules]
if missing:
raise ValueError(
f"activation rotation names {len(missing)} fqn(s) this model does not have "
f"(e.g. {missing[0]!r}); the checkpoint and this build disagree about the model"
)
for fqn in fqns:
module = modules[fqn]
if not isinstance(module, nn.Linear):
raise ValueError(f"activation rotation target {fqn!r} is not an nn.Linear")
if is_rotated_linear(module):
raise ValueError(f"activation rotation target {fqn!r} is already rotated")
if module.in_features % group_size:
raise ValueError(
f"activation rotation target {fqn!r} has in_features {module.in_features}, "
f"which the recorded group {group_size} does not divide"
)
# validate every target before swapping ANY: a partial install still renders, just wrongly
for fqn in fqns:
_install_rotation(modules[fqn], group_size)
try:
setattr(
transformer,
CONVROT_ATTR,
{"kind": CONVROT_KIND, "group": group_size, "linears": len(fqns)},
)
except Exception: # noqa: BLE001 -- the marker is a diagnostic, never the mechanism
pass
if logger is not None:
logger.info(
"diffusion.convrot: installed %s on %d linears at group %d",
CONVROT_KIND,
len(fqns),
group_size,
)
return tuple(fqns)