Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python

h3: optional fused Q/K RMSNorm and RoPE

henry-y79de39c56edd
diff --git a/src/solarwm/backends/minimax_h3/fused_qk_rope.py b/src/solarwm/backends/minimax_h3/fused_qk_rope.py
new file mode 100644
index 0000000..65e908f
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/fused_qk_rope.py
@@ -0,0 +1,193 @@
+"""Frozen per-head RMSNorm and native H3 RoPE, with original bf16 boundaries."""
+from collections import Counter
+import math
+
+import torch
+import triton
+import triton.language as tl
+
+COUNTS = Counter()
+_REASONS = set()
+
+
+def _forward_kernel(Q, K, WQ, WK, C, S, OQ, OK, RQ, RK,
+ N, H: tl.constexpr, QS, QH, KS, KH, CS, CC, SS, SC,
+ EQ: tl.constexpr, EK: tl.constexpr, ROWS: tl.constexpr):
+ which = tl.program_id(1)
+ X = tl.where(which == 0, Q, K)
+ W = tl.where(which == 0, WQ, WK)
+ O = tl.where(which == 0, OQ, OK)
+ R = tl.where(which == 0, RQ, RK)
+ xs = tl.where(which == 0, QS, KS)
+ xh = tl.where(which == 0, QH, KH)
+ eps = tl.where(which == 0, EQ, EK)
+ row = (tl.program_id(0) * ROWS + tl.arange(0, ROWS)).to(tl.int64)
+ token, head = row // H, row % H
+ ch = tl.arange(0, 128)
+ x = tl.load(X + token[:, None] * xs + head[:, None] * xh + ch[None, :],
+ token[:, None] < N, other=0).to(tl.float32)
+ w = tl.load(W + ch).to(tl.float32)
+ r = tl.rsqrt(tl.sum(x * x, 1) / 128.0 + eps)
+ norm = ((x * r[:, None]) * w[None, :]).to(tl.bfloat16).to(tl.float32)
+ partner = tl.where(ch < 96, (ch + 48) % 96, ch)
+ rotated = tl.gather(norm, tl.broadcast_to(partner[None, :], (ROWS, 128)), 1)
+ rotated = tl.where(ch[None, :] < 48, -rotated, rotated)
+ co = tl.load(C + token[:, None] * CS + ch[None, :] * CC,
+ (token[:, None] < N) & (ch[None, :] < 96), other=0).to(tl.bfloat16).to(tl.float32)
+ si = tl.load(S + token[:, None] * SS + ch[None, :] * SC,
+ (token[:, None] < N) & (ch[None, :] < 96), other=0).to(tl.bfloat16).to(tl.float32)
+ a = (norm * co).to(tl.bfloat16).to(tl.float32)
+ b = (rotated * si).to(tl.bfloat16).to(tl.float32)
+ out = tl.where(ch[None, :] < 96, a + b, norm)
+ tl.store(O + row[:, None] * 128 + ch[None, :], out, token[:, None] < N)
+ tl.store(R + row, r, token < N)
+
+
+def _backward_kernel(DQ, DK, Q, K, WQ, WK, C, S, RQ, RK, XQ, XK,
+ N, H: tl.constexpr, QS, QH, KS, KH,
+ DQS, DQH, DQC, DKS, DKH, DKC, CS, CC, SS, SC,
+ ROWS: tl.constexpr):
+ which = tl.program_id(1)
+ X = tl.where(which == 0, Q, K)
+ W = tl.where(which == 0, WQ, WK)
+ DY = tl.where(which == 0, DQ, DK)
+ DX = tl.where(which == 0, XQ, XK)
+ R = tl.where(which == 0, RQ, RK)
+ xs, xh = tl.where(which == 0, QS, KS), tl.where(which == 0, QH, KH)
+ ds = tl.where(which == 0, DQS, DKS)
+ dh = tl.where(which == 0, DQH, DKH)
+ dc = tl.where(which == 0, DQC, DKC)
+ row = (tl.program_id(0) * ROWS + tl.arange(0, ROWS)).to(tl.int64)
+ token, head = row // H, row % H
+ ch = tl.arange(0, 128)
+ dy = tl.load(DY + token[:, None] * ds + head[:, None] * dh + ch[None, :] * dc,
+ token[:, None] < N, other=0).to(tl.float32)
+ partner = tl.where(ch < 96, (ch + 48) % 96, ch)
+ other_dy = tl.gather(dy, tl.broadcast_to(partner[None, :], (ROWS, 128)), 1)
+ co = tl.load(C + token[:, None] * CS + ch[None, :] * CC,
+ (token[:, None] < N) & (ch[None, :] < 96), other=0).to(tl.bfloat16).to(tl.float32)
+ si = tl.load(S + token[:, None] * SS + partner[None, :] * SC,
+ (token[:, None] < N) & (ch[None, :] < 96), other=0).to(tl.bfloat16).to(tl.float32)
+ a = (dy * co).to(tl.bfloat16).to(tl.float32)
+ b = (other_dy * si).to(tl.bfloat16).to(tl.float32)
+ b = tl.where(ch[None, :] < 48, b, -b)
+ dn = tl.where(ch[None, :] < 96, a + b, dy).to(tl.bfloat16).to(tl.float32)
+ x = tl.load(X + token[:, None] * xs + head[:, None] * xh + ch[None, :],
+ token[:, None] < N, other=0).to(tl.float32)
+ w = tl.load(W + ch).to(tl.float32)
+ r = tl.load(R + row, token < N, other=0)
+ g = dn * w[None, :]
+ dot = tl.sum(g * x, 1)
+ dmean = ((-0.5 * dot) * ((r * r) * r)) / 128.0
+ dx = g * r[:, None] + (2.0 * x) * dmean[:, None]
+ tl.store(DX + row[:, None] * 128 + ch[None, :], dx, token[:, None] < N)
+
+
[email protected]_op('solar_qk_rope::forward', mutates_args=(), device_types='cuda')
+def _forward(q: torch.Tensor, k: torch.Tensor, wq: torch.Tensor, wk: torch.Tensor,
+ c: torch.Tensor, s: torch.Tensor, eq: float, ek: float
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+ n, h = q.shape[1:3]
+ oq, ok = (torch.empty(t.shape, device=t.device, dtype=t.dtype) for t in (q, k))
+ rq, rk = (torch.empty((n, h), device=q.device, dtype=torch.float32) for _ in range(2))
+ _forward_kernel[(triton.cdiv(n * h, 8), 2)](
+ q, k, wq, wk, c, s, oq, ok, rq, rk, n, h,
+ q.stride(1), q.stride(2), k.stride(1), k.stride(2), *c.stride(), *s.stride(),
+ eq, ek, 8, num_warps=4, enable_fp_fusion=False)
+ COUNTS['forward'] += 1
+ if COUNTS['forward'] == 1:
+ print('[solar-qk-rope] fused bf16 norm/RoPE forward active', flush=True)
+ return oq, ok, rq, rk
+
+
+@_forward.register_fake
+def _forward_fake(q, k, wq, wk, c, s, eq, ek):
+ return (torch.empty(q.shape, device=q.device, dtype=q.dtype),
+ torch.empty(k.shape, device=k.device, dtype=k.dtype),
+ torch.empty(q.shape[1:3], device=q.device, dtype=torch.float32),
+ torch.empty(q.shape[1:3], device=q.device, dtype=torch.float32))
+
+
[email protected]_op('solar_qk_rope::backward', mutates_args=(), device_types='cuda')
+def _backward(dq: torch.Tensor, dk: torch.Tensor, q: torch.Tensor, k: torch.Tensor,
+ wq: torch.Tensor, wk: torch.Tensor, c: torch.Tensor, s: torch.Tensor,
+ rq: torch.Tensor, rk: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ n, h = q.shape[1:3]
+ xq, xk = (torch.empty(t.shape, device=t.device, dtype=t.dtype) for t in (q, k))
+ _backward_kernel[(triton.cdiv(n * h, 8), 2)](
+ dq, dk, q, k, wq, wk, c, s, rq, rk, xq, xk, n, h,
+ q.stride(1), q.stride(2), k.stride(1), k.stride(2),
+ *dq.stride()[1:], *dk.stride()[1:], *c.stride(), *s.stride(),
+ 8, num_warps=4, enable_fp_fusion=False)
+ COUNTS['backward'] += 1
+ return xq, xk
+
+
+@_backward.register_fake
+def _backward_fake(dq, dk, q, k, wq, wk, c, s, rq, rk):
+ return tuple(torch.empty(t.shape, device=t.device, dtype=t.dtype) for t in (q, k))
+
+
+def _setup(ctx, inputs, output):
+ q, k, wq, wk, c, s, eq, ek = inputs
+ oq, ok, rq, rk = output
+ ctx.save_for_backward(q, k, wq, wk, c, s, rq, rk)
+ ctx.mark_non_differentiable(rq, rk)
+ ctx.set_materialize_grads(False)
+
+
+def _autograd(ctx, dq, dk, drq, drk):
+ q, k, wq, wk, c, s, rq, rk = ctx.saved_tensors
+ used_q, used_k = dq is not None, dk is not None
+ if not used_q and not used_k:
+ return (None,) * 8
+ if dq is None:
+ dq = torch.zeros(q.shape, device=q.device, dtype=q.dtype)
+ if dk is None:
+ dk = torch.zeros(k.shape, device=k.device, dtype=k.dtype)
+ xq, xk = _backward(dq, dk, q, k, wq, wk, c, s, rq, rk)
+ return xq if used_q else None, xk if used_k else None, None, None, None, None, None, None
+
+
+torch.library.register_autograd('solar_qk_rope::forward', _autograd, setup_context=_setup)
+
+
+def apply(q, k, norm_q, norm_k, c, s):
+ """Use the fused path only for frozen native H3 head norms and angles."""
+ wq, wk = norm_q.weight, norm_k.weight
+ valid = (q.is_cuda and q.dtype == torch.bfloat16 and q.ndim == 4
+ and q.shape[0] == 1 and q.shape[-1] == 128 and q.shape[1] > 0
+ and q.shape == k.shape and q.dtype == k.dtype and q.device == k.device
+ and q.stride(-1) == k.stride(-1) == 1
+ and tuple(norm_q.normalized_shape) == tuple(norm_k.normalized_shape) == (128,)
+ and wq is not None and wk is not None and wq.shape == wk.shape == (128,)
+ and wq.dtype == wk.dtype and wq.dtype in (torch.bfloat16, torch.float32)
+ and wq.device == wk.device == q.device and not wq.requires_grad and not wk.requires_grad
+ and wq.stride(0) == wk.stride(0) == 1
+ and c.ndim == s.ndim == 2 and c.shape == s.shape == (q.shape[1], 96)
+ and c.device == s.device == q.device and not c.requires_grad and not s.requires_grad
+ and c.dtype in (torch.bfloat16, torch.float32) and s.dtype in (torch.bfloat16, torch.float32)
+ and isinstance(norm_q.eps, (float, int)) and isinstance(norm_k.eps, (float, int))
+ and math.isfinite(norm_q.eps) and math.isfinite(norm_k.eps)
+ and min(norm_q.eps, norm_k.eps) >= 0)
+ if valid:
+ oq, ok, _, _ = _forward(q, k, wq, wk, c, s, float(norm_q.eps), float(norm_k.eps))
+ return oq, ok
+ COUNTS['fallback'] += 1
+ reason = (tuple(q.shape), str(q.dtype), str(q.device),
+ None if wq is None else (str(wq.dtype), wq.requires_grad),
+ None if wk is None else (str(wk.dtype), wk.requires_grad), tuple(c.shape))
+ if reason not in _REASONS:
+ _REASONS.add(reason)
+ print(f'[solar-qk-rope] original formula fallback: {reason}', flush=True)
+ from diffusers.models.transformers.transformer_minimax_h3 import _apply_rotary_emb
+ return _apply_rotary_emb(norm_q(q), c, s), _apply_rotary_emb(norm_k(k), c, s)
+
+
+def install(model):
+ if len(model.transformer_blocks) != 50:
+ raise RuntimeError('H3 Q/K norm/RoPE expects 50 blocks')
+ for block in model.transformer_blocks:
+ block.attn._solar_fused_qk_rope = True
diff --git a/src/solarwm/backends/minimax_h3/model.py b/src/solarwm/backends/minimax_h3/model.py
index 691c6e0..39e578a 100644
--- a/src/solarwm/backends/minimax_h3/model.py
+++ b/src/solarwm/backends/minimax_h3/model.py
@@ -505,15 +505,21 @@ class SolarMiniMaxH3Transformer3DModel(H3AnyFlowConditioningMixin, MiniMaxH3Tran
query = query.unflatten(-1, (attn.heads, -1))
key = key.unflatten(-1, (attn.heads, -1))
value = value.unflatten(-1, (attn.heads, -1))
- query = attn.norm_q(query)
- key = attn.norm_k(key)
+ if getattr(attn, "_solar_fused_qk_rope", False) and sgf_attention is None:
+ from .fused_qk_rope import apply as apply_qk_rope
- if sgf_attention is not None:
- return cls._sgf_attention_forward(attn, query, key, value, sgf_attention)
+ query, key = apply_qk_rope(query, key, attn.norm_q, attn.norm_k, *rotary_emb)
+ else:
+ query = attn.norm_q(query)
+ key = attn.norm_k(key)
+
+ if sgf_attention is not None:
+ return cls._sgf_attention_forward(attn, query, key, value, sgf_attention)
+
+ query = _apply_rotary_emb(query, *rotary_emb)
+ key = _apply_rotary_emb(key, *rotary_emb)
# H3 native MM-RoPE first, then token-selective camera PRoPE.
- query = _apply_rotary_emb(query, *rotary_emb)
- key = _apply_rotary_emb(key, *rotary_emb)
query, key, value, apply_output, prope_indices = cls._prepare_prope(
query=query,
key=key,
diff --git a/src/solarwm/backends/minimax_h3/optional.py b/src/solarwm/backends/minimax_h3/optional.py
index 5199d00..5947e3c 100644
--- a/src/solarwm/backends/minimax_h3/optional.py
+++ b/src/solarwm/backends/minimax_h3/optional.py
@@ -132,6 +132,10 @@ def load_transformer(model_cfg: Mapping[str, Any], *, device: Any) -> H3RuntimeM
from .fused_adaln import install
install(transformer)
+ if os.environ.get("SOLAR_FUSED_QK_ROPE") == "1":
+ from .fused_qk_rope import install as install_qk_rope
+
+ install_qk_rope(transformer)
if model_cfg.get("transformer_device_map") is None:
transformer = transformer.to(device)
transformer.camera_intrinsics_mode = str(model_cfg.get("camera_intrinsics_mode", "wan_fixed"))
diff --git a/tests/gpu/probe_fused_qk_rope.py b/tests/gpu/probe_fused_qk_rope.py
new file mode 100644
index 0000000..c8d7c29
--- /dev/null
+++ b/tests/gpu/probe_fused_qk_rope.py
@@ -0,0 +1,98 @@
+"""Fused Q/K RMSNorm + RoPE regression probe (one GPU); not collected by pytest.
+
+From the repository root:
+PYTHONPATH=src CUBLAS_WORKSPACE_CONFIG=:4096:8 python tests/gpu/probe_fused_qk_rope.py
+
+Checks the kernels against the eager path; reports correctness diagnostics only.
+"""
+
+if __name__ == "__main__":
+ import tempfile
+ from pathlib import Path
+ probe_dir = Path(tempfile.mkdtemp(prefix="solar-qk-rope-probe-"))
+ import functools,json,torch
+ from torch.profiler import profile,ProfilerActivity
+ from torch.utils.checkpoint import checkpoint,create_selective_checkpoint_contexts
+ from diffusers.models.transformers.transformer_minimax_h3 import _apply_rotary_emb
+ from solarwm.backends.minimax_h3 import fused_qk_rope as f
+ from solarwm.backends.minimax_h3.fsdp import keep_attention
+
+ torch.use_deterministic_algorithms(True)
+ def rel(a,b):return ((a.detach().float()-b.detach().float()).norm()/b.detach().float().norm().clamp_min(1e-20)).item()
+ def norms(dtype,eps):
+ ns=[torch.nn.RMSNorm(128,eps=eps,device='cuda',dtype=dtype).requires_grad_(False)for _ in range(2)]
+ for n in ns:
+ w=torch.randn(150,device='cuda',dtype=dtype)[11:139]
+ n.weight=torch.nn.Parameter(w,requires_grad=False)
+ return ns
+
+ def reference(q,k,nq,nk,c,s):return _apply_rotary_emb(nq(q),c,s),_apply_rotary_emb(nk(k),c,s)
+
+ def case(n,h=4,wd=torch.bfloat16,strided=False,distribution='random',exact=False):
+ torch.manual_seed(7);ns=norms(wd,0. if exact else 1e-6)
+ sh=(1,n,h*2,128)if strided else(1,n,h,128)
+ xs=[torch.randn(sh,device='cuda',dtype=torch.bfloat16)for _ in range(2)]
+ xs=[(x[:,:,::2,:]if strided else x).detach().requires_grad_()for x in xs]
+ with torch.no_grad():
+ if exact:
+ for x in xs:x.copy_(x.sign())
+ elif distribution=='nearzero':
+ for x in xs:x.mul_(1e-6).add_(1e-6)
+ elif distribution=='offset':
+ for x in xs:x.mul_(.1).add_(20)
+ cs=[torch.randn(n,192 if strided else 96,device='cuda',dtype=torch.float32)for _ in range(2)]
+ cs=[t[:,::2]if strided else t for t in cs]
+ dys=[torch.randn(1,n,h,256,device='cuda',dtype=torch.bfloat16)[...,::2]for _ in range(2)]
+ before=torch.cuda.get_rng_state().clone();ys=f.apply(*xs,*ns,*cs);gs=torch.autograd.grad(ys,xs,dys)
+ rs=reference(*xs,*ns,*cs);rg=torch.autograd.grad(rs,xs,dys)
+ errors=[rel(a,b)for a,b in zip((*ys,*gs),(*rs,*rg))]
+ for a,b in zip((*ys,*gs),(*rs,*rg)):
+ torch.testing.assert_close(a,b,atol=.03125,rtol=.03)
+ assert max(errors)<4e-5,errors
+ if exact:
+ assert all(torch.equal(a,b)for a,b in zip(ys,rs)),'unit-RMS RoPE forward boundaries'
+ assert all(torch.equal(a,b)for a,b in zip(gs,rg)),'unit-RMS RoPE backward boundaries'
+ y2=f.apply(*xs,*ns,*cs);g2=torch.autograd.grad(y2,xs,dys)
+ assert all(torch.equal(a,b)for a,b in zip((*ys,*gs),(*y2,*g2))),'repeatability'
+ assert torch.equal(before,torch.cuda.get_rng_state()),'RNG'
+ if n==17:
+ for context in [None,functools.partial(create_selective_checkpoint_contexts,keep_attention)]:
+ kw={}if context is None else{'context_fn':context}
+ out=checkpoint(lambda q,k:f.apply(q,k,*ns,*cs),*xs,use_reentrant=False,**kw)
+ gg=torch.autograd.grad(out,xs,dys)
+ assert all(torch.equal(a,b)for a,b in zip((*ys,*gs),(*out,*gg))),'checkpoint'
+ out=f.apply(*xs,*ns,*cs);only=torch.autograd.grad(out[1],xs,dys[1],allow_unused=True)
+ assert only[0]is None and torch.equal(only[1],gs[1]),'unused Q gradient'
+ print('PASS',n,h,wd,strided,distribution,'exact',exact,'relL2',errors,flush=True)
+
+ for n in(1,7,17,129,193,513):case(n)
+ case(17,strided=True);case(129,wd=torch.float32,strided=True)
+ case(129,distribution='nearzero');case(129,distribution='offset');case(193,strided=True,exact=True)
+ case(52340,h=56)
+ ns=norms(torch.bfloat16,1e-6);q,k=[torch.randn(1,17,4,128,device='cuda',dtype=torch.bfloat16,requires_grad=True)for _ in range(2)];c,s=[torch.randn(17,96,device='cuda')for _ in range(2)]
+ with profile(activities=[ProfilerActivity.CPU,ProfilerActivity.CUDA])as p:
+ out=checkpoint(lambda q,k:f.apply(q,k,*ns,c,s),q,k,use_reentrant=False,context_fn=functools.partial(create_selective_checkpoint_contexts,keep_attention));(out[0].float().sum()+out[1].float().sum()).backward();torch.cuda.synchronize()
+ p.export_chrome_trace(str(probe_dir/'trace.json'))
+ es=json.load(open(str(probe_dir/'trace.json')))['traceEvents'];names=[e['name']for e in es if e.get('cat')=='kernel']
+ assert any('_forward_kernel' in x for x in names)and any('_backward_kernel'in x for x in names),names
+ ns[0].weight.requires_grad_(True);c.requires_grad_(True)
+ out=f.apply(q,k,*ns,c,s);gg=torch.autograd.grad(sum(y.float().sum()for y in out),(q,k,ns[0].weight,c))
+ ref=reference(q,k,*ns,c,s);rr=torch.autograd.grad(sum(y.float().sum()for y in ref),(q,k,ns[0].weight,c))
+ assert all(torch.equal(a,b)for a,b in zip(gg,rr)),'fallback gradients'
+ assert f.COUNTS['fallback']==1
+ print('PASS actual CUDA kernels, fallback trainable affine/angles and counters',dict(f.COUNTS),flush=True)
+
+ # Distinct Q/K strides; mixed angle dtypes/offsets; each unused branch.
+ ns=norms(torch.float32,1e-6)
+ q=torch.randn(1,37,10,128,device='cuda',dtype=torch.bfloat16)[:,:,::2,:].requires_grad_()
+ k=torch.randn(1,74,5,128,device='cuda',dtype=torch.bfloat16)[:,::2,:,:].requires_grad_()
+ c=torch.randn(39,193,device='cuda',dtype=torch.float32)[1:38,1:193:2]
+ s=torch.randn(39,193,device='cuda',dtype=torch.bfloat16)[1:38,1:193:2]
+ dys=[torch.randn(1,37,10,256,device='cuda',dtype=torch.bfloat16)[:,:,::2,::2]for _ in range(2)]
+ y=f.apply(q,k,*ns,c,s);g=torch.autograd.grad(y,(q,k),dys)
+ r=reference(q,k,*ns,c,s);rg=torch.autograd.grad(r,(q,k),dys)
+ assert max(rel(a,b)for a,b in zip((*y,*g),(*r,*rg)))<4e-5
+ for use in range(2):
+ y=f.apply(q,k,*ns,c,s);gg=torch.autograd.grad(y[use],(q,k),dys[use],allow_unused=True)
+ assert gg[1-use]is None and torch.equal(gg[use],g[use])
+ print('PASS distinct Q/K and dy layouts, mixed/offset angles and both unused branches',flush=True)