SolarRenderer: Few-Step Generative Reference Rendering
Python
h3: optional fused Q/K RMSNorm and RoPE
henry-y79de39c56edd
diff --git a/src/solarwm/backends/minimax_h3/fused_qk_rope.py b/src/solarwm/backends/minimax_h3/fused_qk_rope.pynew file mode 100644index 0000000..65e908f--- /dev/null+++ b/src/solarwm/backends/minimax_h3/fused_qk_rope.py@@ -0,0 +1,193 @@+"""Frozen per-head RMSNorm and native H3 RoPE, with original bf16 boundaries."""+from collections import Counter+import math++import torch+import triton+import triton.language as tl++COUNTS = Counter()+_REASONS = set()+++def _forward_kernel(Q, K, WQ, WK, C, S, OQ, OK, RQ, RK,+ N, H: tl.constexpr, QS, QH, KS, KH, CS, CC, SS, SC,+ EQ: tl.constexpr, EK: tl.constexpr, ROWS: tl.constexpr):+ which = tl.program_id(1)+ X = tl.where(which == 0, Q, K)+ W = tl.where(which == 0, WQ, WK)+ O = tl.where(which == 0, OQ, OK)+ R = tl.where(which == 0, RQ, RK)+ xs = tl.where(which == 0, QS, KS)+ xh = tl.where(which == 0, QH, KH)+ eps = tl.where(which == 0, EQ, EK)+ row = (tl.program_id(0) * ROWS + tl.arange(0, ROWS)).to(tl.int64)+ token, head = row // H, row % H+ ch = tl.arange(0, 128)+ x = tl.load(X + token[:, None] * xs + head[:, None] * xh + ch[None, :],+ token[:, None] < N, other=0).to(tl.float32)+ w = tl.load(W + ch).to(tl.float32)+ r = tl.rsqrt(tl.sum(x * x, 1) / 128.0 + eps)+ norm = ((x * r[:, None]) * w[None, :]).to(tl.bfloat16).to(tl.float32)+ partner = tl.where(ch < 96, (ch + 48) % 96, ch)+ rotated = tl.gather(norm, tl.broadcast_to(partner[None, :], (ROWS, 128)), 1)+ rotated = tl.where(ch[None, :] < 48, -rotated, rotated)+ co = tl.load(C + token[:, None] * CS + ch[None, :] * CC,+ (token[:, None] < N) & (ch[None, :] < 96), other=0).to(tl.bfloat16).to(tl.float32)+ si = tl.load(S + token[:, None] * SS + ch[None, :] * SC,+ (token[:, None] < N) & (ch[None, :] < 96), other=0).to(tl.bfloat16).to(tl.float32)+ a = (norm * co).to(tl.bfloat16).to(tl.float32)+ b = (rotated * si).to(tl.bfloat16).to(tl.float32)+ out = tl.where(ch[None, :] < 96, a + b, norm)+ tl.store(O + row[:, None] * 128 + ch[None, :], out, token[:, None] < N)+ tl.store(R + row, r, token < N)+++def _backward_kernel(DQ, DK, Q, K, WQ, WK, C, S, RQ, RK, XQ, XK,+ N, H: tl.constexpr, QS, QH, KS, KH,+ DQS, DQH, DQC, DKS, DKH, DKC, CS, CC, SS, SC,+ ROWS: tl.constexpr):+ which = tl.program_id(1)+ X = tl.where(which == 0, Q, K)+ W = tl.where(which == 0, WQ, WK)+ DY = tl.where(which == 0, DQ, DK)+ DX = tl.where(which == 0, XQ, XK)+ R = tl.where(which == 0, RQ, RK)+ xs, xh = tl.where(which == 0, QS, KS), tl.where(which == 0, QH, KH)+ ds = tl.where(which == 0, DQS, DKS)+ dh = tl.where(which == 0, DQH, DKH)+ dc = tl.where(which == 0, DQC, DKC)+ row = (tl.program_id(0) * ROWS + tl.arange(0, ROWS)).to(tl.int64)+ token, head = row // H, row % H+ ch = tl.arange(0, 128)+ dy = tl.load(DY + token[:, None] * ds + head[:, None] * dh + ch[None, :] * dc,+ token[:, None] < N, other=0).to(tl.float32)+ partner = tl.where(ch < 96, (ch + 48) % 96, ch)+ other_dy = tl.gather(dy, tl.broadcast_to(partner[None, :], (ROWS, 128)), 1)+ co = tl.load(C + token[:, None] * CS + ch[None, :] * CC,+ (token[:, None] < N) & (ch[None, :] < 96), other=0).to(tl.bfloat16).to(tl.float32)+ si = tl.load(S + token[:, None] * SS + partner[None, :] * SC,+ (token[:, None] < N) & (ch[None, :] < 96), other=0).to(tl.bfloat16).to(tl.float32)+ a = (dy * co).to(tl.bfloat16).to(tl.float32)+ b = (other_dy * si).to(tl.bfloat16).to(tl.float32)+ b = tl.where(ch[None, :] < 48, b, -b)+ dn = tl.where(ch[None, :] < 96, a + b, dy).to(tl.bfloat16).to(tl.float32)+ x = tl.load(X + token[:, None] * xs + head[:, None] * xh + ch[None, :],+ token[:, None] < N, other=0).to(tl.float32)+ w = tl.load(W + ch).to(tl.float32)+ r = tl.load(R + row, token < N, other=0)+ g = dn * w[None, :]+ dot = tl.sum(g * x, 1)+ dmean = ((-0.5 * dot) * ((r * r) * r)) / 128.0+ dx = g * r[:, None] + (2.0 * x) * dmean[:, None]+ tl.store(DX + row[:, None] * 128 + ch[None, :], dx, token[:, None] < N)++[email protected]_op('solar_qk_rope::forward', mutates_args=(), device_types='cuda')+def _forward(q: torch.Tensor, k: torch.Tensor, wq: torch.Tensor, wk: torch.Tensor,+ c: torch.Tensor, s: torch.Tensor, eq: float, ek: float+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:+ n, h = q.shape[1:3]+ oq, ok = (torch.empty(t.shape, device=t.device, dtype=t.dtype) for t in (q, k))+ rq, rk = (torch.empty((n, h), device=q.device, dtype=torch.float32) for _ in range(2))+ _forward_kernel[(triton.cdiv(n * h, 8), 2)](+ q, k, wq, wk, c, s, oq, ok, rq, rk, n, h,+ q.stride(1), q.stride(2), k.stride(1), k.stride(2), *c.stride(), *s.stride(),+ eq, ek, 8, num_warps=4, enable_fp_fusion=False)+ COUNTS['forward'] += 1+ if COUNTS['forward'] == 1:+ print('[solar-qk-rope] fused bf16 norm/RoPE forward active', flush=True)+ return oq, ok, rq, rk+++@_forward.register_fake+def _forward_fake(q, k, wq, wk, c, s, eq, ek):+ return (torch.empty(q.shape, device=q.device, dtype=q.dtype),+ torch.empty(k.shape, device=k.device, dtype=k.dtype),+ torch.empty(q.shape[1:3], device=q.device, dtype=torch.float32),+ torch.empty(q.shape[1:3], device=q.device, dtype=torch.float32))++[email protected]_op('solar_qk_rope::backward', mutates_args=(), device_types='cuda')+def _backward(dq: torch.Tensor, dk: torch.Tensor, q: torch.Tensor, k: torch.Tensor,+ wq: torch.Tensor, wk: torch.Tensor, c: torch.Tensor, s: torch.Tensor,+ rq: torch.Tensor, rk: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:+ n, h = q.shape[1:3]+ xq, xk = (torch.empty(t.shape, device=t.device, dtype=t.dtype) for t in (q, k))+ _backward_kernel[(triton.cdiv(n * h, 8), 2)](+ dq, dk, q, k, wq, wk, c, s, rq, rk, xq, xk, n, h,+ q.stride(1), q.stride(2), k.stride(1), k.stride(2),+ *dq.stride()[1:], *dk.stride()[1:], *c.stride(), *s.stride(),+ 8, num_warps=4, enable_fp_fusion=False)+ COUNTS['backward'] += 1+ return xq, xk+++@_backward.register_fake+def _backward_fake(dq, dk, q, k, wq, wk, c, s, rq, rk):+ return tuple(torch.empty(t.shape, device=t.device, dtype=t.dtype) for t in (q, k))+++def _setup(ctx, inputs, output):+ q, k, wq, wk, c, s, eq, ek = inputs+ oq, ok, rq, rk = output+ ctx.save_for_backward(q, k, wq, wk, c, s, rq, rk)+ ctx.mark_non_differentiable(rq, rk)+ ctx.set_materialize_grads(False)+++def _autograd(ctx, dq, dk, drq, drk):+ q, k, wq, wk, c, s, rq, rk = ctx.saved_tensors+ used_q, used_k = dq is not None, dk is not None+ if not used_q and not used_k:+ return (None,) * 8+ if dq is None:+ dq = torch.zeros(q.shape, device=q.device, dtype=q.dtype)+ if dk is None:+ dk = torch.zeros(k.shape, device=k.device, dtype=k.dtype)+ xq, xk = _backward(dq, dk, q, k, wq, wk, c, s, rq, rk)+ return xq if used_q else None, xk if used_k else None, None, None, None, None, None, None+++torch.library.register_autograd('solar_qk_rope::forward', _autograd, setup_context=_setup)+++def apply(q, k, norm_q, norm_k, c, s):+ """Use the fused path only for frozen native H3 head norms and angles."""+ wq, wk = norm_q.weight, norm_k.weight+ valid = (q.is_cuda and q.dtype == torch.bfloat16 and q.ndim == 4+ and q.shape[0] == 1 and q.shape[-1] == 128 and q.shape[1] > 0+ and q.shape == k.shape and q.dtype == k.dtype and q.device == k.device+ and q.stride(-1) == k.stride(-1) == 1+ and tuple(norm_q.normalized_shape) == tuple(norm_k.normalized_shape) == (128,)+ and wq is not None and wk is not None and wq.shape == wk.shape == (128,)+ and wq.dtype == wk.dtype and wq.dtype in (torch.bfloat16, torch.float32)+ and wq.device == wk.device == q.device and not wq.requires_grad and not wk.requires_grad+ and wq.stride(0) == wk.stride(0) == 1+ and c.ndim == s.ndim == 2 and c.shape == s.shape == (q.shape[1], 96)+ and c.device == s.device == q.device and not c.requires_grad and not s.requires_grad+ and c.dtype in (torch.bfloat16, torch.float32) and s.dtype in (torch.bfloat16, torch.float32)+ and isinstance(norm_q.eps, (float, int)) and isinstance(norm_k.eps, (float, int))+ and math.isfinite(norm_q.eps) and math.isfinite(norm_k.eps)+ and min(norm_q.eps, norm_k.eps) >= 0)+ if valid:+ oq, ok, _, _ = _forward(q, k, wq, wk, c, s, float(norm_q.eps), float(norm_k.eps))+ return oq, ok+ COUNTS['fallback'] += 1+ reason = (tuple(q.shape), str(q.dtype), str(q.device),+ None if wq is None else (str(wq.dtype), wq.requires_grad),+ None if wk is None else (str(wk.dtype), wk.requires_grad), tuple(c.shape))+ if reason not in _REASONS:+ _REASONS.add(reason)+ print(f'[solar-qk-rope] original formula fallback: {reason}', flush=True)+ from diffusers.models.transformers.transformer_minimax_h3 import _apply_rotary_emb+ return _apply_rotary_emb(norm_q(q), c, s), _apply_rotary_emb(norm_k(k), c, s)+++def install(model):+ if len(model.transformer_blocks) != 50:+ raise RuntimeError('H3 Q/K norm/RoPE expects 50 blocks')+ for block in model.transformer_blocks:+ block.attn._solar_fused_qk_rope = Truediff --git a/src/solarwm/backends/minimax_h3/model.py b/src/solarwm/backends/minimax_h3/model.pyindex 691c6e0..39e578a 100644--- a/src/solarwm/backends/minimax_h3/model.py+++ b/src/solarwm/backends/minimax_h3/model.py@@ -505,15 +505,21 @@ class SolarMiniMaxH3Transformer3DModel(H3AnyFlowConditioningMixin, MiniMaxH3Tranquery = query.unflatten(-1, (attn.heads, -1))key = key.unflatten(-1, (attn.heads, -1))value = value.unflatten(-1, (attn.heads, -1))- query = attn.norm_q(query)- key = attn.norm_k(key)+ if getattr(attn, "_solar_fused_qk_rope", False) and sgf_attention is None:+ from .fused_qk_rope import apply as apply_qk_rope- if sgf_attention is not None:- return cls._sgf_attention_forward(attn, query, key, value, sgf_attention)+ query, key = apply_qk_rope(query, key, attn.norm_q, attn.norm_k, *rotary_emb)+ else:+ query = attn.norm_q(query)+ key = attn.norm_k(key)++ if sgf_attention is not None:+ return cls._sgf_attention_forward(attn, query, key, value, sgf_attention)++ query = _apply_rotary_emb(query, *rotary_emb)+ key = _apply_rotary_emb(key, *rotary_emb)# H3 native MM-RoPE first, then token-selective camera PRoPE.- query = _apply_rotary_emb(query, *rotary_emb)- key = _apply_rotary_emb(key, *rotary_emb)query, key, value, apply_output, prope_indices = cls._prepare_prope(query=query,key=key,diff --git a/src/solarwm/backends/minimax_h3/optional.py b/src/solarwm/backends/minimax_h3/optional.pyindex 5199d00..5947e3c 100644--- a/src/solarwm/backends/minimax_h3/optional.py+++ b/src/solarwm/backends/minimax_h3/optional.py@@ -132,6 +132,10 @@ def load_transformer(model_cfg: Mapping[str, Any], *, device: Any) -> H3RuntimeMfrom .fused_adaln import installinstall(transformer)+ if os.environ.get("SOLAR_FUSED_QK_ROPE") == "1":+ from .fused_qk_rope import install as install_qk_rope++ install_qk_rope(transformer)if model_cfg.get("transformer_device_map") is None:transformer = transformer.to(device)transformer.camera_intrinsics_mode = str(model_cfg.get("camera_intrinsics_mode", "wan_fixed"))diff --git a/tests/gpu/probe_fused_qk_rope.py b/tests/gpu/probe_fused_qk_rope.pynew file mode 100644index 0000000..c8d7c29--- /dev/null+++ b/tests/gpu/probe_fused_qk_rope.py@@ -0,0 +1,98 @@+"""Fused Q/K RMSNorm + RoPE regression probe (one GPU); not collected by pytest.++From the repository root:+PYTHONPATH=src CUBLAS_WORKSPACE_CONFIG=:4096:8 python tests/gpu/probe_fused_qk_rope.py++Checks the kernels against the eager path; reports correctness diagnostics only.+"""++if __name__ == "__main__":+ import tempfile+ from pathlib import Path+ probe_dir = Path(tempfile.mkdtemp(prefix="solar-qk-rope-probe-"))+ import functools,json,torch+ from torch.profiler import profile,ProfilerActivity+ from torch.utils.checkpoint import checkpoint,create_selective_checkpoint_contexts+ from diffusers.models.transformers.transformer_minimax_h3 import _apply_rotary_emb+ from solarwm.backends.minimax_h3 import fused_qk_rope as f+ from solarwm.backends.minimax_h3.fsdp import keep_attention++ torch.use_deterministic_algorithms(True)+ def rel(a,b):return ((a.detach().float()-b.detach().float()).norm()/b.detach().float().norm().clamp_min(1e-20)).item()+ def norms(dtype,eps):+ ns=[torch.nn.RMSNorm(128,eps=eps,device='cuda',dtype=dtype).requires_grad_(False)for _ in range(2)]+ for n in ns:+ w=torch.randn(150,device='cuda',dtype=dtype)[11:139]+ n.weight=torch.nn.Parameter(w,requires_grad=False)+ return ns++ def reference(q,k,nq,nk,c,s):return _apply_rotary_emb(nq(q),c,s),_apply_rotary_emb(nk(k),c,s)++ def case(n,h=4,wd=torch.bfloat16,strided=False,distribution='random',exact=False):+ torch.manual_seed(7);ns=norms(wd,0. if exact else 1e-6)+ sh=(1,n,h*2,128)if strided else(1,n,h,128)+ xs=[torch.randn(sh,device='cuda',dtype=torch.bfloat16)for _ in range(2)]+ xs=[(x[:,:,::2,:]if strided else x).detach().requires_grad_()for x in xs]+ with torch.no_grad():+ if exact:+ for x in xs:x.copy_(x.sign())+ elif distribution=='nearzero':+ for x in xs:x.mul_(1e-6).add_(1e-6)+ elif distribution=='offset':+ for x in xs:x.mul_(.1).add_(20)+ cs=[torch.randn(n,192 if strided else 96,device='cuda',dtype=torch.float32)for _ in range(2)]+ cs=[t[:,::2]if strided else t for t in cs]+ dys=[torch.randn(1,n,h,256,device='cuda',dtype=torch.bfloat16)[...,::2]for _ in range(2)]+ before=torch.cuda.get_rng_state().clone();ys=f.apply(*xs,*ns,*cs);gs=torch.autograd.grad(ys,xs,dys)+ rs=reference(*xs,*ns,*cs);rg=torch.autograd.grad(rs,xs,dys)+ errors=[rel(a,b)for a,b in zip((*ys,*gs),(*rs,*rg))]+ for a,b in zip((*ys,*gs),(*rs,*rg)):+ torch.testing.assert_close(a,b,atol=.03125,rtol=.03)+ assert max(errors)<4e-5,errors+ if exact:+ assert all(torch.equal(a,b)for a,b in zip(ys,rs)),'unit-RMS RoPE forward boundaries'+ assert all(torch.equal(a,b)for a,b in zip(gs,rg)),'unit-RMS RoPE backward boundaries'+ y2=f.apply(*xs,*ns,*cs);g2=torch.autograd.grad(y2,xs,dys)+ assert all(torch.equal(a,b)for a,b in zip((*ys,*gs),(*y2,*g2))),'repeatability'+ assert torch.equal(before,torch.cuda.get_rng_state()),'RNG'+ if n==17:+ for context in [None,functools.partial(create_selective_checkpoint_contexts,keep_attention)]:+ kw={}if context is None else{'context_fn':context}+ out=checkpoint(lambda q,k:f.apply(q,k,*ns,*cs),*xs,use_reentrant=False,**kw)+ gg=torch.autograd.grad(out,xs,dys)+ assert all(torch.equal(a,b)for a,b in zip((*ys,*gs),(*out,*gg))),'checkpoint'+ out=f.apply(*xs,*ns,*cs);only=torch.autograd.grad(out[1],xs,dys[1],allow_unused=True)+ assert only[0]is None and torch.equal(only[1],gs[1]),'unused Q gradient'+ print('PASS',n,h,wd,strided,distribution,'exact',exact,'relL2',errors,flush=True)++ for n in(1,7,17,129,193,513):case(n)+ case(17,strided=True);case(129,wd=torch.float32,strided=True)+ case(129,distribution='nearzero');case(129,distribution='offset');case(193,strided=True,exact=True)+ case(52340,h=56)+ ns=norms(torch.bfloat16,1e-6);q,k=[torch.randn(1,17,4,128,device='cuda',dtype=torch.bfloat16,requires_grad=True)for _ in range(2)];c,s=[torch.randn(17,96,device='cuda')for _ in range(2)]+ with profile(activities=[ProfilerActivity.CPU,ProfilerActivity.CUDA])as p:+ out=checkpoint(lambda q,k:f.apply(q,k,*ns,c,s),q,k,use_reentrant=False,context_fn=functools.partial(create_selective_checkpoint_contexts,keep_attention));(out[0].float().sum()+out[1].float().sum()).backward();torch.cuda.synchronize()+ p.export_chrome_trace(str(probe_dir/'trace.json'))+ es=json.load(open(str(probe_dir/'trace.json')))['traceEvents'];names=[e['name']for e in es if e.get('cat')=='kernel']+ assert any('_forward_kernel' in x for x in names)and any('_backward_kernel'in x for x in names),names+ ns[0].weight.requires_grad_(True);c.requires_grad_(True)+ out=f.apply(q,k,*ns,c,s);gg=torch.autograd.grad(sum(y.float().sum()for y in out),(q,k,ns[0].weight,c))+ ref=reference(q,k,*ns,c,s);rr=torch.autograd.grad(sum(y.float().sum()for y in ref),(q,k,ns[0].weight,c))+ assert all(torch.equal(a,b)for a,b in zip(gg,rr)),'fallback gradients'+ assert f.COUNTS['fallback']==1+ print('PASS actual CUDA kernels, fallback trainable affine/angles and counters',dict(f.COUNTS),flush=True)++ # Distinct Q/K strides; mixed angle dtypes/offsets; each unused branch.+ ns=norms(torch.float32,1e-6)+ q=torch.randn(1,37,10,128,device='cuda',dtype=torch.bfloat16)[:,:,::2,:].requires_grad_()+ k=torch.randn(1,74,5,128,device='cuda',dtype=torch.bfloat16)[:,::2,:,:].requires_grad_()+ c=torch.randn(39,193,device='cuda',dtype=torch.float32)[1:38,1:193:2]+ s=torch.randn(39,193,device='cuda',dtype=torch.bfloat16)[1:38,1:193:2]+ dys=[torch.randn(1,37,10,256,device='cuda',dtype=torch.bfloat16)[:,:,::2,::2]for _ in range(2)]+ y=f.apply(q,k,*ns,c,s);g=torch.autograd.grad(y,(q,k),dys)+ r=reference(q,k,*ns,c,s);rg=torch.autograd.grad(r,(q,k),dys)+ assert max(rel(a,b)for a,b in zip((*y,*g),(*r,*rg)))<4e-5+ for use in range(2):+ y=f.apply(q,k,*ns,c,s);gg=torch.autograd.grad(y[use],(q,k),dys[use],allow_unused=True)+ assert gg[1-use]is None and torch.equal(gg[use],g[use])+ print('PASS distinct Q/K and dy layouts, mixed/offset angles and both unused branches',flush=True)