SolarRenderer: Few-Step Generative Reference Rendering
Python
h3: optional fused AdaLN norm and gated residual kernels
henry-y9179bc0fbfeb
diff --git a/src/solarwm/backends/minimax_h3/fused_adaln.py b/src/solarwm/backends/minimax_h3/fused_adaln.pynew file mode 100644index 0000000..48e8903--- /dev/null+++ b/src/solarwm/backends/minimax_h3/fused_adaln.py@@ -0,0 +1,256 @@+"""Indexed AdaLN kernels for H3's frozen affine weights and bf16 activations."""+from __future__ import annotations++import logging+import types++import torch+import torch.nn.functional as F+import triton+import triton.language as tl++COUNTS = {"norm": 0, "gate": 0, "norm_backward": 0, "gate_backward": 0, "fallback": 0}+_REASONS = set()+_GATE_CHECKED = False+_GATE_BACKWARD_CHECKED = False+++def _norm_forward_kernel(X, W, SCALE, SHIFT, IDS, Y, RSTD,+ WIDTH: tl.constexpr, XS: tl.constexpr, WS: tl.constexpr,+ SS: tl.constexpr, HS: tl.constexpr, EPS: tl.constexpr,+ BLOCK: tl.constexpr):+ row = tl.program_id(0)+ col = tl.arange(0, BLOCK)+ valid = col < WIDTH+ x = tl.load(X + row * XS + col, valid, other=0).to(tl.float32)+ r = tl.rsqrt(tl.sum(x * x, 0) / WIDTH + EPS)+ w = tl.load(W + col * WS, valid, other=0).to(tl.float32)+ normalized = ((x * r) * w).to(tl.bfloat16).to(tl.float32)+ index = tl.load(IDS + row)+ scale = tl.load(SCALE + index * SS + col, valid, other=0).to(tl.float32)+ shift = tl.load(SHIFT + index * HS + col, valid, other=0).to(tl.float32)+ plus = (scale + 1.0).to(tl.bfloat16).to(tl.float32)+ product = (normalized * plus).to(tl.bfloat16).to(tl.float32)+ tl.store(Y + row * WIDTH + col, (product + shift).to(tl.bfloat16), valid)+ tl.store(RSTD + row, r)+++def _norm_backward_kernel(DY, X, W, SCALE, IDS, RSTD, DX,+ WIDTH: tl.constexpr, XS: tl.constexpr, WS: tl.constexpr,+ SS: tl.constexpr, DYS: tl.constexpr, DYC: tl.constexpr,+ BLOCK: tl.constexpr):+ row = tl.program_id(0)+ col = tl.arange(0, BLOCK)+ valid = col < WIDTH+ x = tl.load(X + row * XS + col, valid, other=0).to(tl.float32)+ dy = tl.load(DY + row * DYS + col * DYC, valid, other=0).to(tl.float32)+ index = tl.load(IDS + row)+ scale = tl.load(SCALE + index * SS + col, valid, other=0).to(tl.float32)+ plus = (scale + 1.0).to(tl.bfloat16).to(tl.float32)+ dn = (dy * plus).to(tl.bfloat16).to(tl.float32)+ w = tl.load(W + col * WS, valid, other=0).to(tl.float32)+ g = dn * w+ r = tl.load(RSTD + row)+ dot = tl.sum(g * x, 0)+ dmean = ((-0.5 * dot) * ((r * r) * r)) / WIDTH+ dx = g * r + (2.0 * x) * dmean+ tl.store(DX + row * WIDTH + col, dx.to(tl.bfloat16), valid)+++def _gate_forward_kernel(R, U, TABLE, IDS, Y, N,+ WIDTH: tl.constexpr, RS: tl.constexpr, US: tl.constexpr,+ TS: tl.constexpr, BLOCK: tl.constexpr):+ pos = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)+ valid = pos < N+ row, col = pos // WIDTH, pos % WIDTH+ index = tl.load(IDS + row, valid, other=0)+ r = tl.load(R + row * RS + col, valid, other=0).to(tl.float32)+ u = tl.load(U + row * US + col, valid, other=0).to(tl.float32)+ gate = tl.load(TABLE + index * TS + col, valid, other=0).to(tl.float32)+ product = (gate * u).to(tl.bfloat16).to(tl.float32)+ tl.store(Y + pos, (r + product).to(tl.bfloat16), valid)+++def _gate_backward_kernel(DY, TABLE, IDS, DU, N,+ WIDTH: tl.constexpr, DYS: tl.constexpr, DYC: tl.constexpr,+ TS: tl.constexpr, BLOCK: tl.constexpr):+ pos = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)+ valid = pos < N+ row, col = pos // WIDTH, pos % WIDTH+ index = tl.load(IDS + row, valid, other=0)+ dy = tl.load(DY + row * DYS + col * DYC, valid, other=0).to(tl.float32)+ gate = tl.load(TABLE + index * TS + col, valid, other=0).to(tl.float32)+ tl.store(DU + pos, (dy * gate).to(tl.bfloat16), valid)+++def _norm_impl(x, weight, scale, shift, indices, eps):+ y = F.rms_norm(x, (x.shape[-1],), weight, eps)+ return y * (1.0 + scale.index_select(0, indices)) + shift.index_select(0, indices)+++def _gate_impl(residual, update, table, indices):+ return residual + table.index_select(0, indices) * update++[email protected]_op("solar_adaln::norm", mutates_args=(), device_types="cuda")+def _norm(x: torch.Tensor, weight: torch.Tensor, scale: torch.Tensor,+ shift: torch.Tensor, indices: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:+ y = torch.empty(x.shape, device=x.device, dtype=x.dtype)+ r = torch.empty((x.shape[1],), device=x.device, dtype=torch.float32)+ block = triton.next_power_of_2(x.shape[-1])+ _norm_forward_kernel[(x.shape[1],)](+ x, weight, scale, shift, indices, y, r, x.shape[-1], x.stride(1), weight.stride(0),+ scale.stride(0), shift.stride(0), eps, block,+ num_warps=min(8, max(1, block // 256)), enable_fp_fusion=False)+ COUNTS["norm"] += 1+ return y, r+++@_norm.register_fake+def _norm_fake(x, weight, scale, shift, indices, eps):+ return torch.empty(x.shape, device=x.device, dtype=x.dtype), x.new_empty((x.shape[1],), dtype=torch.float32)++[email protected]_op("solar_adaln::norm_backward", mutates_args=(), device_types="cuda")+def _norm_backward(dy: torch.Tensor, x: torch.Tensor, weight: torch.Tensor,+ scale: torch.Tensor, indices: torch.Tensor, r: torch.Tensor) -> torch.Tensor:+ dx = torch.empty(x.shape, device=x.device, dtype=x.dtype)+ block = triton.next_power_of_2(x.shape[-1])+ _norm_backward_kernel[(x.shape[1],)](+ dy, x, weight, scale, indices, r, dx, x.shape[-1], x.stride(1), weight.stride(0),+ scale.stride(0), dy.stride(1), dy.stride(2), block,+ num_warps=min(8, max(1, block // 256)), enable_fp_fusion=False)+ COUNTS["norm_backward"] += 1+ return dx+++@_norm_backward.register_fake+def _norm_backward_fake(dy, x, weight, scale, indices, r):+ return torch.empty(x.shape, device=x.device, dtype=x.dtype)+++def _norm_setup(ctx, inputs, output):+ x, weight, scale, shift, indices, eps = inputs+ ctx.save_for_backward(x, weight, scale, indices, output[1])+ ctx.mark_non_differentiable(output[1])+ ctx.set_materialize_grads(False)+++def _norm_autograd(ctx, dy, dr):+ dx = None if dy is None else _norm_backward(dy, *ctx.saved_tensors)+ return dx, None, None, None, None, None+++torch.library.register_autograd("solar_adaln::norm", _norm_autograd, setup_context=_norm_setup)++[email protected]_op("solar_adaln::gate", mutates_args=(), device_types="cuda")+def _gate(residual: torch.Tensor, update: torch.Tensor, table: torch.Tensor,+ indices: torch.Tensor) -> torch.Tensor:+ global _GATE_CHECKED+ y = torch.empty(residual.shape, device=residual.device, dtype=residual.dtype)+ _gate_forward_kernel[(triton.cdiv(y.numel(), 2048),)](+ residual, update, table, indices, y, y.numel(), y.shape[-1], residual.stride(1),+ update.stride(1), table.stride(0), 2048, num_warps=4, enable_fp_fusion=False)+ if not _GATE_CHECKED:+ if not torch.equal(y, _gate_impl(residual, update, table, indices)):+ raise RuntimeError("Indexed gate forward failed its live equality check")+ _GATE_CHECKED = True+ COUNTS["gate"] += 1+ return y+++@_gate.register_fake+def _gate_fake(residual, update, table, indices):+ return torch.empty(residual.shape, device=residual.device, dtype=residual.dtype)++[email protected]_op("solar_adaln::gate_backward", mutates_args=(), device_types="cuda")+def _gate_backward(dy: torch.Tensor, table: torch.Tensor, indices: torch.Tensor) -> torch.Tensor:+ global _GATE_BACKWARD_CHECKED+ du = torch.empty(dy.shape, device=dy.device, dtype=dy.dtype)+ _gate_backward_kernel[(triton.cdiv(du.numel(), 2048),)](+ dy, table, indices, du, du.numel(), du.shape[-1], dy.stride(1), dy.stride(2),+ table.stride(0), 2048, num_warps=4, enable_fp_fusion=False)+ if not _GATE_BACKWARD_CHECKED:+ if not torch.equal(du, dy * table.index_select(0, indices)):+ raise RuntimeError("Indexed gate backward failed its live equality check")+ _GATE_BACKWARD_CHECKED = True+ COUNTS["gate_backward"] += 1+ return du+++@_gate_backward.register_fake+def _gate_backward_fake(dy, table, indices):+ return torch.empty(dy.shape, device=dy.device, dtype=dy.dtype)+++def _gate_setup(ctx, inputs, output):+ ctx.save_for_backward(inputs[2], inputs[3])+++def _gate_autograd(ctx, dy):+ return dy, _gate_backward(dy, *ctx.saved_tensors), None, None+++torch.library.register_autograd("solar_adaln::gate", _gate_autograd, setup_context=_gate_setup)+++def _fallback(reason):+ COUNTS["fallback"] += 1+ if reason not in _REASONS:+ _REASONS.add(reason)+ logging.getLogger(__name__).warning("[solar-adaln] original formula: %s", reason)+++def _layout(x):+ return (x.device.type == "cuda" and x.dtype == torch.bfloat16 and x.ndim == 3+ and x.shape[0] == 1 and x.shape[1] > 0 and x.stride(2) == 1)+++def _table(t, x):+ return (t.device == x.device and t.dtype == x.dtype and t.ndim == 2+ and t.shape[1] == x.shape[2] and t.stride(1) == 1 and not t.requires_grad)+++def norm(x, weight, scale, shift, indices, eps):+ if (not _layout(x) or not 128 <= x.shape[-1] <= 8192+ or weight.device != x.device or weight.ndim != 1 or weight.numel() != x.shape[-1]+ or weight.requires_grad or weight.dtype not in (torch.bfloat16, torch.float32)+ or not _table(scale, x) or not _table(shift, x) or indices.device != x.device+ or indices.dtype != torch.int64 or indices.ndim != 1 or indices.stride(0) != 1+ or indices.numel() != x.shape[1]):+ _fallback("norm needs B=1 CUDA bf16 contiguous channels, width 128..8192, frozen affine tables")+ return _norm_impl(x, weight, scale, shift, indices, eps)+ return _norm(x, weight, scale, shift, indices, float(eps))[0]+++def gate(residual, update, table, indices):+ if (not _layout(residual) or not _layout(update) or update.shape != residual.shape+ or update.device != residual.device or not _table(table, residual)+ or indices.device != residual.device or indices.dtype != torch.int64+ or indices.ndim != 1 or indices.stride(0) != 1 or indices.numel() != residual.shape[1]):+ _fallback("gate needs B=1 CUDA bf16 contiguous channels and a frozen gate table")+ return _gate_impl(residual, update, table, indices)+ return _gate(residual, update, table, indices)+++def _block_forward(self, hidden_states, temb, adaln_indices, rotary_emb, attention_mask=None):+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaln_proj(temb)+ residual = hidden_states+ normalized = norm(hidden_states, self.norm1.weight, scale_msa, shift_msa, adaln_indices, self.norm1.eps)+ attention = self.attn(normalized, rotary_emb, attention_mask)+ hidden_states = gate(residual, attention, gate_msa, adaln_indices)+ residual = hidden_states+ normalized = norm(hidden_states, self.norm2.weight, scale_mlp, shift_mlp, adaln_indices, self.norm2.eps)+ update = self.ff(normalized)+ return gate(residual, update, gate_mlp, adaln_indices)+++def install(model):+ for block in model.transformer_blocks:+ block.forward = types.MethodType(_block_forward, block)diff --git a/src/solarwm/backends/minimax_h3/optional.py b/src/solarwm/backends/minimax_h3/optional.pyindex 1763b15..5199d00 100644--- a/src/solarwm/backends/minimax_h3/optional.py+++ b/src/solarwm/backends/minimax_h3/optional.py@@ -128,6 +128,10 @@ def load_transformer(model_cfg: Mapping[str, Any], *, device: Any) -> H3RuntimeMinitialize()for block in transformer.transformer_blocks:block.attn.processor._attention_backend = "solar_hopper"+ if os.environ.get("SOLAR_FUSED_ADALN") == "1":+ from .fused_adaln import install++ install(transformer)if model_cfg.get("transformer_device_map") is None:transformer = transformer.to(device)transformer.camera_intrinsics_mode = str(model_cfg.get("camera_intrinsics_mode", "wan_fixed"))diff --git a/tests/gpu/probe_fused_adaln.py b/tests/gpu/probe_fused_adaln.pynew file mode 100644index 0000000..1b1620b--- /dev/null+++ b/tests/gpu/probe_fused_adaln.py@@ -0,0 +1,129 @@+"""Fused AdaLN regression probe (one GPU); not collected by pytest.++From the repository root:+PYTHONPATH=src python tests/gpu/probe_fused_adaln.py++Checks the kernels against the eager path; reports correctness diagnostics only.+"""++if __name__ == "__main__":+ from pathlib import Path+ import tempfile+ probe_dir=Path(tempfile.mkdtemp(prefix="solar-adaln-probe-"))+ import json, functools+ import torch+ from torch.utils.checkpoint import checkpoint, create_selective_checkpoint_contexts, CheckpointPolicy+ from torch.profiler import profile, ProfilerActivity+ from solarwm.backends.minimax_h3 import fused_adaln as f++ def error(a,b):+ return ((a.float()-b.float()).norm()/b.float().norm().clamp_min(1e-20)).item()+ def close(a,b,label):+ e=error(a,b)+ print(label,'relative_l2',e,'max_abs',(a.float()-b.float()).abs().max().item(),'equal_fraction',(a==b).float().mean().item(),flush=True)+ assert e<0.003,(label,e)++ torch.manual_seed(321)+ for n,d in ((129,128),(65,511),(512,5376),(1025,5376),(33,4096),(129,7680)):+ x=torch.randn(1,n,d,device='cuda',dtype=torch.bfloat16,requires_grad=True)+ w=torch.randn(d,device='cuda',dtype=torch.bfloat16)+ scale,shift,table=[torch.randn(6,d,device='cuda',dtype=torch.bfloat16) for _ in range(3)]+ ids=torch.randint(6,(n,),device='cuda')+ before=torch.cuda.get_rng_state().clone()+ y=f.norm(x,w,scale,shift,ids,1e-5)+ assert torch.equal(before,torch.cuda.get_rng_state()),'compile/forward advanced RNG'+ r=f._norm_impl(x,w,scale,shift,ids,1e-5)+ close(y,r,f'norm {n}/{d}')+ dy=torch.randn_like(y)+ gy=torch.autograd.grad(y,x,dy)[0]; gr=torch.autograd.grad(r,x,dy)[0]+ close(gy,gr,f'norm gradient {n}/{d}')+ u=torch.randn_like(x,requires_grad=True)+ z=f.gate(x,u,table,ids); rz=f._gate_impl(x,u,table,ids)+ close(z,rz,f'gate {n}/{d}')+ gs=torch.autograd.grad(z,(x,u),dy); rs=torch.autograd.grad(rz,(x,u),dy)+ for a,b in zip(gs,rs):close(a,b,f'gate gradient {n}/{d}')+ print('PASS shape',n,d,flush=True)++ def fn(a):return f.gate(a,f.norm(a,w,scale,shift,ids,1e-5),table,ids)+ ctx=functools.partial(create_selective_checkpoint_contexts,lambda *a,**k:CheckpointPolicy.PREFER_RECOMPUTE)+ # Warm up the checkpoint path, then capture only that path to prove compile survives SAC.+ y=checkpoint(fn,x,use_reentrant=False,context_fn=ctx)+ torch.autograd.grad(y,x,dy)+ with profile(activities=[ProfilerActivity.CPU,ProfilerActivity.CUDA]) as p:+ y=checkpoint(fn,x,use_reentrant=False,context_fn=ctx)+ g=torch.autograd.grad(y,x,dy)[0]+ r=fn(x); rg=torch.autograd.grad(r,x,dy)[0]+ close(g,rg,'checkpoint gradient')+ p.export_chrome_trace(str(probe_dir / 'trace.json'))+ compiled=[e for e in p.events() if any(n in e.name for n in ('_norm_forward_kernel','_gate_forward_kernel','_norm_backward_kernel','_gate_backward_kernel'))]+ print('SAC compiled events',len(compiled),'COUNTS',f.COUNTS,flush=True)+ assert compiled, 'no compiled kernels on selective-checkpoint path'+ assert f.COUNTS['fallback']==0+ # The training-ineligible case must retain gradients for affine tables.+ torch.use_deterministic_algorithms(True)+ s=scale.detach().requires_grad_()+ y=f.norm(x,w,s,shift,ids,1e-5); r=f._norm_impl(x,w,s,shift,ids,1e-5)+ for a,b in zip(torch.autograd.grad(y,(x,s),dy),torch.autograd.grad(r,(x,s),dy)):+ assert torch.equal(a,b)+ print('PASS frozen-table guard and original-formula gradients; final COUNTS',f.COUNTS,flush=True)++ trace=json.load(open(probe_dir / 'trace.json'))['traceEvents']+ names=[z['name'] for z in trace if z.get('cat')=='kernel']+ print('SAC CUDA kernels',names,flush=True)+ for kind in ('_norm_forward_kernel','_gate_forward_kernel','_norm_backward_kernel','_gate_backward_kernel'):+ assert any(kind in n for n in names),kind+ # Production-like noncontiguous affine views and strided activation/gradient rows.+ torch.use_deterministic_algorithms(True)+ n,d=52340,5376+ x=torch.randn(1,2*n,d,device='cuda',dtype=torch.bfloat16)[:,::2,:].requires_grad_()+ pool=torch.randn(6,6*d,device='cuda',dtype=torch.bfloat16)+ shift,scale,table=pool[:,:d],pool[:,d:2*d],pool[:,2*d:3*d]+ w=torch.nn.Parameter(torch.randn(d+7,device='cuda',dtype=torch.bfloat16)[7:],requires_grad=False)+ ids=torch.randint(6,(n,),device='cuda')+ dy=torch.randn(1,2*n,d,device='cuda',dtype=torch.bfloat16)[:,::2,:]+ y=f.norm(x,w,scale,shift,ids,1e-5);r=f._norm_impl(x,w,scale,shift,ids,1e-5)+ close(y,r,'strided norm')+ close(torch.autograd.grad(y,x,dy)[0],torch.autograd.grad(r,x,dy)[0],'strided norm gradient')+ u=torch.randn(1,2*n,d,device='cuda',dtype=torch.bfloat16)[:,::2,:].requires_grad_()+ y=f.gate(x,u,table,ids);r=f._gate_impl(x,u,table,ids)+ assert torch.equal(y,r)+ for a,b in zip(torch.autograd.grad(y,(x,u),dy),torch.autograd.grad(r,(x,u),dy)):assert torch.equal(a,b)+ print('PASS production-like affine/storage offsets and strided rows',f.COUNTS,flush=True)++ rng=torch.cuda.get_rng_state().clone()+ ys=[];gs=[]+ for _ in range(2):+ y=f.norm(x,w,scale,shift,ids,1e-5)+ ys.append(y.detach())+ gs.append(torch.autograd.grad(y,x,dy)[0])+ assert torch.equal(ys[0],ys[1]) and torch.equal(gs[0],gs[1])+ assert torch.equal(rng,torch.cuda.get_rng_state())+ wf=w.detach().float()+ y=f.norm(x,wf,scale,shift,ids,1e-5);r=f._norm_impl(x,wf,scale,shift,ids,1e-5)+ close(y,r,'fp32 frozen norm weight')+ close(torch.autograd.grad(y,x,dy)[0],torch.autograd.grad(r,x,dy)[0],'fp32 frozen norm weight gradient')+ print('PASS production length, deterministic repeatability, RNG and fp32 frozen weight',f.COUNTS,flush=True)++ for d in (1,2,3,5,7,15,16,17,31,32,33,63,64,65,127,128,129,130,131,132,133,134,135,255,256,257,511,512,513,5376,7680):+ n=3+d%5+ a=torch.randn(1,n,d,device='cuda',dtype=torch.bfloat16,requires_grad=True)+ b=torch.randn_like(a,requires_grad=True)+ pool=torch.randn(6,6*d,device='cuda',dtype=torch.bfloat16)+ table=pool[:,2*d:3*d]+ ids=torch.randint(6,(n,),device='cuda')+ dy=torch.randn(1,d,n,device='cuda',dtype=torch.bfloat16).transpose(1,2)+ y=f.gate(a,b,table,ids);r=f._gate_impl(a,b,table,ids)+ assert torch.equal(y,r),('gate',d)+ for ga,gr in zip(torch.autograd.grad(y,(a,b),dy),torch.autograd.grad(r,(a,b),dy)):+ assert torch.equal(ga,gr),('gate backward',d)+ print('PASS 31 additional gate widths/tails and noncontiguous output gradients',flush=True)+ for magnitude,offset in ((1e-4,0.0),(1.0,1000.0)):+ n,d=17,5376+ x=(torch.randn(1,n,d,device='cuda')*magnitude+offset).to(torch.bfloat16).requires_grad_()+ w=torch.randn(d,device='cuda',dtype=torch.bfloat16)+ scale,shift=[torch.randn(6,d,device='cuda',dtype=torch.bfloat16) for _ in range(2)]+ ids=torch.randint(6,(n,),device='cuda');dy=torch.randn_like(x)+ y=f.norm(x,w,scale,shift,ids,1e-5);r=f._norm_impl(x,w,scale,shift,ids,1e-5)+ close(y,r,'offset/scale norm')+ close(torch.autograd.grad(y,x,dy)[0],torch.autograd.grad(r,x,dy)[0],'offset/scale norm gradient')+ print('PASS complete regression probe; trace:',probe_dir/'trace.json',flush=True)