SolarRenderer: Few-Step Generative Reference Rendering
Python
h3: optional attention output cache across activation checkpointing
henry-yd7da68952dae
diff --git a/src/solarwm/backends/minimax_h3/attention_cache.py b/src/solarwm/backends/minimax_h3/attention_cache.pynew file mode 100644index 0000000..70941ef--- /dev/null+++ b/src/solarwm/backends/minimax_h3/attention_cache.py@@ -0,0 +1,129 @@+"""Iteration-local selective attention checkpoints with exact host offload.++Uses torch's selective-checkpoint modes (torch 2.8). Attention outputs must be+immutable, as required by the original selective-checkpoint policy as well.+"""+from __future__ import annotations++import atexit+import json+import weakref+from collections import Counter, defaultdict++import torch+from torch.utils._pytree import tree_map+from torch.utils.checkpoint import _CachingTorchDispatchMode, _CachedTorchDispatchMode++STATS = Counter()+_STREAMS = {}+++def _stream(device):+ if device not in _STREAMS:+ _STREAMS[device] = torch.cuda.Stream(device=device)+ return _STREAMS[device]+++class _HostTensor:+ def __init__(self, tensor):+ self.source = weakref.ref(tensor)+ self.version = tensor._version+ self.shape, self.stride = tensor.shape, tensor.stride()+ self.device, self.dtype = tensor.device, tensor.dtype+ # Rounded pinned allocations are reusable across the reader's lengths.+ extent = 1 + sum((n - 1) * s for n, s in zip(self.shape, self.stride))+ capacity = 1 << (extent - 1).bit_length()+ self.host = torch.empty(capacity, dtype=self.dtype, pin_memory=True).as_strided(+ self.shape, self.stride+ )+ stream = _stream(self.device)+ stream.wait_stream(torch.cuda.current_stream(self.device))+ with torch.cuda.stream(stream):+ self.host.copy_(tensor.detach(), non_blocking=True)+ tensor.record_stream(stream)+ self.saved = torch.cuda.Event()+ self.saved.record(stream)+ self.value = None+ STATS['d2h_bytes'] += tensor.numel() * tensor.element_size()++ def prefetch(self):+ if self.value is not None:+ return+ compute = torch.cuda.current_stream(self.device)+ # Allocate in the compute stream's allocator pool, not the copy pool.+ self.value = torch.empty_strided(+ self.shape, self.stride, dtype=self.dtype, device=self.device+ )+ stream = _stream(self.device)+ stream.wait_stream(compute)+ stream.wait_event(self.saved)+ with torch.cuda.stream(stream):+ self.value.copy_(self.host, non_blocking=True)+ self.value.record_stream(stream)+ self.ready = torch.cuda.Event()+ self.ready.record(stream)+ STATS['h2d_bytes'] += self.value.numel() * self.value.element_size()++ def get_val(self, allow_cache_entry_mutation):+ source = self.source()+ if not allow_cache_entry_mutation and source is not None and source._version != self.version:+ raise RuntimeError('Attention output mutated after checkpoint caching')+ self.prefetch()+ compute = torch.cuda.current_stream(self.device)+ compute.wait_event(self.ready)+ self.value.record_stream(compute)+ return self.value+++class _Save(_CachingTorchDispatchMode):+ def __init__(self, policy, storage, ops, offload):+ super().__init__(policy, storage)+ self.ops, self.offload = ops, offload++ def __torch_dispatch__(self, func, types, args=(), kwargs=None):+ out = super().__torch_dispatch__(func, types, args, kwargs)+ if func in self.ops:+ STATS['saved'] += 1+ if self.offload:+ def transfer(wrapper, original):+ if isinstance(original, torch.Tensor) and original.is_cuda and original.numel():+ return _HostTensor(original)+ return wrapper+ self.storage[func][-1] = tree_map(transfer, self.storage[func][-1], out)+ STATS['cpu_saved'] += 1+ return out+++class _Replay(_CachedTorchDispatchMode):+ def __init__(self, policy, storage, ops, offload):+ super().__init__(policy, storage, False)+ self.ops, self.offload = ops, offload++ def __enter__(self):+ def prefetch(value):+ if isinstance(value, _HostTensor):+ value.prefetch()+ # Only this block's entries. Start H2D before Q/K/V recomputation.+ tree_map(prefetch, dict(self.storage))+ return super().__enter__()++ def __torch_dispatch__(self, func, types, args=(), kwargs=None):+ out = super().__torch_dispatch__(func, types, args, kwargs)+ if func in self.ops:+ STATS['replayed'] += 1+ if self.offload:+ STATS['cpu_replayed'] += 1+ return out+++def contexts(*, offload):+ from .fsdp import keep_attention, _attention_ops+ storage = defaultdict(list)+ ops = _attention_ops()+ return _Save(keep_attention, storage, ops, offload), _Replay(keep_attention, storage, ops, offload)+++def _report():+ if STATS:+ print('[solar-attention-cache] ' + json.dumps(dict(STATS)), flush=True)diff --git a/src/solarwm/backends/minimax_h3/fsdp.py b/src/solarwm/backends/minimax_h3/fsdp.pyindex 7b421c4..8e51ac5 100644--- a/src/solarwm/backends/minimax_h3/fsdp.py+++ b/src/solarwm/backends/minimax_h3/fsdp.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport functoolsimport math+import osfrom collections.abc import Iterablefrom datetime import timedeltafrom typing import Any@@ -101,11 +102,18 @@ def wrap_h3_fsdp()if activation_checkpointing:blocks_in_order = [m for m in wrapped.modules() if isinstance(m, transformer_block_cls)]- keep = {id(m) for m in blocks_in_order[:saved_attention_blocks]}+ cpu_cache = os.environ.get("SOLAR_ATTN_CPU_CACHE")+ offload_count = int(cpu_cache) if cpu_cache is not None else 0+ if not 0 <= offload_count <= len(blocks_in_order):+ raise ValueError("SOLAR_ATTN_CPU_CACHE must be between 0 and the block count")+ keep_count = len(blocks_in_order) if cpu_cache is not None else saved_attention_blocks+ keep = {id(m) for m in blocks_in_order[:keep_count]}+ offload = {id(m) for m in blocks_in_order[:offload_count]}apply_activation_checkpointing(wrapped,checkpoint_wrapper_fn=lambda module: block_checkpoint_wrapper(- module, save_attention=id(module) in keep+ module, save_attention=id(module) in keep,+ cache_all=cpu_cache is not None, offload_attention=id(module) in offload),check_fn=lambda module: isinstance(module, transformer_block_cls),)@@ -140,14 +148,22 @@ def keep_attention(ctx: Any, op: Any, *args: Any, **kwargs: Any) -> CheckpointPoreturn CheckpointPolicy.MUST_SAVE if op in _attention_ops() else CheckpointPolicy.PREFER_RECOMPUTE-def block_checkpoint_wrapper(module: torch.nn.Module, *, save_attention: bool) -> torch.nn.Module:+def block_checkpoint_wrapper(+ module: torch.nn.Module, *, save_attention: bool,+ cache_all: bool = False, offload_attention: bool = False,+) -> torch.nn.Module:"""A block recomputed in backward, optionally without running its attention again."""if not save_attention:return checkpoint_wrapper(module, checkpoint_impl=CheckpointImpl.NO_REENTRANT)+ if cache_all:+ from .attention_cache import contexts+ context_fn = functools.partial(contexts, offload=offload_attention)+ else:+ context_fn = functools.partial(create_selective_checkpoint_contexts, keep_attention)return checkpoint_wrapper(module,checkpoint_impl=CheckpointImpl.NO_REENTRANT,- context_fn=functools.partial(create_selective_checkpoint_contexts, keep_attention),+ context_fn=context_fn,)diff --git a/tests/gpu/probe_attention_cache.py b/tests/gpu/probe_attention_cache.pynew file mode 100644index 0000000..9c6b740--- /dev/null+++ b/tests/gpu/probe_attention_cache.py@@ -0,0 +1,73 @@+"""Attention output cache regression probe (one GPU); not collected by pytest.++From the repository root:+PYTHONPATH=src CUBLAS_WORKSPACE_CONFIG=:4096:8 python tests/gpu/probe_attention_cache.py++Checks that cached and offloaded attention outputs give the same gradients as recomputation; correctness diagnostics+only.+"""++if __name__ == "__main__":+ import copy, functools+ import torch+ from flash_attn import flash_attn_func+ from torch.utils.checkpoint import checkpoint, create_selective_checkpoint_contexts+ from solarwm.backends.minimax_h3.fsdp import keep_attention+ from solarwm.backends.minimax_h3.attention_cache import contexts, STATS++ torch.use_deterministic_algorithms(True)+ class Block(torch.nn.Module):+ def __init__(self):+ super().__init__(); self.qkv=torch.nn.Linear(512,1536,device='cuda',dtype=torch.bfloat16);self.out=torch.nn.Linear(512,512,device='cuda',dtype=torch.bfloat16)+ def forward(self,x,drop):+ q,k,v=self.qkv(x).reshape(1,-1,3,4,128).unbind(2)+ a=flash_attn_func(q,k,v,dropout_p=drop,deterministic=True)+ return x+self.out(a.flatten(2)).tanh()++ def run(blocks,inp,seed,new,drop):+ torch.manual_seed(seed); x=inp.detach().clone().requires_grad_(); leaf=x+ for i,b in enumerate(blocks):+ kw={}+ if new: kw['context_fn']=functools.partial(contexts,offload=i<2)+ elif i<2: kw['context_fn']=functools.partial(create_selective_checkpoint_contexts,keep_attention)+ x=checkpoint(b,x,drop,use_reentrant=False,**kw)+ loss=x.float().square().mean();loss.backward()+ return x.detach(),[leaf.grad.detach().clone()]+[p.grad.detach().clone() for b in blocks for p in b.parameters()],torch.cuda.get_rng_state().clone()++ torch.manual_seed(13)+ a=torch.nn.ModuleList([Block() for _ in range(4)]);b=copy.deepcopy(a)+ x=torch.randn(1,257,512,device='cuda',dtype=torch.bfloat16)+ for step,drop in enumerate([0.,0.05,1e-30,0.]):+ for m in [a,b]:m.zero_grad(set_to_none=True)+ ra=run(a,x,40+step,False,drop); rb=run(b,x,40+step,True,drop)+ assert torch.equal(ra[0],rb[0]),'output'+ assert torch.equal(ra[2],rb[2]),'rng'+ for i,(ga,gb) in enumerate(zip(ra[1],rb[1])):assert torch.equal(ga,gb),('grad',i,(ga-gb).abs().max())+ with torch.no_grad():+ for pa,pb in zip(a.parameters(),b.parameters()):pa.add_(pa.grad,alpha=-0.01);pb.add_(pb.grad,alpha=-0.01)+ torch.cuda.synchronize();print('PASS step',step,'dropout',drop,dict(STATS),flush=True)+ assert STATS['saved']==STATS['replayed']==16+ assert STATS['cpu_saved']==STATS['cpu_replayed']==8+ print('PASS all exact, RNG unchanged, no stale steps',flush=True)++ from solarwm.backends.minimax_h3.attention_cache import _HostTensor+ for dtype in [torch.bfloat16,torch.float32,torch.int64]:+ for shape in [(3,17,29),(1,104680,28,128)]:+ base=torch.arange(torch.tensor(shape).prod().item()+11,device='cuda',dtype=torch.int64).to(dtype)+ t=base[11:].view(shape).transpose(-1,-2)+ expected=t.cpu(); stride=t.stride()+ holder=_HostTensor(t); del t,base+ # A second stream must observe the completed D2H and producer dependencies.+ other=torch.cuda.Stream()+ with torch.cuda.stream(other):+ holder.prefetch(); result=holder.get_val(False)+ assert result.stride()==stride+ assert torch.equal(result.cpu(),expected)+ del holder,result,expected+ torch.cuda.synchronize()+ print('PASS strided transfer',dtype,shape,flush=True)+ t=torch.ones(13,device='cuda');h=_HostTensor(t);t.add_(1)+ try:h.get_val(False)+ except RuntimeError:pass+ else:raise AssertionError('missing version check')+ print('PASS strided/dtypes/lifetime/stream/version checks',flush=True)