Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python

h3: optional attention output cache across activation checkpointing

henry-yd7da68952dae
diff --git a/src/solarwm/backends/minimax_h3/attention_cache.py b/src/solarwm/backends/minimax_h3/attention_cache.py
new file mode 100644
index 0000000..70941ef
--- /dev/null
+++ b/src/solarwm/backends/minimax_h3/attention_cache.py
@@ -0,0 +1,129 @@
+"""Iteration-local selective attention checkpoints with exact host offload.
+
+Uses torch's selective-checkpoint modes (torch 2.8). Attention outputs must be
+immutable, as required by the original selective-checkpoint policy as well.
+"""
+from __future__ import annotations
+
+import atexit
+import json
+import weakref
+from collections import Counter, defaultdict
+
+import torch
+from torch.utils._pytree import tree_map
+from torch.utils.checkpoint import _CachingTorchDispatchMode, _CachedTorchDispatchMode
+
+STATS = Counter()
+_STREAMS = {}
+
+
+def _stream(device):
+ if device not in _STREAMS:
+ _STREAMS[device] = torch.cuda.Stream(device=device)
+ return _STREAMS[device]
+
+
+class _HostTensor:
+ def __init__(self, tensor):
+ self.source = weakref.ref(tensor)
+ self.version = tensor._version
+ self.shape, self.stride = tensor.shape, tensor.stride()
+ self.device, self.dtype = tensor.device, tensor.dtype
+ # Rounded pinned allocations are reusable across the reader's lengths.
+ extent = 1 + sum((n - 1) * s for n, s in zip(self.shape, self.stride))
+ capacity = 1 << (extent - 1).bit_length()
+ self.host = torch.empty(capacity, dtype=self.dtype, pin_memory=True).as_strided(
+ self.shape, self.stride
+ )
+ stream = _stream(self.device)
+ stream.wait_stream(torch.cuda.current_stream(self.device))
+ with torch.cuda.stream(stream):
+ self.host.copy_(tensor.detach(), non_blocking=True)
+ tensor.record_stream(stream)
+ self.saved = torch.cuda.Event()
+ self.saved.record(stream)
+ self.value = None
+ STATS['d2h_bytes'] += tensor.numel() * tensor.element_size()
+
+ def prefetch(self):
+ if self.value is not None:
+ return
+ compute = torch.cuda.current_stream(self.device)
+ # Allocate in the compute stream's allocator pool, not the copy pool.
+ self.value = torch.empty_strided(
+ self.shape, self.stride, dtype=self.dtype, device=self.device
+ )
+ stream = _stream(self.device)
+ stream.wait_stream(compute)
+ stream.wait_event(self.saved)
+ with torch.cuda.stream(stream):
+ self.value.copy_(self.host, non_blocking=True)
+ self.value.record_stream(stream)
+ self.ready = torch.cuda.Event()
+ self.ready.record(stream)
+ STATS['h2d_bytes'] += self.value.numel() * self.value.element_size()
+
+ def get_val(self, allow_cache_entry_mutation):
+ source = self.source()
+ if not allow_cache_entry_mutation and source is not None and source._version != self.version:
+ raise RuntimeError('Attention output mutated after checkpoint caching')
+ self.prefetch()
+ compute = torch.cuda.current_stream(self.device)
+ compute.wait_event(self.ready)
+ self.value.record_stream(compute)
+ return self.value
+
+
+class _Save(_CachingTorchDispatchMode):
+ def __init__(self, policy, storage, ops, offload):
+ super().__init__(policy, storage)
+ self.ops, self.offload = ops, offload
+
+ def __torch_dispatch__(self, func, types, args=(), kwargs=None):
+ out = super().__torch_dispatch__(func, types, args, kwargs)
+ if func in self.ops:
+ STATS['saved'] += 1
+ if self.offload:
+ def transfer(wrapper, original):
+ if isinstance(original, torch.Tensor) and original.is_cuda and original.numel():
+ return _HostTensor(original)
+ return wrapper
+ self.storage[func][-1] = tree_map(transfer, self.storage[func][-1], out)
+ STATS['cpu_saved'] += 1
+ return out
+
+
+class _Replay(_CachedTorchDispatchMode):
+ def __init__(self, policy, storage, ops, offload):
+ super().__init__(policy, storage, False)
+ self.ops, self.offload = ops, offload
+
+ def __enter__(self):
+ def prefetch(value):
+ if isinstance(value, _HostTensor):
+ value.prefetch()
+ # Only this block's entries. Start H2D before Q/K/V recomputation.
+ tree_map(prefetch, dict(self.storage))
+ return super().__enter__()
+
+ def __torch_dispatch__(self, func, types, args=(), kwargs=None):
+ out = super().__torch_dispatch__(func, types, args, kwargs)
+ if func in self.ops:
+ STATS['replayed'] += 1
+ if self.offload:
+ STATS['cpu_replayed'] += 1
+ return out
+
+
+def contexts(*, offload):
+ from .fsdp import keep_attention, _attention_ops
+ storage = defaultdict(list)
+ ops = _attention_ops()
+ return _Save(keep_attention, storage, ops, offload), _Replay(keep_attention, storage, ops, offload)
+
+
+def _report():
+ if STATS:
+ print('[solar-attention-cache] ' + json.dumps(dict(STATS)), flush=True)
diff --git a/src/solarwm/backends/minimax_h3/fsdp.py b/src/solarwm/backends/minimax_h3/fsdp.py
index 7b421c4..8e51ac5 100644
--- a/src/solarwm/backends/minimax_h3/fsdp.py
+++ b/src/solarwm/backends/minimax_h3/fsdp.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import functools
import math
+import os
from collections.abc import Iterable
from datetime import timedelta
from typing import Any
@@ -101,11 +102,18 @@ def wrap_h3_fsdp(
)
if activation_checkpointing:
blocks_in_order = [m for m in wrapped.modules() if isinstance(m, transformer_block_cls)]
- keep = {id(m) for m in blocks_in_order[:saved_attention_blocks]}
+ cpu_cache = os.environ.get("SOLAR_ATTN_CPU_CACHE")
+ offload_count = int(cpu_cache) if cpu_cache is not None else 0
+ if not 0 <= offload_count <= len(blocks_in_order):
+ raise ValueError("SOLAR_ATTN_CPU_CACHE must be between 0 and the block count")
+ keep_count = len(blocks_in_order) if cpu_cache is not None else saved_attention_blocks
+ keep = {id(m) for m in blocks_in_order[:keep_count]}
+ offload = {id(m) for m in blocks_in_order[:offload_count]}
apply_activation_checkpointing(
wrapped,
checkpoint_wrapper_fn=lambda module: block_checkpoint_wrapper(
- module, save_attention=id(module) in keep
+ module, save_attention=id(module) in keep,
+ cache_all=cpu_cache is not None, offload_attention=id(module) in offload
),
check_fn=lambda module: isinstance(module, transformer_block_cls),
)
@@ -140,14 +148,22 @@ def keep_attention(ctx: Any, op: Any, *args: Any, **kwargs: Any) -> CheckpointPo
return CheckpointPolicy.MUST_SAVE if op in _attention_ops() else CheckpointPolicy.PREFER_RECOMPUTE
-def block_checkpoint_wrapper(module: torch.nn.Module, *, save_attention: bool) -> torch.nn.Module:
+def block_checkpoint_wrapper(
+ module: torch.nn.Module, *, save_attention: bool,
+ cache_all: bool = False, offload_attention: bool = False,
+) -> torch.nn.Module:
"""A block recomputed in backward, optionally without running its attention again."""
if not save_attention:
return checkpoint_wrapper(module, checkpoint_impl=CheckpointImpl.NO_REENTRANT)
+ if cache_all:
+ from .attention_cache import contexts
+ context_fn = functools.partial(contexts, offload=offload_attention)
+ else:
+ context_fn = functools.partial(create_selective_checkpoint_contexts, keep_attention)
return checkpoint_wrapper(
module,
checkpoint_impl=CheckpointImpl.NO_REENTRANT,
- context_fn=functools.partial(create_selective_checkpoint_contexts, keep_attention),
+ context_fn=context_fn,
)
diff --git a/tests/gpu/probe_attention_cache.py b/tests/gpu/probe_attention_cache.py
new file mode 100644
index 0000000..9c6b740
--- /dev/null
+++ b/tests/gpu/probe_attention_cache.py
@@ -0,0 +1,73 @@
+"""Attention output cache regression probe (one GPU); not collected by pytest.
+
+From the repository root:
+PYTHONPATH=src CUBLAS_WORKSPACE_CONFIG=:4096:8 python tests/gpu/probe_attention_cache.py
+
+Checks that cached and offloaded attention outputs give the same gradients as recomputation; correctness diagnostics
+only.
+"""
+
+if __name__ == "__main__":
+ import copy, functools
+ import torch
+ from flash_attn import flash_attn_func
+ from torch.utils.checkpoint import checkpoint, create_selective_checkpoint_contexts
+ from solarwm.backends.minimax_h3.fsdp import keep_attention
+ from solarwm.backends.minimax_h3.attention_cache import contexts, STATS
+
+ torch.use_deterministic_algorithms(True)
+ class Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__(); self.qkv=torch.nn.Linear(512,1536,device='cuda',dtype=torch.bfloat16);self.out=torch.nn.Linear(512,512,device='cuda',dtype=torch.bfloat16)
+ def forward(self,x,drop):
+ q,k,v=self.qkv(x).reshape(1,-1,3,4,128).unbind(2)
+ a=flash_attn_func(q,k,v,dropout_p=drop,deterministic=True)
+ return x+self.out(a.flatten(2)).tanh()
+
+ def run(blocks,inp,seed,new,drop):
+ torch.manual_seed(seed); x=inp.detach().clone().requires_grad_(); leaf=x
+ for i,b in enumerate(blocks):
+ kw={}
+ if new: kw['context_fn']=functools.partial(contexts,offload=i<2)
+ elif i<2: kw['context_fn']=functools.partial(create_selective_checkpoint_contexts,keep_attention)
+ x=checkpoint(b,x,drop,use_reentrant=False,**kw)
+ loss=x.float().square().mean();loss.backward()
+ return x.detach(),[leaf.grad.detach().clone()]+[p.grad.detach().clone() for b in blocks for p in b.parameters()],torch.cuda.get_rng_state().clone()
+
+ torch.manual_seed(13)
+ a=torch.nn.ModuleList([Block() for _ in range(4)]);b=copy.deepcopy(a)
+ x=torch.randn(1,257,512,device='cuda',dtype=torch.bfloat16)
+ for step,drop in enumerate([0.,0.05,1e-30,0.]):
+ for m in [a,b]:m.zero_grad(set_to_none=True)
+ ra=run(a,x,40+step,False,drop); rb=run(b,x,40+step,True,drop)
+ assert torch.equal(ra[0],rb[0]),'output'
+ assert torch.equal(ra[2],rb[2]),'rng'
+ for i,(ga,gb) in enumerate(zip(ra[1],rb[1])):assert torch.equal(ga,gb),('grad',i,(ga-gb).abs().max())
+ with torch.no_grad():
+ for pa,pb in zip(a.parameters(),b.parameters()):pa.add_(pa.grad,alpha=-0.01);pb.add_(pb.grad,alpha=-0.01)
+ torch.cuda.synchronize();print('PASS step',step,'dropout',drop,dict(STATS),flush=True)
+ assert STATS['saved']==STATS['replayed']==16
+ assert STATS['cpu_saved']==STATS['cpu_replayed']==8
+ print('PASS all exact, RNG unchanged, no stale steps',flush=True)
+
+ from solarwm.backends.minimax_h3.attention_cache import _HostTensor
+ for dtype in [torch.bfloat16,torch.float32,torch.int64]:
+ for shape in [(3,17,29),(1,104680,28,128)]:
+ base=torch.arange(torch.tensor(shape).prod().item()+11,device='cuda',dtype=torch.int64).to(dtype)
+ t=base[11:].view(shape).transpose(-1,-2)
+ expected=t.cpu(); stride=t.stride()
+ holder=_HostTensor(t); del t,base
+ # A second stream must observe the completed D2H and producer dependencies.
+ other=torch.cuda.Stream()
+ with torch.cuda.stream(other):
+ holder.prefetch(); result=holder.get_val(False)
+ assert result.stride()==stride
+ assert torch.equal(result.cpu(),expected)
+ del holder,result,expected
+ torch.cuda.synchronize()
+ print('PASS strided transfer',dtype,shape,flush=True)
+ t=torch.ones(13,device='cuda');h=_HostTensor(t);t.add_(1)
+ try:h.get_val(False)
+ except RuntimeError:pass
+ else:raise AssertionError('missing version check')
+ print('PASS strided/dtypes/lifetime/stream/version checks',flush=True)