SolarRenderer: Few-Step Generative Reference Rendering
Python
"""Inference-only caches and FA4 for the native four-step causal schedule."""
import numpy as np
import torch
from torch import nn
class ExactInputCache(nn.Module):
"""One-entry cache, invalidated whenever the actual input changes."""
def __init__(self, module):
super().__init__()
self.module = module
self.saved_input = None
self.saved_output = None
def forward(self, value):
if value.shape[1] == 0:
return value
if torch.is_grad_enabled():
return self.module(value)
if self.saved_input is None or not torch.equal(value, self.saved_input):
self.saved_output = self.module(value)
self.saved_input = value.detach().clone()
return self.saved_output
def cache_text_refiner(core):
model = core.model.get_base_model()
model.token_refiner = ExactInputCache(model.token_refiner)
class ModulationTable(nn.Module):
def __init__(self, tables, cursor):
super().__init__()
self.tables = tables
self.cursor = cursor
def forward(self, temb):
if torch.is_grad_enabled():
raise RuntimeError("Precomputed modulation is inference-only")
return self.tables[self.cursor[0]]
@torch.no_grad()
def precompute_adaln(core):
from solarwm.backends.minimax_h3.flow import make_shifted_schedule
from solarwm.backends.minimax_h3.model import get_parameter_dtype
from solarwm.backends.minimax_h3.omni_causal import CLEAN_LEVEL, STATIC_TIME
model = core.model.get_base_model()
schedule = make_shifted_schedule(5, shift=12.0)
# A reference-free static prefill contains text rows only (all STATIC_TIME).
keys = [(float(np.float32(STATIC_TIME)),)]
for level in [CLEAN_LEVEL, *(1 - float(s) for s in schedule.sigmas[:4])]:
key = tuple(
np.unique(
np.asarray([STATIC_TIME, CLEAN_LEVEL, level], dtype=np.float32)
).tolist()
)
if key not in keys:
keys.append(key)
# Static-prefix reuse removes text rows from later forwards.
key = tuple(
np.unique(np.asarray([CLEAN_LEVEL, level], dtype=np.float32)).tolist()
)
if key not in keys:
keys.append(key)
embeddings = [
model._time_condition(
torch.tensor(k, device=core.device, dtype=torch.float32),
None,
parameter_dtype=get_parameter_dtype,
)
for k in keys
]
cursor = [0]
freed = 0
for block in model.transformer_blocks:
projection = block.adaln_proj
# Each call has the native shape and preserves the original precision boundaries.
tables = [tuple(x.detach().clone() for x in projection(e)) for e in embeddings]
freed += sum(p.numel() * p.element_size() for p in projection.parameters())
block.adaln_proj = ModulationTable(tables, cursor)
original = core._run
def run(**kw):
key = tuple(np.unique(kw["times"]).tolist())
if key not in keys:
raise ValueError("Unprepared timestep set; refuse stale modulation")
cursor[0] = keys.index(key)
return original(**kw)
core._run = run
del projection, embeddings
torch.cuda.empty_cache()
return {"adaln_weight_bytes_removed": freed, "time_sets": keys}
def install_fa4():
"""Swap only the causal segmented attention kernel; keep original plan/cache."""
from flash_attn.cute.interface import flash_attn_func
from solarwm.backends.minimax_h3 import omni_causal
def attention(query, key, value, plan, layer):
if int(query.shape[1]) != plan.sequence_length:
raise ValueError("Segment sequence length mismatch")
outputs = []
for start, stop, spans, cached, commit in plan.segments:
keys = [key[:, a:b] for a, b in spans]
values = [value[:, a:b] for a, b in spans]
for chunk in cached:
k, v = plan.cache.get(layer, chunk)
if chunk[0] == "prefix":
keys.insert(0, k)
values.insert(0, v)
else:
keys.append(k)
values.append(v)
output = flash_attn_func(
query[:, start:stop].contiguous(),
torch.cat(keys, dim=1).contiguous(),
torch.cat(values, dim=1).contiguous(),
causal=False,
num_splits=1,
)
outputs.append(output[0] if isinstance(output, tuple) else output)
if commit is not None:
plan.cache.put(layer, commit, key[:, start:stop], value[:, start:stop])
return torch.cat(outputs, dim=1)
omni_causal.segment_attention = attention
from solarrenderer_sampling import SolarCausalRun
from solarwm.backends.minimax_h3.omni_causal import (
CLEAN_LEVEL,
STATIC_TIME,
H3SegmentPlan,
_merge,
_remap,
select_rows,
)
class StaticPrefixRun(SolarCausalRun):
"""Cache text/reference KV once; they attend only to each other in native plan."""
@torch.no_grad()
def __init__(self, core, batch, generator, **kwargs):
super().__init__(core, batch, generator, **kwargs)
keep = np.concatenate([np.arange(a, b) for a, b in sorted(self.rows.static)])
arrays = select_rows(self.layout, keep)
cond_rows = self.layout.source.video_indices[
: int(self.layout.source.num_condition_video_rows)
]
kept = cond_rows[np.isin(cond_rows, keep)]
pick = torch.from_numpy(np.searchsorted(cond_rows, kept)).to(core.device)
times = np.full(len(keep), CLEAN_LEVEL, dtype=np.float32)
times[: self.rows.text[1] - self.rows.text[0]] = STATIC_TIME
plan = H3SegmentPlan(
((0, len(keep), ((0, len(keep)),), (), ("prefix", 0)),),
len(keep),
self.cache,
)
core._run(
video_rows=self.observations.index_select(1, pick),
audio_rows=self.audio[:, :0],
prompt=self.prompt,
arrays=arrays,
times=times,
plan=plan,
model=None,
)
self.prompt = self.prompt[:, :0]
def _document(self, target, extra):
keep = np.concatenate(
[np.arange(a, b) for a, b in sorted([self.rows.audio, *extra, target])]
)
where = np.full(self.rows.sequence_length, -1, dtype=np.int64)
where[keep] = np.arange(len(keep))
source = self.layout.source
video_rows = source.video_indices[: int(source.num_condition_video_rows)]
kept = video_rows[where[video_rows] >= 0]
pick = torch.from_numpy(np.searchsorted(video_rows, kept)).to(self.core.device)
times = np.full(len(keep), CLEAN_LEVEL, dtype=np.float32)
return select_rows(self.layout, keep), where, pick, _remap(target, where), times
def _plan(self, where, mine, size, keys, cached, commit, extra):
audio = _remap(self.rows.audio, where)
pieces = [(audio, [audio])] + [
(_remap(e, where), [_remap(e, where)]) for e in extra
]
segments = [
(a, b, _merge(k), (("prefix", 0),), None)
for (a, b), k in sorted(pieces)
if b > a
]
segments.append(
(mine[0], mine[1], _merge([*keys, mine]), (("prefix", 0), *cached), commit)
)
return H3SegmentPlan(tuple(sorted(segments)), size, self.cache)