Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python
15.7 KB
Raw Blame
"""Long clips, chunk by chunk: SolarRenderer's causal LoRA pair on ComfyUI's own MiniMax-H3.

The causal renderer generates a clip five latent frames (about 17 video frames) at a time, each chunk seeing the prompt,
the first frame, the soundtrack, the structure of its own and the five chunks before it, and the rendered frames of
those five chunks - nothing later. So a clip can run far past the 158 frames of a single render, with the memory of a
six-chunk window. This is the sampler of the released ``--causal`` path (scripts/render.py, omni_causal.stream):

* the prompt and the first frame (time 1 and 0.999) and the held silence (0.999) are written to a key/value cache once;
* each chunk's structure (0.999) is written once, seeing the prompt, the first frame and itself;
* each chunk is denoised in four evaluations against the cache, re-noised between them to the next level, and written
  back at 0.999 with its own clean frames.

Every forward runs only the rows it adds - one chunk - against the cached keys and values of the rest.
"""

import logging

import torch

import comfy.model_management as mm
import comfy.nested_tensor
import comfy.quant_ops
import comfy.utils
import latent_preview
from comfy.ldm.minimax.model import PackedLayout, pack_audio, patchify_video, rope_rotation_table, unpatchify_video

from .nodes import CATEGORY, SHIFT_AUDIO, SHIFT_VIDEO, SILENCE, _fa4, shifted_sigmas

STATIC_TIME = 1.0  # the prompt rows
CLEAN_LEVEL = 0.999  # the first frame, the structure, the soundtrack and every rendered chunk as the next ones see it
CHUNK, WINDOW, STEPS = 5, 6, 4
TAG_VIDEO, TAG_AUDIO = 0, 2  # ComfyUI's modality tags (text rows carry their own, from the text encoder)


def _silence(length: int, edge: int = 20):
    """H3's encoded silence over ``length`` audio latents, ``[1, 32, 2, length]``: away from its first and last
    ``edge`` columns the encoded silence is constant, so a longer one keeps both ends and repeats a middle column."""
    silence = comfy.utils.load_torch_file(SILENCE)["silence_170f"].float().permute(1, 0, 2).unsqueeze(0)
    columns = silence.shape[-1]
    if length <= columns:
        return silence[..., :length]
    middle = silence[..., columns // 2: columns // 2 + 1].expand(*silence.shape[:-1], length - 2 * edge)
    return torch.cat((silence[..., :edge], middle, silence[..., columns - edge:]), dim=-1).contiguous()


def _attention_kernel():
    fa4 = _fa4()

    def attend(q, k, v):  # [1, tokens, heads, head_dim]
        if fa4 is not None and q.is_cuda:
            out = fa4(q, k, v)
            return out[0] if isinstance(out, tuple) else out
        from torch.nn.attention import SDPBackend, sdpa_kernel

        order = [SDPBackend.CUDNN_ATTENTION, SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH]
        with sdpa_kernel(order, set_priority=True):
            out = torch.nn.functional.scaled_dot_product_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2))
        return out.transpose(1, 2)

    return attend


class _Cache:
    """Per layer, the keys and values (after the rotary) of written rows under ``(kind, chunk)``; chunks of a kind
    older than the window are dropped as new ones arrive. Kept on ``device`` (the GPU, or system memory for a small
    GPU: about 1 MB per row over all layers)."""

    def __init__(self, window: int, device):
        self.layers, self.window, self.device = {}, window, device

    def put(self, layer, name, key, value):
        entries = self.layers.setdefault(layer, {})
        entries[name] = (key.to(self.device, copy=True), value.to(self.device, copy=True))
        kind, chunk = name
        for old in [n for n in entries if n[0] == kind and n[1] <= chunk - self.window]:
            entries.pop(old)

    def get(self, layer, name, device):
        key, value = self.layers[layer][name]
        return key.to(device, non_blocking=True), value.to(device, non_blocking=True)


class CausalRun:
    """One clip on a loaded MiniMax-H3 (``dm``): its document's rows, the cache, and the chunk loop."""

    def __init__(self, dm, device, dtype, context, extra, latent_shapes, seed, cache_device):
        self.dm, self.device, self.dtype = dm, device, dtype
        self.attend = _attention_kernel()
        self.generator = torch.Generator(device=device).manual_seed(int(seed))
        (_, _, frames, height, width), audio_t = latent_shapes[0], latent_shapes[1][-1]
        if height % 2 or width % 2:
            raise ValueError("the latent's height and width must be even (width and height multiples of 32)")
        self.frames, self.height, self.width = frames, height, width
        self.per_frame = (height // 2) * (width // 2)

        keyframes = extra.get("minimax_keyframes") or []
        if not keyframes or keyframes[0]["latent"].shape[2] != frames:
            raise ValueError("SolarRenderer Causal Sampler takes the conditioning of SolarRenderer Structure to Video")
        context = context.to(device, dtype)
        layout = PackedLayout(context.shape[1], frames, height, width, audio_t, keyframes=keyframes)
        self.positions = layout.position_ids
        segments = layout.segments
        self.text = segments[0][:2]
        conds = [s[:2] for s in segments if s[2] == "cond"]
        self.structure_start = conds[0][0]
        self.first = conds[1] if len(conds) > 1 else None
        self.audio = next(s[:2] for s in segments if s[2] == "audio")
        self.video_start = next(s[0] for s in segments if s[2] == "video")

        # the condition rows (structure, then the first frame), as ComfyUI noises them to 0.999 for every render
        self.cond_rows = dm._cond_video_rows({"cond_video_latents": [k["latent"] for k in keyframes],
                                              "seed": int(seed)}, device)
        self.text_embed = dm.preprocess_text_embeds(context)[0]
        tags = extra.get("minimax_token_tags")
        self.text_tags = tags.view(-1).tolist() if tags is not None else [1] * context.shape[1]
        clean = _silence(audio_t).to(device)
        noise = torch.randn(clean.shape, generator=self.generator, device=device, dtype=torch.float32)
        self.audio_embed = dm.audio_patch_proj(pack_audio(CLEAN_LEVEL * clean + (1.0 - CLEAN_LEVEL) * noise)).to(dtype)
        self.cache = _Cache(WINDOW, cache_device)

    # ------------------------------------------------------------------------------------------------ one forward

    def _video_embed(self, rows):
        return self.dm.video_patch_proj(rows.to(self.device, torch.float32)).to(self.dtype)

    def _forward(self, pieces, cached, commit, sigma=None):
        """Run the rows ``pieces`` [(embed, (start, stop) in the document, time, tag)] against the cached ``cached``;
        write their keys and values as ``commit``; with ``sigma``, return the last piece's velocity rows."""
        dm = self.dm
        h = torch.cat([p[0] for p in pieces]).contiguous()
        positions = torch.cat([self.positions[a:b] for _, (a, b), _, _ in pieces])
        times = sorted({p[2] for p in pieces})
        index = {t: i for i, t in enumerate(times)}
        mod, row = [], 0
        for embed, _, t, tag in pieces:
            n = embed.shape[0]
            if isinstance(tag, list):  # the prompt: runs of its rows' own tags
                start = 0
                for i in range(1, n + 1):
                    if i == n or tag[i] != tag[start]:
                        mod.append((row + start, row + i, index[t] * 3 + int(tag[start])))
                        start = i
            else:
                mod.append((row, row + n, index[t] * 3 + tag))
            row += n

        t_vals = torch.tensor(times, dtype=torch.float32, device=self.device)
        if dm.use_adaln_curves:
            table = mm.cast_to(dm.adaln_t_table, device=self.device)
            pos = t_vals.clamp(0.0, 1.0) * (table.shape[0] - 1)
            i0 = pos.floor().long().clamp(max=table.shape[0] - 2)
            t_emb = torch.lerp(table[i0], table[i0 + 1], (pos - i0).unsqueeze(1))
        else:
            t_emb = dm.time_embedder(t_vals).to(self.dtype)
        rope = rope_rotation_table(dm.rope_freqs(positions, self.device), self.dtype)

        for layer, block in enumerate(dm.blocks):
            h = block(h, t_emb, mod, rope, transformer_options={},
                      attention=self._attention(block.attn, layer, cached, commit))
        if sigma is None:
            return None
        n = pieces[-1][0].shape[0]
        a, b = row - n, row
        video, _ = dm.final_layer(h, t_emb, (a, b, index[pieces[-1][2]]), (b, b, index[pieces[-1][2]]),
                                  torch.tensor(sigma), None, (SHIFT_VIDEO, SHIFT_AUDIO))
        return video

    def _attention(self, attn, layer, cached, commit):
        """The block's attention over its own rows and the cached ones: H3's q/k norm and rotary as ComfyUI computes
        them, the new keys and values written to the cache when the forward commits."""

        def run(x, rope_freqs=None, transformer_options={}):
            s, heads, dim = x.shape[0], attn.heads, attn.head_dim
            q, k, v = attn.qkv_proj(x).split(heads * dim, dim=-1)
            v = v.view(s, heads, dim)
            q, k = q.view(1, s, heads, dim), k.view(1, s, heads, dim)
            qw = mm.cast_to(attn.q_norm.weight, device=x.device)
            kw = mm.cast_to(attn.k_norm.weight, device=x.device)
            comfy.quant_ops.ck.rms_rope_split_half_(q, k, rope_freqs, qw, kw, epsilon=attn.q_norm.eps,
                                                    rot_dim=rope_freqs.shape[-3] * 2)
            k = k[0]
            if commit is not None:
                self.cache.put(layer, commit, k, v)
            keys, values = [], []
            for name in cached:
                ck, cv = self.cache.get(layer, name, x.device)
                keys.append(ck)
                values.append(cv)
            keys.append(k)
            values.append(v)
            out = self.attend(q.contiguous(), torch.cat(keys).unsqueeze(0).contiguous(),
                              torch.cat(values).unsqueeze(0).contiguous())
            return attn.out_proj(out.reshape(s, heads * dim))

        return run

    # ------------------------------------------------------------------------------------------------ the clip

    def prefill(self):
        """The prompt and the first frame, then the soundtrack, into the cache."""
        static = [(self.text_embed, self.text, STATIC_TIME, self.text_tags)]
        if self.first is not None:
            n = self.frames * self.per_frame
            static.append((self._video_embed(self.cond_rows[n:n + self.per_frame]), self.first, CLEAN_LEVEL, TAG_VIDEO))
        self._forward(static, (), ("static", 0))
        self._forward([(self.audio_embed, self.audio, CLEAN_LEVEL, TAG_AUDIO)], [("static", 0)], ("audio", 0))

    def chunk(self, c, f0, f1):
        """Render latent frames ``f0:f1`` (chunk ``c``); returns them ``[1, 24, f1 - f0, h, w]``."""
        rows = (f0 * self.per_frame, f1 * self.per_frame)
        structure = (self.structure_start + rows[0], self.structure_start + rows[1])
        target = (self.video_start + rows[0], self.video_start + rows[1])
        self._forward([(self._video_embed(self.cond_rows[rows[0]:rows[1]]), structure, CLEAN_LEVEL, TAG_VIDEO)],
                      [("static", 0)], ("structure", c))
        window = range(max(0, c - WINDOW + 1), c + 1)
        history = [("static", 0), ("audio", 0), *(("structure", j) for j in window),
                   *(("clean", j) for j in window if j < c)]

        shape = (1, 24, f1 - f0, self.height, self.width)
        draw = lambda: torch.randn(shape, generator=self.generator, device=self.device, dtype=torch.float32)  # noqa: E731
        sigmas = shifted_sigmas(STEPS)
        current = draw()
        for i in range(STEPS):
            sigma, sigma_next = float(sigmas[i]), float(sigmas[i + 1])
            embed = self._video_embed(patchify_video(current, self.dm.patch_size))
            velocity = self._forward([(embed, target, 1.0 - sigma, TAG_VIDEO)], history, None, sigma=sigma)
            velocity = unpatchify_video(velocity, f1 - f0, self.height // 2, self.width // 2, 24,
                                        self.dm.patch_size).float()
            estimate = current + sigma * velocity
            current = estimate if i == STEPS - 1 else (1.0 - sigma_next) * estimate + sigma_next * draw()
        written = CLEAN_LEVEL * current + (1.0 - CLEAN_LEVEL) * draw()
        self._forward([(self._video_embed(patchify_video(written, self.dm.patch_size)), target, CLEAN_LEVEL,
                        TAG_VIDEO)], history, ("clean", c))
        return current


class SolarRendererCausalSampler:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "model": ("MODEL", {"tooltip": "MiniMax-H3 with the two causal SolarRenderer LoRAs loaded "
                                               "(solarrenderer_causal_4step_lora, then solarrenderer_causal_control_lora)."}),
                "positive": ("CONDITIONING", {"tooltip": "From SolarRenderer Structure to Video, at any length."}),
                "latent": ("LATENT",),
                "seed": ("INT", {"default": 0, "min": 0, "max": 0xFFFFFFFFFFFFFFFF}),
                "cache": (["auto", "gpu", "system memory"],
                          {"tooltip": "Where the six-chunk key/value cache lives (about 60 GB at 1344x768). 'system "
                                      "memory' lets a smaller GPU render long clips, more slowly; 'auto' picks "
                                      "the GPU when it has room."}),
            }
        }

    RETURN_TYPES = ("LATENT",)
    FUNCTION = "sample"
    CATEGORY = CATEGORY
    DESCRIPTION = ("Renders the clip chunk by chunk with the causal LoRA pair: any length, the memory of a six-chunk "
                   "window. Split the result with Separate AV Latent and decode the video with VAE Decode.")

    @torch.no_grad()
    def sample(self, model, positive, latent, seed, cache):
        streams = latent["samples"]
        shapes = [t.shape for t in streams.tensors] if getattr(streams, "is_nested", False) else None
        if shapes is None or len(shapes) < 2:
            raise ValueError("SolarRenderer Causal Sampler takes the latent of SolarRenderer Structure to Video")
        mm.load_models_gpu([model])
        device, dtype = model.load_device, model.model.get_dtype_inference()
        dm = model.model.diffusion_model
        context, extra = positive[0][0], positive[0][1]

        _, _, frames, height, width = shapes[0]
        rows = (height // 2) * (width // 2)
        need = rows * CHUNK * (2 * WINDOW) * 2 * dm.blocks[0].attn.heads * dm.blocks[0].attn.head_dim * 2 * len(dm.blocks)
        on_gpu = cache == "gpu" or (cache == "auto" and mm.get_free_memory(device) > need * 1.3)
        run = CausalRun(dm, device, dtype, context, extra, shapes, seed, device if on_gpu else torch.device("cpu"))
        chunks = [(f, min(f + CHUNK, frames)) for f in range(0, frames, CHUNK)]
        logging.info(f"SolarRenderer: {len(chunks)} chunks, cache on {'the GPU' if on_gpu else 'system memory'} "
                     f"(up to {need / 2**30:.0f} GB)")

        callback = latent_preview.prepare_callback(model, len(chunks))
        run.prefill()
        clip = torch.zeros(shapes[0], dtype=torch.float32)
        for c, (f0, f1) in enumerate(chunks):
            mm.throw_exception_if_processing_interrupted()
            rendered = run.chunk(c, f0, f1)
            clip[:, :, f0:f1] = rendered.cpu()
            try:
                callback(c, rendered, None, len(chunks))
            except Exception as exc:  # noqa: BLE001 - a preview never stops the render
                logging.debug(f"SolarRenderer: no preview for chunk {c}: {exc}")
        del run
        audio = _silence(shapes[1][-1]).to(clip.dtype)
        out = latent.copy()
        out["samples"] = comfy.nested_tensor.NestedTensor((clip.to(mm.intermediate_device()), audio))
        return (out,)