Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python
24.8 KB
Raw Blame
"""SolarRenderer nodes for ComfyUI.

They run the released renderer (scripts/render.py) on ComfyUI's own MiniMax-H3 (Comfy-Org/MiniMax-H3, the Ref2VA
transformer):

* SolarRendererLoraLoader - a released SolarRenderer LoRA as it is on the Hub, each module patched onto its part of
  ComfyUI's fused weights (q, k and v onto their thirds of attn.qkv_proj, the two halves of the feed-forward input
  onto theirs of mlp.fc1);
* SolarRendererConditioning - the clip's inputs as training encoded them: the structure video laid over every target
  frame and the first frame over frame 0, both as frame-aligned keyframes, and the prompt read by H3's Qwen3-VL with
  the two videos shown to it as <Video 1> and <Video 2> at 2 fps;
* SolarRendererModel - the soundtrack held at H3's encoded silence through sampling, as the renderer holds it (it
  never generates sound, and a generated one would sit in every attention);
* SolarRendererSigmas - the four-step grid (shift 12) the four-step LoRA was trained on; sample it with the "lcm"
  sampler, which takes the clean estimate at every step and noises it afresh, as the renderer does;
* SolarRendererPrompt - the six-part prompt assembled from its free parts with the fixed sentences training used
  (skills/solarrenderer-caption);
* SolarRendererWebCapture - a web page's 3D canvas (three.js, any WebGL) recorded as a structure video, with its
  sound, optionally played by a key script and drawn as a rough early build (web_capture.py).
"""

import importlib.util
import json
import logging
import os
import re

import torch

import comfy.lora
import comfy.patcher_extension
import comfy.utils
import folder_paths
from comfy.ldm.minimax.model import time_shift_sigma
from comfy_extras.nodes_minimax_h3 import _empty_av_latent

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SILENCE = os.path.join(ROOT, "assets", "h3_silence_153_158_170.safetensors")
FPS = 24
LONG, SHOWN_LONG = 362, 158  # past LONG frames, the text encoder sees the first SHOWN_LONG
STILL_FRAMES = 13  # H3 takes no reference video shorter than this; a first frame is shown to it held that long
SHIFT_VIDEO, SHIFT_AUDIO = 12.0, 3.0
CATEGORY = "SolarRenderer"


# ------------------------------------------------------------------------------------------------- LoRA

_BLOCK = re.compile(r"^(transformer_blocks|token_refiner\.refiner_blocks)\.(\d+)\.(.+)$")


def lora_patches(lora: dict, inner: int, ffn: int) -> tuple[dict, dict]:
    """A SolarRenderer LoRA (diffusers PEFT names) as ComfyUI LoRA entries and the weights they patch.

    Returns (state dict, key map) for comfy.lora.load_lora. A key map entry may name a slice of a fused weight,
    (key, (dim, offset, size)), so nothing is stacked or padded: q, k, v land on rows [0, inner), [inner, 2 inner),
    [2 inner, 3 inner) of qkv_proj, and the feed-forward input, [value; gate] in diffusers, on rows [ffn, 2 ffn) and
    [0, ffn) of fc1, which ComfyUI holds as [gate; value]."""
    entries, key_map = {}, {}
    for key in lora:
        if not key.endswith(".lora_A.weight"):
            continue
        base = key[: -len(".lora_A.weight")]
        match = _BLOCK.match(base)
        if match is None:
            raise ValueError(f"{key}: not a MiniMax-H3 transformer block or token refiner module")
        prefix, index, module = match.groups()
        root = "diffusion_model." + ("blocks" if prefix == "transformer_blocks" else "token_refiner.blocks") + f".{index}"
        a, b = lora[key], lora[base + ".lora_B.weight"]
        if module in ("attn.to_q", "attn.to_k", "attn.to_v"):
            third = "qkv".index(module[-1])
            entries[base + ".lora_A.weight"], entries[base + ".lora_B.weight"] = a, b
            key_map[base] = (f"{root}.attn.qkv_proj.weight", (0, third * inner, inner))
        elif module == "ff.net.0.proj":
            half = b.shape[0] // 2
            for part, rows, start in (("value", b[:half], ffn), ("gate", b[half:], 0)):
                entries[f"{base}.{part}.lora_A.weight"], entries[f"{base}.{part}.lora_B.weight"] = a, rows
                key_map[f"{base}.{part}"] = (f"{root}.mlp.fc1.weight", (0, start, ffn))
        elif module in ("attn.to_out.0", "ff.net.2"):
            entries[base + ".lora_A.weight"], entries[base + ".lora_B.weight"] = a, b
            key_map[base] = f"{root}." + ("attn.out_proj.weight" if module == "attn.to_out.0" else "mlp.fc2.weight")
        else:
            raise ValueError(f"{key}: no ComfyUI counterpart for {module}")
    return entries, key_map


class SolarRendererLoraLoader:
    @classmethod
    def INPUT_TYPES(cls):
        return {"required": {
            "model": ("MODEL",),
            "lora_name": (folder_paths.get_filename_list("loras"),
                          {"tooltip": "A released SolarRenderer LoRA, as downloaded from the Hub "
                                      "(solarrenderer_4step_lora / solarrenderer_control_lora)."}),
            "strength": ("FLOAT", {"default": 1.0, "min": -4.0, "max": 4.0, "step": 0.05}),
        }}

    RETURN_TYPES = ("MODEL",)
    FUNCTION = "load"
    CATEGORY = CATEGORY

    def load(self, model, lora_name, strength):
        lora = comfy.utils.load_torch_file(folder_paths.get_full_path_or_raise("loras", lora_name), safe_load=True)
        if not any(k.startswith("transformer_blocks.") for k in lora):  # already in ComfyUI's own names
            key_map = comfy.lora.model_lora_keys_unet(model.model, {})
            patches = comfy.lora.load_lora(lora, key_map)
        else:
            dm = model.model.diffusion_model
            inner = dm.blocks[0].attn.out_proj.weight.shape[1]
            ffn = dm.blocks[0].mlp.fc2.weight.shape[1]
            entries, key_map = lora_patches(lora, inner, ffn)
            patches = comfy.lora.load_lora(entries, key_map)
        model = model.clone()
        missed = set(patches) - set(model.add_patches(patches, strength))
        if missed:
            raise ValueError(f"{lora_name}: {len(missed)} modules found no weight in this model")
        logging.info(f"SolarRenderer: {lora_name} patched {len(patches)} weights")
        return (model,)


# ------------------------------------------------------------------------------------------------- conditioning

def _fit(frames, width, height):
    """[T, H, W, C] -> [T, height, width, 3]: cut to the canvas's aspect at the centre and LANCZOS-resized, as the
    renderer lays a structure video or a first frame over the target's canvas."""
    samples = frames[..., :3].movedim(-1, 1)
    if tuple(samples.shape[-2:]) != (height, width):
        samples = comfy.utils.common_upscale(samples, width, height, "lanczos", "center")
    return samples.movedim(1, -1).clamp(0.0, 1.0)


def _shown(frames):
    """A reference as H3's Qwen3-VL reads it: every 12th of the 24 fps frames, timestamped at 2 fps."""
    index = list(range(0, frames.shape[0], FPS // 2))
    return {"type": "video", "data": frames[index], "timestamps": [i / 2.0 for i in range(len(index))]}


class SolarRendererConditioning:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "clip": ("CLIP",),
                "vae": ("VAE",),
                "structure": ("IMAGE", {"tooltip": "The structure video at 24 fps: a grey box, a segmentation, "
                                                    "skeletons, depth, a game's frames, a sketch. The render follows "
                                                    "it frame for frame."}),
                "prompt": ("STRING", {"multiline": True, "dynamic_prompts": True,
                                      "tooltip": "The six-part prompt (SolarRenderer Prompt builds it): <Video 1> is "
                                                 "the structure video, <Video 2> the first frame."}),
                "width": ("INT", {"default": 1344, "min": 256, "max": 2048, "step": 32}),
                "height": ("INT", {"default": 768, "min": 256, "max": 2048, "step": 32}),
                "length": ("INT", {"default": 158, "min": 22, "max": 3609, "step": 1,
                                   "tooltip": "Frames to render at 24 fps, at most the structure video's; snapped down "
                                              "to the 17n + 5 frames the video VAE takes. Trained at 158 (6.6 s); "
                                              "past 362, render with SolarRenderer Causal Sampler."}),
            },
            "optional": {
                "first_frame": ("IMAGE", {"tooltip": "The look of the shot: its first frame, already in that look."}),
                "structure_vae": ("VAE", {"tooltip": "Encode the structure video with this one instead (taeh3: ten "
                                                     "times faster, a little less exact); the first frame keeps vae."}),
            },
        }

    RETURN_TYPES = ("CONDITIONING", "LATENT")
    RETURN_NAMES = ("positive", "latent")
    FUNCTION = "encode"
    CATEGORY = CATEGORY

    def encode(self, clip, vae, structure, prompt, width, height, length, first_frame=None, structure_vae=None):
        frames = min(int(length), int(structure.shape[0]))
        if frames < 22:
            raise ValueError(f"SolarRenderer needs at least 22 frames of structure video, got {frames}")
        while frames % 17 != 5:
            frames -= 1
        video = _fit(structure[:frames], width, height)
        # a long clip (the causal sampler's) is described to the text encoder by its opening, as the renderer streams it
        items = [_shown(video if frames <= LONG else video[:SHOWN_LONG])]
        still = None
        if first_frame is not None:
            still = _fit(first_frame[:1], width, height)
            items.append(_shown(still.repeat(STILL_FRAMES, 1, 1, 1)))
        positive = clip.encode_from_tokens_scheduled(clip.tokenize(prompt, minimax_ref_items=items))
        # both laid over the target from its frame 0: the structure over every frame, the first frame over the first
        keyframes = [{"resolved_frame_index": 0, "latent": (structure_vae or vae).encode(video)}]
        if still is not None:
            keyframes.append({"resolved_frame_index": 0, "latent": vae.encode(still)})
        positive = [[c[0], {**c[1], "minimax_keyframes": keyframes}] for c in positive]
        latent, _ = _empty_av_latent(width, height, frames)
        return (positive, latent)


# ------------------------------------------------------------------------------------------------- sampling

class _SilentSoundtrack:
    """Holds the audio stream at H3's encoded silence, noised to the step's audio time with one fixed draw, before
    every evaluation: the renderer's soundtrack, which it never samples."""

    def __init__(self, silence, seed):
        self.silence, self.seed, self.noise = silence, seed, None

    def __call__(self, executor, x, timestep, context, transformer_options={}, **kwargs):
        video, audio = x[0], x[1]
        t = audio.shape[-1]
        if self.silence.shape[-1] < t:
            raise ValueError(f"the held silence covers 170 frames ({self.silence.shape[-1]} audio latents); "
                             f"this clip needs {t}")
        clean = self.silence[..., :t].to(audio.device, torch.float32)
        if self.noise is None or self.noise.shape != clean.shape:
            generator = torch.Generator("cpu").manual_seed(self.seed)
            self.noise = torch.randn(clean.shape, generator=generator, dtype=torch.float32)
        sigma_v = (timestep.flatten()[0] / 1000.0).float().clamp(min=1e-6)
        shift_v = float(transformer_options.get("minimax_h3_sigma_shift_video", SHIFT_VIDEO))
        shift_a = float(transformer_options.get("minimax_h3_sigma_shift_audio", SHIFT_AUDIO))
        sigma_a = float(time_shift_sigma(sigma_v, shift_v, shift_a))
        held = ((1.0 - sigma_a) * clean + sigma_a * self.noise.to(clean.device)).to(audio.dtype)
        return executor([video, held], timestep, context, transformer_options, **kwargs)


class SolarRendererModel:
    @classmethod
    def INPUT_TYPES(cls):
        return {"required": {"model": ("MODEL",),
                             "seed": ("INT", {"default": 0, "min": 0, "max": 0xFFFFFFFFFFFFFFFF,
                                              "tooltip": "The draw that noises the held silence."})}}

    RETURN_TYPES = ("MODEL",)
    FUNCTION = "patch"
    CATEGORY = CATEGORY

    def patch(self, model, seed):
        silence = comfy.utils.load_torch_file(SILENCE)["silence_170f"].float()  # [channels 2, 32, T]
        silence = silence.permute(1, 0, 2).unsqueeze(0)  # ComfyUI's audio latent is [1, 32, channels, T]
        model = model.clone()
        model.add_wrapper(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, _SilentSoundtrack(silence, seed))
        return (model,)


def shifted_sigmas(steps: int, shift: float = SHIFT_VIDEO):
    base = torch.linspace(1.0, 0.0, steps + 1, dtype=torch.float32)
    return shift * base / (1.0 + (shift - 1.0) * base)


class SolarRendererSigmas:
    @classmethod
    def INPUT_TYPES(cls):
        return {"required": {"steps": ("INT", {"default": 4, "min": 1, "max": 50,
                                               "tooltip": "Model evaluations; the four-step LoRA was trained for 4."})}}

    RETURN_TYPES = ("SIGMAS",)
    FUNCTION = "sigmas"
    CATEGORY = CATEGORY

    def sigmas(self, steps):
        return (shifted_sigmas(steps),)


# ------------------------------------------------------------------------------------------------- speed

def _fa4():
    try:
        from flash_attn.cute import flash_attn_func  # FlashAttention-4 (pip install flash-attn-4)
        return flash_attn_func
    except Exception:  # noqa: BLE001 - not installed, or built for another torch
        return None


class _FastAttention:
    """The DiT's attention on another kernel: FlashAttention-4 when it is installed, otherwise PyTorch's cuDNN
    attention. Which is fastest depends on the GPU and the software on it; the output is the same."""

    def __init__(self, backend):
        self.backend, self.fa4 = backend, _fa4() if backend in ("auto", "flash_attn_4") else None

    def __call__(self, func, q, k, v, heads, mask=None, skip_reshape=False, **kwargs):
        if mask is not None or not skip_reshape or q.dtype not in (torch.bfloat16, torch.float16) or not q.is_cuda:
            return func(q, k, v, heads, mask=mask, skip_reshape=skip_reshape, **kwargs)
        b, h, s, d = q.shape  # [B, heads, tokens, head_dim]
        if self.fa4 is not None:
            out = self.fa4(*(x.transpose(1, 2) for x in (q, k, v)))  # FA4 takes [B, tokens, heads, head_dim]
            out = out[0] if isinstance(out, tuple) else out
            return out.reshape(b, s, h * d)
        from torch.nn.attention import SDPBackend, sdpa_kernel

        q, k, v = (x.contiguous() for x in (q, k, v))  # cuDNN takes no strided views of the fused qkv
        order = [SDPBackend.CUDNN_ATTENTION, SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH]
        with sdpa_kernel(order, set_priority=True):  # ComfyUI puts flash first; cuDNN first is the point here
            out = torch.nn.functional.scaled_dot_product_attention(q, k, v)
        return out.transpose(1, 2).reshape(b, s, h * d)


class SolarRendererFastAttention:
    @classmethod
    def INPUT_TYPES(cls):
        return {"required": {"model": ("MODEL",),
                             "backend": (["auto", "flash_attn_4", "cudnn"],
                                         {"tooltip": "auto: FlashAttention-4 if installed (pip install flash-attn-4), "
                                                     "else PyTorch's cuDNN attention."})}}

    RETURN_TYPES = ("MODEL",)
    FUNCTION = "patch"
    CATEGORY = CATEGORY

    def patch(self, model, backend):
        attention = _FastAttention(backend)
        if backend == "flash_attn_4" and attention.fa4 is None:
            raise RuntimeError("FlashAttention-4 is not installed: pip install flash-attn-4")
        logging.info(f"SolarRenderer: attention on {'FlashAttention-4' if attention.fa4 else 'cuDNN'}")
        model = model.clone()
        model.model_options.setdefault("transformer_options", {})["optimized_attention_override"] = attention
        return (model,)


# ------------------------------------------------------------------------------------------------- prompt

def _caption_helpers():
    spec = importlib.util.spec_from_file_location(
        "solarrenderer_conditions", os.path.join(ROOT, "skills", "solarrenderer-caption", "conditions.py"))
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


class SolarRendererPrompt:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "structure_type": (["scene_graybox", "semantic", "scene_whitebox", "mask_skeleton", "smpl_mesh",
                                    "near_depth", "custom"],
                                   {"tooltip": "What the structure video is. The six trained types use the sentences "
                                               "training used; anything else is 'custom'."}),
                "with_first_frame": ("BOOLEAN", {"default": True}),
                "summary": ("STRING", {"multiline": True,
                                       "tooltip": "One paragraph: who does what, and how the camera moves."}),
                "detailed_description": ("STRING", {"multiline": True,
                                                    "tooltip": "350-500 words: the look first (style, light, weather), "
                                                               "then [Shot 1] and what is seen, in order."}),
                "soundscape": ("STRING", {"multiline": True, "default": "N/A"}),
            },
            "optional": {
                "custom_looks_like": ("STRING", {"default": "", "tooltip": "For 'custom': what the structure video "
                                                                           "looks like, e.g. 'a black pencil line sketch of the scene on white paper'."}),
                "custom_not_shown": ("STRING", {"default": "", "tooltip": "For 'custom': what of its look must not "
                                                                          "appear, e.g. 'the pencil lines and white paper'."}),
            },
        }

    RETURN_TYPES = ("STRING",)
    RETURN_NAMES = ("prompt",)
    FUNCTION = "build"
    CATEGORY = CATEGORY

    def build(self, structure_type, with_first_frame, summary, detailed_description, soundscape,
              custom_looks_like="", custom_not_shown=""):
        helpers = _caption_helpers()
        if structure_type == "custom":
            if not custom_looks_like.strip() or not custom_not_shown.strip():
                raise ValueError("a custom structure needs custom_looks_like and custom_not_shown")
            condition = helpers.custom_condition(custom_looks_like.strip(), custom_not_shown.strip())
        else:
            condition = structure_type
        free = {"summary": summary, "detailed_description": detailed_description, "overall_soundscape": soundscape}
        words = len(detailed_description.split())
        if not 350 <= words <= 500:
            logging.warning(f"SolarRenderer Prompt: detailed_description has {words} words; it renders best with "
                            f"350-500")
        return (helpers.assemble(free, "full" if with_first_frame else "structure", condition),)


# ------------------------------------------------------------------------------------------------- web input

def _decode_sound(path):
    """A WebAudio recording (.webm) as ComfyUI audio: {"waveform": [1, channels, samples], "sample_rate": rate}."""
    import av
    import numpy as np

    chunks, rate = [], 48000
    with av.open(path) as box:
        stream = box.streams.audio[0]
        rate = stream.rate or rate
        for frame in box.decode(stream):
            array = frame.to_ndarray()
            if array.dtype.kind in "iu":
                array = array.astype(np.float32) / float(np.iinfo(array.dtype).max)
            if frame.format.is_packed:
                array = array.reshape(-1, len(frame.layout.channels)).T
            chunks.append(array.astype(np.float32))
    waveform = torch.from_numpy(np.concatenate(chunks, axis=1)) if chunks else torch.zeros(2, 1)
    return {"waveform": waveform.unsqueeze(0), "sample_rate": int(rate)}


class SolarRendererWebCapture:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "url": ("STRING", {"default": "", "tooltip": "A page drawing a 3D canvas: three.js, Babylon, any WebGL "
                                                             "(http(s):// or file://)."}),
                "frames": ("INT", {"default": 158, "min": 22, "max": 2400, "tooltip": "Frames to record at 24 fps."}),
                "width": ("INT", {"default": 1344, "min": 256, "max": 2048, "step": 32}),
                "height": ("INT", {"default": 768, "min": 256, "max": 2048, "step": 32}),
            },
            "optional": {
                "keys": ("STRING", {"multiline": True, "default": "",
                                    "tooltip": "What to press, one action per line: '1.5:KeyW:2' (W held 2 s), "
                                               "'3:down:ShiftLeft' / '4:up:ShiftLeft', '5:mouse:200:0' (look right), "
                                               "'6:click'."}),
                "click": ("STRING", {"default": "", "tooltip": "The selector of a start button to press first."}),
                "setup": ("STRING", {"default": "", "tooltip": "JavaScript run once the page has started."}),
                "warm": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 30.0, "step": 0.5,
                                   "tooltip": "Seconds the page runs before the first frame."}),
                "frame_exact": ("BOOLEAN", {"default": True, "tooltip": "Run the page on a virtual clock, 1/24 s per "
                                                                        "frame, so a slow machine still records exactly "
                                                                        "24 fps. Off records in real time, with the "
                                                                        "page's sound."}),
            },
        }

    RETURN_TYPES = ("IMAGE", "AUDIO")
    RETURN_NAMES = ("frames", "sound")
    FUNCTION = "capture"
    CATEGORY = CATEGORY

    def capture(self, url, frames, width, height, keys="", click="", setup="", warm=1.0, frame_exact=True):
        import subprocess
        import sys
        import tempfile

        import numpy as np

        if not url.strip():
            raise ValueError("SolarRenderer Web Capture needs a url")
        try:
            import playwright  # noqa: F401
        except ImportError as error:
            raise RuntimeError("SolarRenderer Web Capture records the page in headless Chromium: "
                               "pip install playwright && python -m playwright install chromium") from error
        with tempfile.TemporaryDirectory() as work:
            out = os.path.join(work, "capture")
            command = [sys.executable, os.path.join(os.path.dirname(os.path.abspath(__file__)), "web_capture.py"),
                       "--url", url.strip(), "--out", out, "--frames", str(frames), "--width", str(width),
                       "--height", str(height), "--keys", keys, "--click", click, "--setup", setup,
                       "--warm", str(warm)] + (["--exact"] if frame_exact else [])
            done = subprocess.run(command, capture_output=True, text=True)
            if done.returncode != 0:
                raise RuntimeError(f"web capture failed: {done.stderr.strip()[-2000:]}")
            report = json.loads(done.stdout.strip().splitlines()[-1])
            if not frame_exact and report["real_time"] < 0.9:
                logging.warning(f"SolarRenderer Web Capture grabbed at {report['real_time']}x real time: the page ran "
                                f"ahead of its frames, so the clip plays fast. Turn frame_exact on, or record smaller.")
            images = torch.from_numpy(np.load(f"{out}.npy")).float() / 255.0
            sound = _decode_sound(f"{out}.webm") if os.path.exists(f"{out}.webm") else \
                {"waveform": torch.zeros(1, 2, 1), "sample_rate": 48000}
        return (images, sound)


NODE_CLASS_MAPPINGS = {
    "SolarRendererLoraLoader": SolarRendererLoraLoader,
    "SolarRendererConditioning": SolarRendererConditioning,
    "SolarRendererModel": SolarRendererModel,
    "SolarRendererSigmas": SolarRendererSigmas,
    "SolarRendererFastAttention": SolarRendererFastAttention,
    "SolarRendererPrompt": SolarRendererPrompt,
    "SolarRendererWebCapture": SolarRendererWebCapture,
}
NODE_DISPLAY_NAME_MAPPINGS = {
    "SolarRendererLoraLoader": "SolarRenderer LoRA Loader",
    "SolarRendererConditioning": "SolarRenderer Structure to Video",
    "SolarRendererModel": "SolarRenderer Model (silent soundtrack)",
    "SolarRendererSigmas": "SolarRenderer Four-Step Sigmas",
    "SolarRendererFastAttention": "SolarRenderer Fast Attention",
    "SolarRendererPrompt": "SolarRenderer Prompt",
    "SolarRendererWebCapture": "SolarRenderer Web Capture (three.js / WebGL page)",
}