SolarRenderer: Few-Step Generative Reference Rendering
Python
"""Render one clip with SolarRenderer: a structure video, an optional first frame and a six-part prompt in, a video out.
python scripts/render.py --h3 ./MiniMax-H3 \
--control solarrenderer_control_lora.safetensors --fewstep solarrenderer_4step_lora.safetensors \
--structure structure.mp4 --first first.png --prompt prompt.txt --out render.mp4
# streaming: the causal pair, rendered block by block (5 latents, about 20 frames) through a key/value cache
python scripts/render.py --h3 ./MiniMax-H3 --causal \
--control solarrenderer_causal_control_lora.safetensors --fewstep solarrenderer_causal_4step_lora.safetensors \
--structure structure.mp4 --first first.png --prompt prompt.txt --out render.mp4
# any length up to what the structure video holds (default 6.6 s)
python scripts/render.py ... --seconds 60
The inputs are encoded exactly as the training samples were (tools/omni_preencode.py): the structure video laid over
every target frame, the first frame over latent 0, the prompt through the released Qwen3-VL presentation. The
renderer is MiniMax-H3's Ref2VA transformer with the four-step LoRA and the control LoRA merged into its weights at
once, sampled in four evaluations with fresh noise between them. Output: 1344x768 (or 768x1344), 24 fps, 158 frames.
With --causal the causal pair renders block-causally: each block of at most 5 latents sees the prompt, the first frame,
the structure of the last 6 blocks and the clean frames of the 5 blocks before it - never a later block - and is
sampled in four evaluations before it is committed to the cache.
--seconds renders a longer clip, whole: the bidirectional pair samples every frame at once (its memory grows with the
length), the causal pair streams it block by block (its memory stays that of a few blocks).
The text encoder and the transformer do not share the card: the clip is encoded first, the encoder is released, then
the transformer is loaded. One GPU runs it end to end. eval/render_benchmark.py uses the same functions to
render many clips with each model loaded once.
"""
from __future__ import annotations
import argparse
import gc
import json
import os
import sys
import tempfile
import time
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parent / "src"))
sys.path.insert(0, str(HERE.parent / "tools"))
FRAMES = 158 # the trained length, 6.6 s; a clip is 17n + 5 frames, 5n + 2 latents
FPS = 24
RANK = ALPHA = 384
SILENCE = HERE.parent / "assets" / "h3_silence_153_158_170.safetensors"
def parse() -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--h3", required=True, help="local MiniMaxAI/MiniMax-H3 (scripts/prepare_h3.py)")
p.add_argument("--control", required=True, help="solarrenderer_control_lora.safetensors")
p.add_argument("--fewstep", required=True, help="solarrenderer_4step_lora.safetensors")
p.add_argument("--structure", required=True, help="the structure video, frame-aligned with the target")
p.add_argument("--first", default="", help="the first frame (optional)")
p.add_argument("--prompt", required=True, help="a text file holding the six-part prompt (skills/solarrenderer-caption)")
p.add_argument("--out", required=True, help="the output .mp4")
p.add_argument("--seed", type=int, default=0)
p.add_argument("--forwards", type=int, default=4, help="model evaluations; the four-step LoRA was trained for 4")
p.add_argument("--causal", action="store_true", help="the causal LoRA pair: stream block by block")
p.add_argument("--seconds", type=float, default=0.0,
help="the length (default 6.6 s), up to what the structure video holds")
p.add_argument("--device", default="cuda:0")
return p.parse_args()
# ---------------------------------------------------------------------------------------------- encoding
def load_encoder(h3: str, device: str):
"""The released ref2va front end (text encoder, processor, reference VAE encoder), as training encoded with it."""
os.environ["H3_REF_ROOT"] = str(Path(h3).resolve())
import omni_preencode as pre
pre.ROOT = os.environ["H3_REF_ROOT"]
return pre, pre.build_pipelines(device)
def sample_of(name: str, prompt: str, structure: str | None, first: str | None, target: str | None = None) -> dict:
"""A clip's inputs as a training sample. The target only sets the canvas and the length - sampling never reads its
latents - so it is the structure video itself, or a given clip when there is no structure video."""
observations = []
if structure:
observations.append({"media": structure, "frames": "all"})
if first:
observations.append({"media": first, "frames": [0]})
return {"id": name, "target": target or structure, "text": prompt, "observations": observations}
def encode_sample(encoder, sample: dict, member: str, frames: int = FRAMES) -> None:
from safetensors.torch import save_file
pre, pipes = encoder
tensors, meta = pre.encode(sample, pipes, frames)
save_file(tensors, member, metadata={"omni": json.dumps(meta)})
def clip_frames(seconds: float, held: int) -> tuple[int, int]:
"""``(frames to encode, frames to keep)`` for a clip of ``seconds``: encoded at the next 17n + 5 length the
structure video (``held`` frames) covers, never under the trained 158, and cut to the length asked for."""
keep = min(round(seconds * FPS), held) if seconds else FRAMES
encoded = max(min(17 * -(-(keep - 5) // 17) + 5, 17 * ((held - 5) // 17) + 5), FRAMES)
return encoded, min(keep, encoded)
def free_cuda() -> None:
"""After the caller dropped its last reference to the encoder."""
import torch
gc.collect()
torch.cuda.empty_cache()
# ---------------------------------------------------------------------------------------------- rendering
def lora_pairs(path: str) -> tuple[dict, float]:
from safetensors import safe_open
from safetensors.torch import load_file
with safe_open(path, framework="pt") as handle:
meta = handle.metadata() or {}
return load_file(path), float(meta.get("alpha", ALPHA))
CAUSAL = {"mode": "teacher", "chunk": 5, "window": 6, "corrupt": 0.0} # the causal pair's training plan
def load_renderer(h3: str, control: str, fewstep: str, device: str, causal: bool = False, frames: int = FRAMES):
"""MiniMax-H3's Ref2VA transformer with the four-step and control LoRAs merged in; returns a renderer for clips of
up to ``frames``."""
import torch
from solarwm.backends.minimax_h3.artifacts import extend_silence, load_silence_latents
from solarwm.backends.minimax_h3.lora import merge_lora_delta
from solarwm.backends.minimax_h3.observations import observation_profile_from_train_config
from solarwm.backends.minimax_h3.optional import load_conditioners, load_transformer
from solarwm.backends.minimax_h3.stage0p5 import H3Stage0p5Core
torch.cuda.set_device(torch.device(device))
model_cfg = {
"checkpoint_path": str(Path(h3).resolve()),
"transformer_subfolder": "transformer_ref",
"torch_dtype": "bfloat16",
"attention_backend": "flash",
"adapter": {"rank": RANK, "alpha": ALPHA, "dropout": 0.0},
}
modules = load_transformer(model_cfg, device=torch.device(device))
# Both LoRAs are merged into the weights at once: their deltas summed in float32 and rounded into bf16 a single
# time, nothing left running on top (a LoRA computed beside bf16 weights, or merged one at a time, softens the
# picture). Each pair's up projection carries its own alpha/rank, so the stack adds exactly B1 A1 + B2 A2.
stacked: dict = {}
for path in (fewstep, control):
pairs, alpha = lora_pairs(path)
rank = int(next(v for k, v in pairs.items() if k.endswith("lora_A.weight")).shape[0])
for key, value in pairs.items():
value = value if key.endswith("lora_A.weight") else value.float() * (alpha / rank)
stacked[key] = torch.cat((stacked[key], value.float()), dim=0 if key.endswith("lora_A.weight") else 1) \
if key in stacked else value.float()
merged = merge_lora_delta(modules.transformer, stacked, alpha=None, rank=None, scale=1.0)
print(f"both LoRAs merged into {merged} weights", flush=True)
wrapped = modules.transformer
conditioners = load_conditioners(
model_cfg, device=torch.device(device), qwen=False, video_vae=True, audio_vae=False, schedulers=False
)
silence, _ = load_silence_latents(str(SILENCE), pixel_frames=FRAMES)
if frames > FRAMES: # a longer clip's silent soundtrack: the encoded silence is steady away from its ends
silence = extend_silence(load_silence_latents(str(SILENCE), pixel_frames=170)[0],
H3Stage0p5Core.audio_latents(5 * ((frames - 5) // 17) + 2))
if causal:
from solarwm.backends.minimax_h3.omni_causal import H3OmniCausalCore
core = H3OmniCausalCore(wrapped, silence, torch.device(device), dict(CAUSAL), geometry=None,
observations=observation_profile_from_train_config({}))
else:
core = H3Stage0p5Core(wrapped, silence, torch.device(device), geometry=None,
observations=observation_profile_from_train_config({}), flow_map=False)
return core, conditioners, torch.device(device)
def render_member(renderer, member: str, out: str, seed: int = 0, forwards: int = 4, keep: int = 0) -> float:
"""Sample one encoded clip and write its mp4 (its first ``keep`` frames, if given); returns the sampling time in
seconds."""
import torch
from solarwm.backends.minimax_h3.inference import _mp4_bytes, decode_h3_latents
from solarwm.backends.minimax_h3.omni_data import H3OmniStream
core, conditioners, device = renderer
stream = object.__new__(H3OmniStream)
stream.root, stream.current_plan_fingerprint, stream.aligned_control = "/", "render", False
batch = stream.read({"id": Path(out).stem, "member": member})
began = time.perf_counter()
with torch.no_grad():
if hasattr(core, "stream"):
# block by block: each block's clean estimate re-noised between evaluations, as the student was trained;
# the structure keys and values are recomputed per evaluation so one card holds the clip
from solarwm.backends.minimax_h3.flow import make_shifted_schedule
torch.manual_seed(seed)
latents, _inputs, _times, _jitters = core.stream(
batch, generator=torch.Generator(device=device).manual_seed(seed),
schedule=make_shifted_schedule(forwards + 1, shift=12.0), exit_index=forwards - 1,
cache_structure=False,
)
else:
# generate() counts schedule points: N points run N - 1 evaluations
latents = core.generate(batch, noise_seed=seed, num_inference_steps=forwards + 1, renoise=True)
took = time.perf_counter() - began
decoded = decode_h3_latents(conditioners.video_vae, latents, device=device)
Path(out).parent.mkdir(parents=True, exist_ok=True)
Path(out + ".tmp").write_bytes(_mp4_bytes(decoded[:, :, : keep or None]))
os.replace(out + ".tmp", out)
return took
def main() -> None:
args = parse()
with tempfile.TemporaryDirectory() as work:
began = time.perf_counter()
if args.seconds:
# H3's pipeline holds a clip to 15 s; a longer structure video is encoded whole
os.environ["OMNI_MAX_SECONDS"] = str(max(args.seconds + 1, 15))
encoder = load_encoder(args.h3, args.device)
pre, _pipes = encoder
held = int(pre.frame_count(args.structure) * FPS / pre.frame_rate(args.structure))
frames, keep = clip_frames(args.seconds, held)
member = f"{work}/member.safetensors"
prompt = Path(args.prompt).read_text()
encode_sample(encoder, sample_of(Path(args.out).stem, prompt, args.structure, args.first or None), member,
frames)
del encoder, pre, _pipes
free_cuda()
print(f"encoded {frames} frames in {time.perf_counter() - began:.0f} s", flush=True)
renderer = load_renderer(args.h3, args.control, args.fewstep, args.device, causal=args.causal, frames=frames)
took = render_member(renderer, member, args.out, args.seed, args.forwards, keep)
print(f"sampled {args.forwards} steps in {took:.0f} s; wrote {keep} frames ({keep / FPS:.1f} s) to {args.out}",
flush=True)
if __name__ == "__main__":
main()