SolarRenderer: Few-Step Generative Reference Rendering
Python
feat: render.py renders any length (--seconds)
Enderfga7cae4bed6c35
diff --git a/docs/inference.md b/docs/inference.mdindex 24015ca..7804770 100644--- a/docs/inference.md+++ b/docs/inference.md@@ -60,6 +60,22 @@ python scripts/render.py --h3 ./MiniMax-H3 --causal \The inputs and the output are the same as above.+## Render a longer clip++`--seconds` sets the length, up to what the structure video holds (the default is the trained 6.6 s), with either pair:++```bash+python scripts/render.py --h3 ./MiniMax-H3 --causal \+ --control weights/solarrenderer_causal_control_lora.safetensors \+ --fewstep weights/solarrenderer_causal_4step_lora.safetensors \+ --structure structure.mp4 --first first.png --prompt prompt.txt --out render.mp4 --seconds 60+```++The clip is rendered whole either way. The bidirectional pair samples every frame at once, so its memory and time grow+with the length (attention over every frame). The causal pair streams it block by block, each block seeing the+structure of its last 6 blocks and the frames of the 5 blocks before it, so its memory stays that of a few blocks and+its time grows in step with the length. Write the prompt for the whole clip.+## Render the benchmark`eval/render_benchmark.py` renders every benchmark case with each model loaded once per GPU; see [`eval/README.md`](../eval/README.md).diff --git a/scripts/render.py b/scripts/render.pyindex fa580f9..702a3dd 100644--- a/scripts/render.py+++ b/scripts/render.py@@ -9,6 +9,9 @@--control solarrenderer_causal_control_lora.safetensors --fewstep solarrenderer_causal_4step_lora.safetensors \--structure structure.mp4 --first first.png --prompt prompt.txt --out render.mp4+ # any length up to what the structure video holds (default 6.6 s)+ python scripts/render.py ... --seconds 60+The inputs are encoded exactly as the training samples were (tools/omni_preencode.py): the structure video laid overevery target frame, the first frame over latent 0, the prompt through the released Qwen3-VL presentation. Therenderer is MiniMax-H3's Ref2VA transformer with the four-step LoRA merged into its weights and the control LoRA on@@ -17,6 +20,9 @@ With --causal the causal pair renders block-causally: each block of at most 5 lathe structure of the last 6 blocks and the clean frames of the 5 blocks before it - never a later block - and issampled in four evaluations before it is committed to the cache.+--seconds renders a longer clip, whole: the bidirectional pair samples every frame at once (its memory grows with the+length), the causal pair streams it block by block (its memory stays that of a few blocks).+The text encoder and the transformer do not share the card: the clip is encoded first, the encoder is released, thenthe transformer is loaded. One GPU runs it end to end. eval/render_benchmark.py uses the same functions torender many clips with each model loaded once.@@ -37,7 +43,8 @@ HERE = Path(__file__).resolve().parentsys.path.insert(0, str(HERE.parent / "src"))sys.path.insert(0, str(HERE.parent / "tools"))-FRAMES = 158+FRAMES = 158 # the trained length, 6.6 s; a clip is 17n + 5 frames, 5n + 2 latents+FPS = 24RANK = ALPHA = 384SILENCE = HERE.parent / "assets" / "h3_silence_153_158_170.safetensors"@@ -54,6 +61,8 @@ def parse() -> argparse.Namespace:p.add_argument("--seed", type=int, default=0)p.add_argument("--forwards", type=int, default=4, help="model evaluations; the four-step LoRA was trained for 4")p.add_argument("--causal", action="store_true", help="the causal LoRA pair: stream block by block")+ p.add_argument("--seconds", type=float, default=0.0,+ help="the length (default 6.6 s), up to what the structure video holds")p.add_argument("--device", default="cuda:0")return p.parse_args()@@ -82,14 +91,23 @@ def sample_of(name: str, prompt: str, structure: str | None, first: str | None,return {"id": name, "target": target or structure, "text": prompt, "observations": observations}-def encode_sample(encoder, sample: dict, member: str) -> None:+def encode_sample(encoder, sample: dict, member: str, frames: int = FRAMES) -> None:from safetensors.torch import save_filepre, pipes = encoder- tensors, meta = pre.encode(sample, pipes, FRAMES)+ tensors, meta = pre.encode(sample, pipes, frames)save_file(tensors, member, metadata={"omni": json.dumps(meta)})+def clip_frames(seconds: float, held: int) -> tuple[int, int]:+ """``(frames to encode, frames to keep)`` for a clip of ``seconds``: encoded at the next 17n + 5 length the+ structure video (``held`` frames) covers, never under the trained 158, and cut to the length asked for."""++ keep = min(round(seconds * FPS), held) if seconds else FRAMES+ encoded = max(min(17 * -(-(keep - 5) // 17) + 5, 17 * ((held - 5) // 17) + 5), FRAMES)+ return encoded, min(keep, encoded)++def free_cuda() -> None:"""After the caller dropped its last reference to the encoder."""@@ -113,12 +131,13 @@ def lora_pairs(path: str) -> tuple[dict, float]:CAUSAL = {"mode": "teacher", "chunk": 5, "window": 6, "corrupt": 0.0} # the causal pair's training plan-def load_renderer(h3: str, control: str, fewstep: str, device: str, causal: bool = False):- """MiniMax-H3's Ref2VA transformer, the four-step LoRA merged in, the control LoRA on top; returns a renderer."""+def load_renderer(h3: str, control: str, fewstep: str, device: str, causal: bool = False, frames: int = FRAMES):+ """MiniMax-H3's Ref2VA transformer, the four-step LoRA merged in, the control LoRA on top; returns a renderer for+ clips of up to ``frames``."""import torch- from solarwm.backends.minimax_h3.artifacts import load_silence_latents+ from solarwm.backends.minimax_h3.artifacts import extend_silence, load_silence_latentsfrom solarwm.backends.minimax_h3.lora import inject_h3_lora, merge_lora_deltafrom solarwm.backends.minimax_h3.observations import observation_profile_from_train_configfrom solarwm.backends.minimax_h3.optional import load_conditioners, load_transformer@@ -144,6 +163,9 @@ def load_renderer(h3: str, control: str, fewstep: str, device: str, causal: boolmodel_cfg, device=torch.device(device), qwen=False, video_vae=True, audio_vae=False, schedulers=False)silence, _ = load_silence_latents(str(SILENCE), pixel_frames=FRAMES)+ if frames > FRAMES: # a longer clip's silent soundtrack: the encoded silence is steady away from its ends+ silence = extend_silence(load_silence_latents(str(SILENCE), pixel_frames=170)[0],+ H3Stage0p5Core.audio_latents(5 * ((frames - 5) // 17) + 2))if causal:from solarwm.backends.minimax_h3.omni_causal import H3OmniCausalCore@@ -155,8 +177,9 @@ def load_renderer(h3: str, control: str, fewstep: str, device: str, causal: boolreturn core, conditioners, torch.device(device)-def render_member(renderer, member: str, out: str, seed: int = 0, forwards: int = 4) -> float:- """Sample one encoded clip and write its mp4; returns the sampling time in seconds."""+def render_member(renderer, member: str, out: str, seed: int = 0, forwards: int = 4, keep: int = 0) -> float:+ """Sample one encoded clip and write its mp4 (its first ``keep`` frames, if given); returns the sampling time in+ seconds."""import torch@@ -186,7 +209,7 @@ def render_member(renderer, member: str, out: str, seed: int = 0, forwards: inttook = time.perf_counter() - begandecoded = decode_h3_latents(conditioners.video_vae, latents, device=device)Path(out).parent.mkdir(parents=True, exist_ok=True)- Path(out + ".tmp").write_bytes(_mp4_bytes(decoded))+ Path(out + ".tmp").write_bytes(_mp4_bytes(decoded[:, :, : keep or None]))os.replace(out + ".tmp", out)return took@@ -195,16 +218,24 @@ def main() -> None:args = parse()with tempfile.TemporaryDirectory() as work:began = time.perf_counter()+ if args.seconds:+ # H3's pipeline holds a clip to 15 s; a longer structure video is encoded whole+ os.environ["OMNI_MAX_SECONDS"] = str(max(args.seconds + 1, 15))encoder = load_encoder(args.h3, args.device)+ pre, _pipes = encoder+ held = int(pre.frame_count(args.structure) * FPS / pre.frame_rate(args.structure))+ frames, keep = clip_frames(args.seconds, held)member = f"{work}/member.safetensors"prompt = Path(args.prompt).read_text()- encode_sample(encoder, sample_of(Path(args.out).stem, prompt, args.structure, args.first or None), member)- del encoder+ encode_sample(encoder, sample_of(Path(args.out).stem, prompt, args.structure, args.first or None), member,+ frames)+ del encoder, pre, _pipesfree_cuda()- print(f"encoded in {time.perf_counter() - began:.0f} s", flush=True)- renderer = load_renderer(args.h3, args.control, args.fewstep, args.device, causal=args.causal)- took = render_member(renderer, member, args.out, args.seed, args.forwards)- print(f"sampled {args.forwards} steps in {took:.0f} s; wrote {args.out}", flush=True)+ print(f"encoded {frames} frames in {time.perf_counter() - began:.0f} s", flush=True)+ renderer = load_renderer(args.h3, args.control, args.fewstep, args.device, causal=args.causal, frames=frames)+ took = render_member(renderer, member, args.out, args.seed, args.forwards, keep)+ print(f"sampled {args.forwards} steps in {took:.0f} s; wrote {keep} frames ({keep / FPS:.1f} s) to {args.out}",+ flush=True)if __name__ == "__main__":diff --git a/src/solarwm/backends/minimax_h3/artifacts.py b/src/solarwm/backends/minimax_h3/artifacts.pyindex 527b166..319d908 100644--- a/src/solarwm/backends/minimax_h3/artifacts.py+++ b/src/solarwm/backends/minimax_h3/artifacts.py@@ -883,6 +883,19 @@ def read_geometry_latents(tensors: Mapping[str, Any], geometry: Any) -> dict[strreturn values+def extend_silence(silence: Any, length: int, *, edge: int = 20) -> Any:+ """The AudioVAE silence of a longer clip: the encoded silence is constant away from its first and last ``edge``+ columns (start and end transients), so a longer one keeps both ends and repeats a steady middle column."""++ import torch++ columns = int(silence.shape[-1])+ if length <= columns:+ return silence[..., :length] if length < columns else silence+ middle = silence[..., columns // 2 : columns // 2 + 1].expand(*silence.shape[:-1], length - 2 * edge)+ return torch.cat((silence[..., :edge], middle, silence[..., columns - edge :]), dim=-1).contiguous()++def load_silence_latents(path: str | Path, *, pixel_frames: int = 158) -> tuple[Any, Mapping[str, Any]]:@@ -921,6 +934,7 @@ __all__ = ["H3PlanMultiplexer","H3PreencodedStream","align_h3_camera",+ "extend_silence","h3_encoder_contract","h3_geometry_encoding_profile","h3_silence_profile",diff --git a/tools/omni_preencode.py b/tools/omni_preencode.pyindex 97c2841..516a7e6 100644--- a/tools/omni_preencode.py+++ b/tools/omni_preencode.py@@ -408,11 +408,62 @@ def reference(return MiniMaxH3VideoReference(frames=prepared(frames, fps, num_frames), fps=FPS)+def _encode_conditions_clipwise(max_whole_frames: int = 1500) -> None:+ """Past a minute, a visual condition is normalized and encoded clip by clip from the host.++ The released encode_vae_condition turns the whole video into float32 on the card before the VAE splits it into+ 17-frame clips - three minutes is ~50 GB per copy, more than a card holds beside the text encoder. The VAE encodes+ every clip on its own, so normalizing each clip as it goes to the card and joining the moments gives the same+ numbers; shorter videos keep the released path."""+ import torch+ from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution+ from diffusers.models.modeling_utils import get_parameter_dtype+ from diffusers.modular_pipelines.minimax_h3 import encoders++ whole = encoders.encode_vae_condition++ def clipwise(vae, pixels, pixel_mean, pixel_std, encode_seed=42):+ if pixels.shape[2] <= max_whole_frames:+ return whole(vae, pixels, pixel_mean, pixel_std, encode_seed)+ device, dtype, clip = pixels.device, get_parameter_dtype(vae.encoder), vae.config.clip_length+ mean = torch.tensor(pixel_mean, device=device).view(1, -1, 1, 1, 1)+ std = torch.tensor(pixel_std, device=device).view(1, -1, 1, 1, 1)+ host = pixels.cpu()+ if host.shape[2] % clip:+ host = torch.cat([host, host[:, :, -1:].repeat(1, 1, (-host.shape[2]) % clip, 1, 1)], dim=2)+ moments = torch.cat([+ vae._encode_clip(((host[:, :, i:i + clip].to(device).to(torch.float32).div(255.0) - mean) / std).to(dtype))+ for i in range(0, host.shape[2], clip)], dim=2)+ if vae.config.token_drop > 0:+ moments = moments[:, :, :-vae.config.token_drop]+ latents = DiagonalGaussianDistribution(moments).sample(generator=torch.Generator().manual_seed(encode_seed))+ latents = latents.to(torch.float16).float().cpu()+ latents_mean = torch.tensor(vae.config.latents_mean).view(1, -1, 1, 1, 1)+ latents_std = torch.tensor(vae.config.latents_std).view(1, -1, 1, 1, 1)+ return (latents - latents_mean) / latents_std++ encoders.encode_vae_condition = clipwise++def build_pipelines(device: str):from diffusers import ComponentsManager, ModularPipelinefrom diffusers.modular_pipelines import SequentialPipelineBlocksblocks = ModularPipeline.from_pretrained(ROOT).blocks+ if os.environ.get("OMNI_MAX_SECONDS"):+ # A clip longer than 15 s (scripts/render.py --seconds): the pipeline holds generation to 15 s; encoding a+ # longer structure video and target lifts that ceiling.+ from diffusers.modular_pipelines.minimax_h3.modular_pipeline import MiniMaxH3ModularPipeline++ limit = float(os.environ["OMNI_MAX_SECONDS"])+ MiniMaxH3ModularPipeline.max_duration = property(lambda self: limit)+ # The VAE encodes every 17-frame clip on its own; a clip sliced from a minute of frames keeps the whole video's+ # strides and overflows 32-bit indexing in its padding - a contiguous copy of each clip encodes the same numbers.+ from diffusers.models.autoencoders.autoencoder_kl_minimax_h3 import AutoencoderKLMiniMaxH3++ clip_encode = AutoencoderKLMiniMaxH3._encode_clip+ AutoencoderKLMiniMaxH3._encode_clip = lambda self, x: clip_encode(self, x.contiguous())+ _encode_conditions_clipwise()manager = ComponentsManager()# Weights stay on the card. Offloading moved the text encoder over PCIe on every call, which# cost little per sample once warm but made the first call of every process several times