Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python

feat: render.py renders any length (--seconds)

Enderfga7cae4bed6c35
diff --git a/docs/inference.md b/docs/inference.md
index 24015ca..7804770 100644
--- a/docs/inference.md
+++ b/docs/inference.md
@@ -60,6 +60,22 @@ python scripts/render.py --h3 ./MiniMax-H3 --causal \
The inputs and the output are the same as above.
+## Render a longer clip
+
+`--seconds` sets the length, up to what the structure video holds (the default is the trained 6.6 s), with either pair:
+
+```bash
+python scripts/render.py --h3 ./MiniMax-H3 --causal \
+ --control weights/solarrenderer_causal_control_lora.safetensors \
+ --fewstep weights/solarrenderer_causal_4step_lora.safetensors \
+ --structure structure.mp4 --first first.png --prompt prompt.txt --out render.mp4 --seconds 60
+```
+
+The clip is rendered whole either way. The bidirectional pair samples every frame at once, so its memory and time grow
+with the length (attention over every frame). The causal pair streams it block by block, each block seeing the
+structure of its last 6 blocks and the frames of the 5 blocks before it, so its memory stays that of a few blocks and
+its time grows in step with the length. Write the prompt for the whole clip.
+
## Render the benchmark
`eval/render_benchmark.py` renders every benchmark case with each model loaded once per GPU; see [`eval/README.md`](../eval/README.md).
diff --git a/scripts/render.py b/scripts/render.py
index fa580f9..702a3dd 100644
--- a/scripts/render.py
+++ b/scripts/render.py
@@ -9,6 +9,9 @@
--control solarrenderer_causal_control_lora.safetensors --fewstep solarrenderer_causal_4step_lora.safetensors \
--structure structure.mp4 --first first.png --prompt prompt.txt --out render.mp4
+ # any length up to what the structure video holds (default 6.6 s)
+ python scripts/render.py ... --seconds 60
+
The inputs are encoded exactly as the training samples were (tools/omni_preencode.py): the structure video laid over
every target frame, the first frame over latent 0, the prompt through the released Qwen3-VL presentation. The
renderer is MiniMax-H3's Ref2VA transformer with the four-step LoRA merged into its weights and the control LoRA on
@@ -17,6 +20,9 @@ With --causal the causal pair renders block-causally: each block of at most 5 la
the structure of the last 6 blocks and the clean frames of the 5 blocks before it - never a later block - and is
sampled in four evaluations before it is committed to the cache.
+--seconds renders a longer clip, whole: the bidirectional pair samples every frame at once (its memory grows with the
+length), the causal pair streams it block by block (its memory stays that of a few blocks).
+
The text encoder and the transformer do not share the card: the clip is encoded first, the encoder is released, then
the transformer is loaded. One GPU runs it end to end. eval/render_benchmark.py uses the same functions to
render many clips with each model loaded once.
@@ -37,7 +43,8 @@ HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parent / "src"))
sys.path.insert(0, str(HERE.parent / "tools"))
-FRAMES = 158
+FRAMES = 158 # the trained length, 6.6 s; a clip is 17n + 5 frames, 5n + 2 latents
+FPS = 24
RANK = ALPHA = 384
SILENCE = HERE.parent / "assets" / "h3_silence_153_158_170.safetensors"
@@ -54,6 +61,8 @@ def parse() -> argparse.Namespace:
p.add_argument("--seed", type=int, default=0)
p.add_argument("--forwards", type=int, default=4, help="model evaluations; the four-step LoRA was trained for 4")
p.add_argument("--causal", action="store_true", help="the causal LoRA pair: stream block by block")
+ p.add_argument("--seconds", type=float, default=0.0,
+ help="the length (default 6.6 s), up to what the structure video holds")
p.add_argument("--device", default="cuda:0")
return p.parse_args()
@@ -82,14 +91,23 @@ def sample_of(name: str, prompt: str, structure: str | None, first: str | None,
return {"id": name, "target": target or structure, "text": prompt, "observations": observations}
-def encode_sample(encoder, sample: dict, member: str) -> None:
+def encode_sample(encoder, sample: dict, member: str, frames: int = FRAMES) -> None:
from safetensors.torch import save_file
pre, pipes = encoder
- tensors, meta = pre.encode(sample, pipes, FRAMES)
+ tensors, meta = pre.encode(sample, pipes, frames)
save_file(tensors, member, metadata={"omni": json.dumps(meta)})
+def clip_frames(seconds: float, held: int) -> tuple[int, int]:
+ """``(frames to encode, frames to keep)`` for a clip of ``seconds``: encoded at the next 17n + 5 length the
+ structure video (``held`` frames) covers, never under the trained 158, and cut to the length asked for."""
+
+ keep = min(round(seconds * FPS), held) if seconds else FRAMES
+ encoded = max(min(17 * -(-(keep - 5) // 17) + 5, 17 * ((held - 5) // 17) + 5), FRAMES)
+ return encoded, min(keep, encoded)
+
+
def free_cuda() -> None:
"""After the caller dropped its last reference to the encoder."""
@@ -113,12 +131,13 @@ def lora_pairs(path: str) -> tuple[dict, float]:
CAUSAL = {"mode": "teacher", "chunk": 5, "window": 6, "corrupt": 0.0} # the causal pair's training plan
-def load_renderer(h3: str, control: str, fewstep: str, device: str, causal: bool = False):
- """MiniMax-H3's Ref2VA transformer, the four-step LoRA merged in, the control LoRA on top; returns a renderer."""
+def load_renderer(h3: str, control: str, fewstep: str, device: str, causal: bool = False, frames: int = FRAMES):
+ """MiniMax-H3's Ref2VA transformer, the four-step LoRA merged in, the control LoRA on top; returns a renderer for
+ clips of up to ``frames``."""
import torch
- from solarwm.backends.minimax_h3.artifacts import load_silence_latents
+ from solarwm.backends.minimax_h3.artifacts import extend_silence, load_silence_latents
from solarwm.backends.minimax_h3.lora import inject_h3_lora, merge_lora_delta
from solarwm.backends.minimax_h3.observations import observation_profile_from_train_config
from solarwm.backends.minimax_h3.optional import load_conditioners, load_transformer
@@ -144,6 +163,9 @@ def load_renderer(h3: str, control: str, fewstep: str, device: str, causal: bool
model_cfg, device=torch.device(device), qwen=False, video_vae=True, audio_vae=False, schedulers=False
)
silence, _ = load_silence_latents(str(SILENCE), pixel_frames=FRAMES)
+ if frames > FRAMES: # a longer clip's silent soundtrack: the encoded silence is steady away from its ends
+ silence = extend_silence(load_silence_latents(str(SILENCE), pixel_frames=170)[0],
+ H3Stage0p5Core.audio_latents(5 * ((frames - 5) // 17) + 2))
if causal:
from solarwm.backends.minimax_h3.omni_causal import H3OmniCausalCore
@@ -155,8 +177,9 @@ def load_renderer(h3: str, control: str, fewstep: str, device: str, causal: bool
return core, conditioners, torch.device(device)
-def render_member(renderer, member: str, out: str, seed: int = 0, forwards: int = 4) -> float:
- """Sample one encoded clip and write its mp4; returns the sampling time in seconds."""
+def render_member(renderer, member: str, out: str, seed: int = 0, forwards: int = 4, keep: int = 0) -> float:
+ """Sample one encoded clip and write its mp4 (its first ``keep`` frames, if given); returns the sampling time in
+ seconds."""
import torch
@@ -186,7 +209,7 @@ def render_member(renderer, member: str, out: str, seed: int = 0, forwards: int
took = time.perf_counter() - began
decoded = decode_h3_latents(conditioners.video_vae, latents, device=device)
Path(out).parent.mkdir(parents=True, exist_ok=True)
- Path(out + ".tmp").write_bytes(_mp4_bytes(decoded))
+ Path(out + ".tmp").write_bytes(_mp4_bytes(decoded[:, :, : keep or None]))
os.replace(out + ".tmp", out)
return took
@@ -195,16 +218,24 @@ def main() -> None:
args = parse()
with tempfile.TemporaryDirectory() as work:
began = time.perf_counter()
+ if args.seconds:
+ # H3's pipeline holds a clip to 15 s; a longer structure video is encoded whole
+ os.environ["OMNI_MAX_SECONDS"] = str(max(args.seconds + 1, 15))
encoder = load_encoder(args.h3, args.device)
+ pre, _pipes = encoder
+ held = int(pre.frame_count(args.structure) * FPS / pre.frame_rate(args.structure))
+ frames, keep = clip_frames(args.seconds, held)
member = f"{work}/member.safetensors"
prompt = Path(args.prompt).read_text()
- encode_sample(encoder, sample_of(Path(args.out).stem, prompt, args.structure, args.first or None), member)
- del encoder
+ encode_sample(encoder, sample_of(Path(args.out).stem, prompt, args.structure, args.first or None), member,
+ frames)
+ del encoder, pre, _pipes
free_cuda()
- print(f"encoded in {time.perf_counter() - began:.0f} s", flush=True)
- renderer = load_renderer(args.h3, args.control, args.fewstep, args.device, causal=args.causal)
- took = render_member(renderer, member, args.out, args.seed, args.forwards)
- print(f"sampled {args.forwards} steps in {took:.0f} s; wrote {args.out}", flush=True)
+ print(f"encoded {frames} frames in {time.perf_counter() - began:.0f} s", flush=True)
+ renderer = load_renderer(args.h3, args.control, args.fewstep, args.device, causal=args.causal, frames=frames)
+ took = render_member(renderer, member, args.out, args.seed, args.forwards, keep)
+ print(f"sampled {args.forwards} steps in {took:.0f} s; wrote {keep} frames ({keep / FPS:.1f} s) to {args.out}",
+ flush=True)
if __name__ == "__main__":
diff --git a/src/solarwm/backends/minimax_h3/artifacts.py b/src/solarwm/backends/minimax_h3/artifacts.py
index 527b166..319d908 100644
--- a/src/solarwm/backends/minimax_h3/artifacts.py
+++ b/src/solarwm/backends/minimax_h3/artifacts.py
@@ -883,6 +883,19 @@ def read_geometry_latents(tensors: Mapping[str, Any], geometry: Any) -> dict[str
return values
+def extend_silence(silence: Any, length: int, *, edge: int = 20) -> Any:
+ """The AudioVAE silence of a longer clip: the encoded silence is constant away from its first and last ``edge``
+ columns (start and end transients), so a longer one keeps both ends and repeats a steady middle column."""
+
+ import torch
+
+ columns = int(silence.shape[-1])
+ if length <= columns:
+ return silence[..., :length] if length < columns else silence
+ middle = silence[..., columns // 2 : columns // 2 + 1].expand(*silence.shape[:-1], length - 2 * edge)
+ return torch.cat((silence[..., :edge], middle, silence[..., columns - edge :]), dim=-1).contiguous()
+
+
def load_silence_latents(
path: str | Path, *, pixel_frames: int = 158
) -> tuple[Any, Mapping[str, Any]]:
@@ -921,6 +934,7 @@ __all__ = [
"H3PlanMultiplexer",
"H3PreencodedStream",
"align_h3_camera",
+ "extend_silence",
"h3_encoder_contract",
"h3_geometry_encoding_profile",
"h3_silence_profile",
diff --git a/tools/omni_preencode.py b/tools/omni_preencode.py
index 97c2841..516a7e6 100644
--- a/tools/omni_preencode.py
+++ b/tools/omni_preencode.py
@@ -408,11 +408,62 @@ def reference(
return MiniMaxH3VideoReference(frames=prepared(frames, fps, num_frames), fps=FPS)
+def _encode_conditions_clipwise(max_whole_frames: int = 1500) -> None:
+ """Past a minute, a visual condition is normalized and encoded clip by clip from the host.
+
+ The released encode_vae_condition turns the whole video into float32 on the card before the VAE splits it into
+ 17-frame clips - three minutes is ~50 GB per copy, more than a card holds beside the text encoder. The VAE encodes
+ every clip on its own, so normalizing each clip as it goes to the card and joining the moments gives the same
+ numbers; shorter videos keep the released path."""
+ import torch
+ from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution
+ from diffusers.models.modeling_utils import get_parameter_dtype
+ from diffusers.modular_pipelines.minimax_h3 import encoders
+
+ whole = encoders.encode_vae_condition
+
+ def clipwise(vae, pixels, pixel_mean, pixel_std, encode_seed=42):
+ if pixels.shape[2] <= max_whole_frames:
+ return whole(vae, pixels, pixel_mean, pixel_std, encode_seed)
+ device, dtype, clip = pixels.device, get_parameter_dtype(vae.encoder), vae.config.clip_length
+ mean = torch.tensor(pixel_mean, device=device).view(1, -1, 1, 1, 1)
+ std = torch.tensor(pixel_std, device=device).view(1, -1, 1, 1, 1)
+ host = pixels.cpu()
+ if host.shape[2] % clip:
+ host = torch.cat([host, host[:, :, -1:].repeat(1, 1, (-host.shape[2]) % clip, 1, 1)], dim=2)
+ moments = torch.cat([
+ vae._encode_clip(((host[:, :, i:i + clip].to(device).to(torch.float32).div(255.0) - mean) / std).to(dtype))
+ for i in range(0, host.shape[2], clip)], dim=2)
+ if vae.config.token_drop > 0:
+ moments = moments[:, :, :-vae.config.token_drop]
+ latents = DiagonalGaussianDistribution(moments).sample(generator=torch.Generator().manual_seed(encode_seed))
+ latents = latents.to(torch.float16).float().cpu()
+ latents_mean = torch.tensor(vae.config.latents_mean).view(1, -1, 1, 1, 1)
+ latents_std = torch.tensor(vae.config.latents_std).view(1, -1, 1, 1, 1)
+ return (latents - latents_mean) / latents_std
+
+ encoders.encode_vae_condition = clipwise
+
+
def build_pipelines(device: str):
from diffusers import ComponentsManager, ModularPipeline
from diffusers.modular_pipelines import SequentialPipelineBlocks
blocks = ModularPipeline.from_pretrained(ROOT).blocks
+ if os.environ.get("OMNI_MAX_SECONDS"):
+ # A clip longer than 15 s (scripts/render.py --seconds): the pipeline holds generation to 15 s; encoding a
+ # longer structure video and target lifts that ceiling.
+ from diffusers.modular_pipelines.minimax_h3.modular_pipeline import MiniMaxH3ModularPipeline
+
+ limit = float(os.environ["OMNI_MAX_SECONDS"])
+ MiniMaxH3ModularPipeline.max_duration = property(lambda self: limit)
+ # The VAE encodes every 17-frame clip on its own; a clip sliced from a minute of frames keeps the whole video's
+ # strides and overflows 32-bit indexing in its padding - a contiguous copy of each clip encodes the same numbers.
+ from diffusers.models.autoencoders.autoencoder_kl_minimax_h3 import AutoencoderKLMiniMaxH3
+
+ clip_encode = AutoencoderKLMiniMaxH3._encode_clip
+ AutoencoderKLMiniMaxH3._encode_clip = lambda self, x: clip_encode(self, x.contiguous())
+ _encode_conditions_clipwise()
manager = ComponentsManager()
# Weights stay on the card. Offloading moved the text encoder over PCIe on every call, which
# cost little per sample once warm but made the first call of every process several times