WorldPlay2: Extending Real-Time Interactive World Models in Control and Horizon
Python
"""Per-chunk streaming helpers for ``WorldPlay2Pipeline.stream_world``."""
from __future__ import annotations
import logging
import torch
import torch.nn.functional as F
import torchvision.transforms.functional as TF
from .vae import _same_causal_state, amp
class FrameEncoder:
"""One step of the causal VAE encoder: pixel frames -> one y latent ``[16, 1, h, w]``
in the VAE's dtype, carrying the causal state passed in."""
def __init__(self, vae):
self.vae = vae
def new_state(self):
from .vae import count_conv3d
return [None] * count_conv3d(self.vae.model.encoder)
def step(self, x, state):
model = self.vae.model
model._enc_conv_idx = [0]
out = model.encoder(x.to(self.vae.dtype), feat_cache=state, feat_idx=model._enc_conv_idx)
mu, _ = model.conv1(out).chunk(2, dim=1)
scale = self.vae.scale
mu = (mu - scale[0].view(1, model.z_dim, 1, 1, 1)) * scale[1].view(1, model.z_dim, 1, 1, 1)
return mu.to(self.vae.dtype).squeeze(0)
def agree(self, flag):
return flag
def _y_row(latent, first, dtype):
mask = torch.zeros(4, 1, *latent.shape[2:], device=latent.device, dtype=latent.dtype)
if first:
mask.fill_(1)
return torch.cat([mask, latent], dim=0).to(dtype)
class _CausalY:
"""``y = [4-ch I2V mask | VAE latent of the clip [image, zeros...]]`` produced latent by latent.
Indexed like the full-horizon volume (``y[:, a:b]``).
"""
def __init__(self, encoder, head, tail, dtype):
self.encoder, self.head, self.dtype = encoder, head.to(encoder.vae.dtype), dtype
self.fixed_state, self.tail = tail
self.state = None
self.rows = []
self.converged = False
def _extend(self, upto):
vae = self.encoder.vae
with amp.autocast(dtype=vae.dtype):
while not self.converged and len(self.rows) < upto:
k = len(self.rows)
if k == 0:
self.state = self.encoder.new_state()
latent = self.encoder.step(self.head, self.state)
else:
zeros = self.head.new_zeros(*self.head.shape[:2], 4, *self.head.shape[3:])
latent = self.encoder.step(zeros, self.state)
self.rows.append(_y_row(latent, k == 0, self.dtype))
if k > 1 and self.encoder.agree(_same_causal_state(self.fixed_state, self.state)):
self.converged = True
self.state = None
def __getitem__(self, index):
_, sl = index
start, stop = sl.start or 0, sl.stop
self._extend(stop)
known = len(self.rows)
parts = [self.rows[k] for k in range(start, min(stop, known))]
if stop > known:
parts.append(self.tail.expand(-1, stop - max(start, known), -1, -1))
return torch.cat(parts, dim=1)
def zero_tail(encoder, height, width, dtype, steps_cap=256):
"""Encoder state and y latent for the zero frames of clips of ``height x width``."""
vae = encoder.vae
device = vae.mean.device
head = torch.linspace(-1, 1, height * width * 3, device=device,
dtype=vae.dtype).view(1, 3, 1, height, width)
with amp.autocast(dtype=vae.dtype):
state = encoder.new_state()
zeros = head.new_zeros(1, 3, 4, height, width)
encoder.step(head, state)
for step in range(1, steps_cap):
before = [v.clone() if isinstance(v, torch.Tensor) else v for v in state]
latent = encoder.step(zeros, state)
if step > 1 and encoder.agree(_same_causal_state(before, state)):
logging.info("I2V zero tail for %dx%d: fixed point after %d encoder steps", height, width, step)
return ([v.clone() if isinstance(v, torch.Tensor) else v for v in state],
_y_row(latent, False, dtype))
raise RuntimeError("the VAE encoder did not reach a zero-input fixed point")
class _LatentWindow:
"""The trailing ``capacity`` latent frames of a world, indexed by absolute latent frame."""
def __init__(self, buffers, total):
self.buffers = buffers # two [16, capacity, h, w] fp32
# Shape of the full-horizon latent volume this window stands in for.
self.shape = (buffers[0].shape[0], total, *buffers[0].shape[2:])
self.which = 0
self.base = 0 # absolute index of buffer frame 0
self.length = 0
def write(self, start, latent):
n = latent.shape[1]
cap = self.buffers[0].shape[1]
if start + n - self.base > cap:
keep = cap - n
src = self.buffers[self.which]
dst = self.buffers[1 - self.which]
first = start - keep
dst[:, :keep].copy_(src[:, first - self.base:start - self.base])
self.which, self.base = 1 - self.which, first
self.buffers[self.which][:, start - self.base:start - self.base + n].copy_(latent)
self.length = start + n
def __getitem__(self, index):
_, sl = index
start, stop = sl.start or 0, sl.stop
if start < self.base:
raise IndexError(f"latent {start} is outside the kept window (from {self.base})")
buf = self.buffers[self.which]
return buf[:, start - self.base:stop - self.base]