Skip to content

WorldPlay2: Extending Real-Time Interactive World Models in Control and Horizon

Python
37.2 KB
Raw Blame
#!/usr/bin/env python3
"""
Causal (left-padded) TAE-style ENCODER for Wan 2.1, adapted for 4n+1 frame
video inputs.

This is a variant of the TAEW2_1 encoder (see ``taew2_1.py``) with ONE
architectural change in the temporal-pooling behaviour:

    * The original ``TPool`` reshapes every ``stride`` consecutive frames
      into the channel axis directly. It relied on ``encode_video`` padding
      the time axis at the *end* to a multiple of ``t_downscale``.

    * ``CausalTPool`` (here) instead pads ONE zero frame on the *left*
      (the temporal-causal side) whenever the incoming time length is not a
      multiple of ``stride`` (i.e. for stride>1 on an odd-length input),
      then pools. For a 4n+1 input this cascades cleanly:
          T=4n+1 --pad1--> 4n+2 --pool2--> 2n+1 --pad1--> 2n+2 --pool2--> n+1.

``MemBlock`` keeps the SAME behaviour as the original parallel path: it
takes the "previous frame" as memory by left-padding one zero frame along
the time axis and concatenating it on the channel dim inside the block.
That part is unchanged — this file only re-implements the pooling.

Scope (per request):
    * ENCODER ONLY. No decoder / TGrow.
    * ONE-SHOT (parallel) encode only. No streaming work-queue path.

I/O convention (same as TAEW2_1):
    encode_video: input  [N, T, C=3, H, W] in [0, 1], T = 4n+1
                  output [N, T_lat, latent_channels, H/8, W/8]
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from tqdm.auto import tqdm


def conv(n_in, n_out, **kwargs):
    return nn.Conv2d(n_in, n_out, 3, padding=1, **kwargs)


class MemBlock(nn.Module):
    """Residual block that also consumes the previous frame's activation
    (its "memory") concatenated on the channel axis. Identical to the
    TAEW2_1 MemBlock."""

    def __init__(self, n_in, n_out):
        super().__init__()
        self.conv = nn.Sequential(conv(n_in * 2, n_out), nn.ReLU(inplace=True), conv(n_out, n_out), nn.ReLU(inplace=True), conv(n_out, n_out))
        self.skip = nn.Conv2d(n_in, n_out, 1, bias=False) if n_in != n_out else nn.Identity()
        self.act = nn.ReLU(inplace=True)

    def forward(self, x, past):
        return self.act(self.conv(torch.cat([x, past], 1)) + self.skip(x))

    def forward_chunk(self, x, N, cache):
        """Streaming forward over one chunk in [N*T, C, H, W] layout.

        The per-frame memory is the *previous* frame's input. Within the
        chunk this is the left-shifted version of ``x``; the very first
        frame of the chunk uses ``cache`` (the last input frame of the
        PREVIOUS chunk) instead of a zero frame -- except when ``cache`` is
        None (the first chunk overall), where it falls back to zeros. This
        makes the chunked result identical to the one-shot path.

        Args:
            x: [N*T, C, H, W] chunk input.
            N: batch size.
            cache: previous chunk's last-frame input, [N, 1, C, H, W], or None.
        Returns:
            (out, new_cache) where out is [N*T, C, H, W] and new_cache is
            this chunk's last-frame input [N, 1, C, H, W].
        """
        NT, C, H, W = x.shape
        T = NT // N
        _x = x.view(N, T, C, H, W)
        if cache is None:
            head = _x.new_zeros(N, 1, C, H, W)     # first chunk: zero history
        else:
            head = cache                            # continuation: prev last frame
        # memory[t] = input[t-1]; memory[0] = head
        block_memory = torch.cat([head, _x[:, :T - 1]], dim=1).reshape(x.shape)
        new_cache = _x[:, -1:].clone()
        out = self.forward(x, block_memory)
        return out, new_cache


class CausalTPool(nn.Module):
    """Temporal pooling that LEFT-pads one zero frame when needed so a
    4n+1 (odd) time length becomes even before pooling every ``stride``
    frames into the channel axis.

    Operates on the flattened [N*T, C, H, W] layout used by the parallel
    encoder path, and therefore needs ``N`` (batch size) to recover the
    time dimension.
    """

    def __init__(self, n_f, stride):
        super().__init__()
        self.stride = stride
        self.conv = nn.Conv2d(n_f * stride, n_f, 1, bias=False)

    def forward(self, x, N):
        # x: [N*T, C, H, W]
        NT, C, H, W = x.shape
        T = NT // N
        if self.stride == 1:
            # no temporal pooling; the 1x1 conv still maps n_f -> n_f
            return self.conv(x)

        x = x.view(N, T, C, H, W)
        # left-pad zeros along the time axis until T is a multiple of stride
        # (for 4n+1 inputs and stride=2 this pads exactly one frame).
        rem = T % self.stride
        if rem != 0:
            n_pad = self.stride - rem
            pad = x.new_zeros(N, n_pad, C, H, W)
            x = torch.cat([pad, x], dim=1)  # LEFT pad (causal side)
            T = x.shape[1]

        # group every ``stride`` consecutive frames onto the channel axis,
        # matching the original TPool's reshape ordering (frame t occupies
        # channels [0:C], frame t+1 -> [C:2C], ...).
        x = x.view(N, T // self.stride, self.stride * C, H, W)
        x = x.reshape(N * (T // self.stride), self.stride * C, H, W)
        return self.conv(x)

    def forward_chunk(self, x, N, is_first_chunk):
        """Streaming pool for the 1,4,4,4 chunk schedule.

        With that schedule and stride=2, only the FIRST chunk is odd
        (length 1 -> left-pad 1 -> 2 -> pool -> 1); every subsequent chunk
        already has a length that is a multiple of ``stride`` (4 -> pool ->
        2, and at the 2nd pooling stage 2 -> pool -> 1), so no cross-chunk
        residual buffering is needed and the concatenation of per-chunk
        outputs is bit-for-bit identical to the one-shot ``forward``.

        Args:
            x: [N*T, C, H, W] chunk input.
            N: batch size.
            is_first_chunk: True only for the very first chunk.
        Returns:
            [N*(T_out), C, H, W] pooled output for this chunk.
        """
        NT, C, H, W = x.shape
        T = NT // N
        if self.stride == 1:
            return self.conv(x)

        _x = x.view(N, T, C, H, W)
        rem = T % self.stride
        if rem != 0:
            assert is_first_chunk, (
                f"CausalTPool.forward_chunk: non-first chunk has time length "
                f"{T} which is not a multiple of stride {self.stride}; the "
                f"1,4,4,4 schedule should keep continuation chunks aligned.")
            n_pad = self.stride - rem
            pad = _x.new_zeros(N, n_pad, C, H, W)
            _x = torch.cat([pad, _x], dim=1)  # LEFT pad only on the first chunk
            T = _x.shape[1]

        _x = _x.view(N, T // self.stride, self.stride * C, H, W)
        _x = _x.reshape(N * (T // self.stride), self.stride * C, H, W)
        return self.conv(_x)


class Clamp(nn.Module):
    """Soft clamp to ~[-3, 3] on the latent, used at the head of the decoder
    (identical to the TAEW2_1 Clamp)."""

    def forward(self, x):
        return torch.tanh(x / 3) * 3


class CausalTGrow(nn.Module):
    """Temporal upsampling, inverse of ``CausalTPool``.

    ``conv`` expands the channel axis to ``stride * n_f`` and the frames are
    then split back out along the time axis (1 latent frame -> ``stride``
    pixel-time frames), matching the original TGrow reshape ordering.

    For CAUSAL first-frame handling, the VERY FIRST latent frame of a clip
    maps to a single output frame with ``output == input`` (it skips both the
    conv and the temporal expansion entirely), while every subsequent latent
    frame is expanded to ``stride`` frames. This mirrors the encoder, where
    the first pixel frame maps to one latent; here the first latent maps back
    to one frame, so:

        T_lat = n + 1  ->  T_pix = 1 + n * (product of TGrow strides) = 4n+1
    """

    def __init__(self, n_f, stride):
        super().__init__()
        self.stride = stride
        self.conv = nn.Conv2d(n_f, n_f * stride, 1, bias=False)

    def _expand(self, x):
        """conv + reshape: [N*T, C, H, W] -> [N*T*stride, C, H, W]."""
        NT, C, H, W = x.shape
        x = self.conv(x)                       # [N*T, stride*C, H, W]
        return x.reshape(NT * self.stride, C, H, W)

    def forward(self, x, N, is_first_clip=True):
        """One-shot temporal grow over the whole clip.

        Args:
            x: [N*T_lat, C, H, W].
            N: batch size.
            is_first_clip: if True, the first latent frame of the clip
                expands to a single output frame (causal startup), the rest
                expand to ``stride`` frames each. If False (continuation
                chunk in streaming decode), ALL frames expand to ``stride``.
        Returns:
            [N*T_out, C, H, W].
        """
        NT, C, H, W = x.shape
        T = NT // N
        if self.stride == 1:
            # no temporal upsampling; conv maps n_f -> n_f*1 = n_f, reshape no-op
            return self._expand(x)

        _x = x.view(N, T, C, H, W)
        if is_first_clip:
            # first latent frame: output == input (no temporal expansion,
            # no conv), so it maps to exactly ONE output frame.
            first = _x[:, :1]                             # [N, 1, C, H, W]
            rest = _x[:, 1:].reshape(N * (T - 1), C, H, W) if T > 1 else None
            outs = [first]
            if rest is not None:
                rest_grown = self._expand(rest)           # [N*(T-1)*stride, C, H, W]
                rest_grown = rest_grown.view(N, (T - 1) * self.stride, C, H, W)
                outs.append(rest_grown)
            out = torch.cat(outs, dim=1)                  # [N, T_out, C, H, W]
            return out.reshape(N * out.shape[1], C, H, W)
        else:
            # continuation chunk: every latent frame expands to stride frames
            return self._expand(x)


class CausalTAEEncoder(nn.Module):
    """One-shot causal TAE-style encoder for Wan 2.1 (4n+1 frame inputs)."""

    def __init__(self,
                 encoder_time_downscale=(True, True, False),
                 latent_channels=16,
                 patch_size=1):
        super().__init__()
        self.patch_size = patch_size
        self.latent_channels = latent_channels
        self.image_channels = 3

        self.encoder = nn.Sequential(
            conv(self.image_channels * self.patch_size ** 2, 64), nn.ReLU(inplace=True),
            CausalTPool(64, 2 if encoder_time_downscale[0] else 1), conv(64, 64, stride=2, bias=False), MemBlock(64, 64), MemBlock(64, 64), MemBlock(64, 64),
            CausalTPool(64, 2 if encoder_time_downscale[1] else 1), conv(64, 64, stride=2, bias=False), MemBlock(64, 64), MemBlock(64, 64), MemBlock(64, 64),
            CausalTPool(64, 2 if encoder_time_downscale[2] else 1), conv(64, 64, stride=2, bias=False), MemBlock(64, 64), MemBlock(64, 64), MemBlock(64, 64),
            conv(64, self.latent_channels),
        )

        # temporal downscale factor (product of the pooling strides)
        self.t_downscale = 1
        for m in self.encoder:
            if isinstance(m, CausalTPool):
                self.t_downscale *= m.stride

    def preprocess_input_frames(self, x):
        """Preprocess RGB input frames prior to the main encoder sequence."""
        if self.patch_size > 1:
            x = F.pixel_unshuffle(x, self.patch_size)
        return x

    def _apply_encoder(self, x, show_progress_bar):
        """Parallel (one-shot) application of the encoder sequence over the
        time axis.

        Args:
            x: [N, T, C, H, W] tensor.
        Returns:
            [N, T_lat, latent_channels, H/8, W/8] tensor.
        """
        assert x.ndim == 5, f"expected NTCHW tensor, got {x.ndim}-dim"
        N, T, C, H, W = x.shape
        x = x.reshape(N * T, C, H, W)

        for b in tqdm(self.encoder, disable=not show_progress_bar):
            if isinstance(b, MemBlock):
                # "previous frame" memory = left-pad one zero frame along
                # time, then drop the last frame so lengths line up. This is
                # the same causal memory the original parallel path uses.
                NT, C, H, W = x.shape
                T = NT // N
                _x = x.reshape(N, T, C, H, W)
                block_memory = F.pad(_x, (0, 0, 0, 0, 0, 0, 1, 0), value=0)[:, :T].reshape(x.shape)
                x = b(x, block_memory)
            elif isinstance(b, CausalTPool):
                # CausalTPool needs N to recover the time dim and does the
                # left-pad-then-pool internally.
                x = b(x, N)
            else:
                x = b(x)

        NT, C, H, W = x.shape
        T = NT // N
        return x.view(N, T, C, H, W)

    def encode_video(self, x, show_progress_bar=True):
        """Encode a sequence of frames (one-shot / parallel).

        Args:
            x: input [N, T, C=3, H, W] RGB tensor in [0, 1], with T = 4n+1.
            show_progress_bar: enable tqdm over encoder blocks.
        Returns:
            [N, T_lat, latent_channels, H/8, W/8] latent tensor.

        Unlike TAEW2_1.encode_video, there is NO end-of-sequence padding:
        the left-padding inside each CausalTPool handles odd (4n+1) lengths
        causally.
        """
        x = self.preprocess_input_frames(x)
        return self._apply_encoder(x, show_progress_bar)

    # ------------------------------------------------------------------
    # streaming / chunked encode (1, 4, 4, 4, ... schedule)
    # ------------------------------------------------------------------

    def init_cache(self):
        """Create a fresh per-block streaming cache.

        Returns a list with one slot per encoder block; only ``MemBlock``
        slots are ever populated (each holds that block's last-frame input
        from the previous chunk). Pass the same object to every
        :meth:`encode_chunk` call of one stream.
        """
        return [None] * len(self.encoder)

    def _apply_encoder_chunk(self, x, cache, is_first_chunk):
        """Streaming application of the encoder over one chunk.

        Args:
            x: [N, T_chunk, C, H, W] chunk (T_chunk == 1 for the first chunk,
               then 4 for each continuation chunk).
            cache: list from :meth:`init_cache`, mutated in place.
            is_first_chunk: True only for the first chunk of the stream.
        Returns:
            [N, T_lat_chunk, latent_channels, H/8, W/8] latent for this chunk.
        """
        assert x.ndim == 5, f"expected NTCHW tensor, got {x.ndim}-dim"
        N = x.shape[0]
        x = x.reshape(N * x.shape[1], *x.shape[2:])

        for i, b in enumerate(self.encoder):
            if isinstance(b, MemBlock):
                x, cache[i] = b.forward_chunk(x, N, cache[i])
            elif isinstance(b, CausalTPool):
                x = b.forward_chunk(x, N, is_first_chunk)
            else:
                x = b(x)

        NT, C, H, W = x.shape
        return x.view(N, NT // N, C, H, W)

    def encode_chunk(self, x, cache, is_first_chunk=False):
        """Encode ONE chunk of frames, continuing the streaming state.

        Feed the video in the ``1, 4, 4, 4, ...`` schedule: the first call
        with ``is_first_chunk=True`` passes a single frame ([N, 1, C, H, W]),
        and each subsequent call passes 4 frames ([N, 4, C, H, W]) with
        ``is_first_chunk=False``. Concatenating the returned latents along
        the time axis reproduces :meth:`encode_video` on the full clip
        exactly.

        Args:
            x: [N, T_chunk, C=3, H, W] RGB chunk in [0, 1].
            cache: list from :meth:`init_cache`, reused across the stream.
            is_first_chunk: True only for the very first chunk.
        Returns:
            [N, T_lat_chunk, latent_channels, H/8, W/8] latent for this chunk.
        """
        x = self.preprocess_input_frames(x)
        return self._apply_encoder_chunk(x, cache, is_first_chunk)

    def encode_video_chunked(self, x, show_progress_bar=False):
        """Convenience wrapper: split a full 4n+1 clip into the 1,4,4,4,...
        schedule, run :meth:`encode_chunk` over each piece, and concatenate.

        Args:
            x: [N, T=4n+1, C=3, H, W] RGB clip in [0, 1].
        Returns:
            [N, T_lat, latent_channels, H/8, W/8] latent (== encode_video).
        """
        T = x.shape[1]
        assert T % 4 == 1, f"chunked encode expects 4n+1 frames, got T={T}"
        cache = self.init_cache()
        outs = []
        # first chunk: 1 frame
        outs.append(self.encode_chunk(x[:, :1], cache, is_first_chunk=True))
        # continuation chunks: 4 frames each
        starts = range(1, T, 4)
        for s in tqdm(starts, disable=not show_progress_bar):
            outs.append(self.encode_chunk(x[:, s:s + 4], cache, is_first_chunk=False))
        return torch.cat(outs, dim=1)

    def forward(self, x, show_progress_bar=False):
        return self.encode_video(x, show_progress_bar=show_progress_bar)


class CausalTAEDecoder(nn.Module):
    """Causal TAE-style decoder for Wan 2.1, inverse of CausalTAEEncoder.

    The architecture mirrors the TAEW2_1 base decoder. The only causal
    change is in temporal upsampling: the FIRST latent frame of a clip
    decodes to a SINGLE output frame (via CausalTGrow's first-frame path),
    so the whole decoder maps

        T_lat = n + 1  latent frames  ->  T_pix = 4n + 1  output frames

    which is the exact inverse of the encoder. No end-of-sequence trimming
    is required (unlike the original TAEW2_1.decode_video which trims
    ``frames_to_trim`` startup frames).
    """

    def __init__(self,
                 decoder_time_upscale=(False, True, True),
                 decoder_space_upscale=(True, True, True),
                 latent_channels=16,
                 patch_size=1):
        super().__init__()
        self.patch_size = patch_size
        self.latent_channels = latent_channels
        self.image_channels = 3

        n_f = [256, 128, 64, 64]
        self.decoder = nn.Sequential(
            Clamp(), conv(self.latent_channels, n_f[0]), nn.ReLU(inplace=True),
            MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), MemBlock(n_f[0], n_f[0]), nn.Upsample(scale_factor=2 if decoder_space_upscale[0] else 1), CausalTGrow(n_f[0], 2 if decoder_time_upscale[0] else 1), conv(n_f[0], n_f[1], bias=False),
            MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), MemBlock(n_f[1], n_f[1]), nn.Upsample(scale_factor=2 if decoder_space_upscale[1] else 1), CausalTGrow(n_f[1], 2 if decoder_time_upscale[1] else 1), conv(n_f[1], n_f[2], bias=False),
            MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), MemBlock(n_f[2], n_f[2]), nn.Upsample(scale_factor=2 if decoder_space_upscale[2] else 1), CausalTGrow(n_f[2], 2 if decoder_time_upscale[2] else 1), conv(n_f[2], n_f[3], bias=False),
            nn.ReLU(inplace=True), conv(n_f[3], self.image_channels * self.patch_size ** 2),
        )

        # temporal upscale factor (product of the grow strides)
        self.t_upscale = 1
        for m in self.decoder:
            if isinstance(m, CausalTGrow):
                self.t_upscale *= m.stride

    def postprocess_output_frames(self, x):
        """Postprocess RGB frames after the main decoder sequence."""
        if self.patch_size > 1:
            x = F.pixel_shuffle(x, self.patch_size)
        return x.clamp_(0, 1)

    def _apply_decoder(self, x, is_first_clip, show_progress_bar):
        """Apply the decoder sequence over the time axis (one-shot).

        Args:
            x: [N, T_lat, C, H, W] latent tensor.
            is_first_clip: True if this batch of latents starts a clip (the
                first latent frame decodes to a single output frame).
        Returns:
            [N, T_out, image_channels*patch_size**2, H*8, W*8] tensor.
        """
        assert x.ndim == 5, f"expected NTCHW tensor, got {x.ndim}-dim"
        N, T, C, H, W = x.shape
        x = x.reshape(N * T, C, H, W)

        for b in tqdm(self.decoder, disable=not show_progress_bar):
            if isinstance(b, MemBlock):
                NT, C, H, W = x.shape
                T = NT // N
                _x = x.reshape(N, T, C, H, W)
                block_memory = F.pad(_x, (0, 0, 0, 0, 0, 0, 1, 0), value=0)[:, :T].reshape(x.shape)
                x = b(x, block_memory)
            elif isinstance(b, CausalTGrow):
                x = b(x, N, is_first_clip=is_first_clip)
            else:
                x = b(x)

        NT, C, H, W = x.shape
        T = NT // N
        return x.view(N, T, C, H, W)

    def decode_video(self, x, show_progress_bar=True):
        """Decode a full sequence of latents (one-shot / parallel).

        Args:
            x: input [N, T_lat=n+1, C=latent_channels, H, W] latent tensor.
            show_progress_bar: enable tqdm over decoder blocks.
        Returns:
            [N, T_pix=4n+1, 3, H*8, W*8] RGB tensor in [0, 1].
        """
        x = self._apply_decoder(x, is_first_clip=True, show_progress_bar=show_progress_bar)
        return self.postprocess_output_frames(x)

    def decode_latent(self, x, is_first_latent, show_progress_bar=False):
        """Decode ONE (or a few) latent frame(s) with explicit first-frame
        control, WITHOUT threading a cross-latent cache -- the low-level
        stateless building block.

        NOTE: without a MemBlock cache, calling this repeatedly is NOT
        numerically identical to ``decode_video`` at chunk boundaries (each
        call restarts the MemBlock memory from zeros). For an equivalent
        one-latent-at-a-time decode, use :meth:`init_cache` +
        :meth:`decode_chunk` instead.

        Args:
            x: [N, T_sub, C=latent_channels, H, W] latent(s).
            is_first_latent: True if these latents include the clip's first
                latent frame (-> single output frame for that frame).
        Returns:
            [N, T_out, 3, H*8, W*8] RGB tensor in [0, 1].
        """
        x = self._apply_decoder(x, is_first_clip=is_first_latent,
                                show_progress_bar=show_progress_bar)
        return self.postprocess_output_frames(x)

    # ------------------------------------------------------------------
    # streaming / chunked decode (cross-latent cache)
    # ------------------------------------------------------------------

    def init_cache(self):
        """Create a fresh per-block streaming cache.

        Returns a list with one slot per decoder block; only ``MemBlock``
        slots are ever populated (each holds that block's last-frame input
        from the previous chunk). Pass the same object to every
        :meth:`decode_chunk` call of one stream.
        """
        return [None] * len(self.decoder)

    def _apply_decoder_chunk(self, x, cache, is_first_chunk):
        """Streaming application of the decoder over one chunk of latents.

        MemBlock threads its "previous frame" memory through ``cache`` (same
        contract as the encoder), and CausalTGrow uses ``is_first_chunk`` so
        that only the very first latent of the stream yields a single frame
        (output == input); continuation chunks expand every latent frame by
        ``stride``.

        Args:
            x: [N, T_sub, C, H, W] latent chunk.
            cache: list from :meth:`init_cache`, mutated in place.
            is_first_chunk: True only for the first chunk of the stream.
        Returns:
            [N, T_out, image_channels*patch_size**2, H*8, W*8] tensor.
        """
        assert x.ndim == 5, f"expected NTCHW tensor, got {x.ndim}-dim"
        N = x.shape[0]
        x = x.reshape(N * x.shape[1], *x.shape[2:])

        for i, b in enumerate(self.decoder):
            if isinstance(b, MemBlock):
                x, cache[i] = b.forward_chunk(x, N, cache[i])
            elif isinstance(b, CausalTGrow):
                x = b(x, N, is_first_clip=is_first_chunk)
            else:
                x = b(x)

        NT, C, H, W = x.shape
        return x.view(N, NT // N, C, H, W)

    def decode_chunk(self, x, cache, is_first_chunk=False):
        """Decode ONE chunk of latents, continuing the streaming state.

        Feed latents one (or a few) at a time. The first call with
        ``is_first_chunk=True`` decodes the clip's first latent to a single
        output frame; every subsequent call (``is_first_chunk=False``)
        expands each latent frame to ``t_upscale`` output frames.
        Concatenating the returned frames along the time axis reproduces
        :meth:`decode_video` on the full latent sequence exactly.

        Args:
            x: [N, T_sub, C=latent_channels, H, W] latent chunk.
            cache: list from :meth:`init_cache`, reused across the stream.
            is_first_chunk: True only for the very first chunk.
        Returns:
            [N, T_out, 3, H*8, W*8] RGB frames in [0, 1].
        """
        x = self._apply_decoder_chunk(x, cache, is_first_chunk)
        return self.postprocess_output_frames(x)

    def decode_video_streamed(self, x, show_progress_bar=False):
        """Convenience wrapper: decode a full latent sequence one latent at a
        time through :meth:`decode_chunk` and concatenate.

        Numerically identical to :meth:`decode_video` (up to float rounding).

        Args:
            x: [N, T_lat=n+1, C=latent_channels, H, W] latent tensor.
        Returns:
            [N, T_pix=4n+1, 3, H*8, W*8] RGB tensor in [0, 1].
        """
        T = x.shape[1]
        cache = self.init_cache()
        outs = []
        for i in tqdm(range(T), disable=not show_progress_bar):
            outs.append(self.decode_chunk(x[:, i:i + 1], cache,
                                          is_first_chunk=(i == 0)))
        return torch.cat(outs, dim=1)

    def forward(self, x, show_progress_bar=False):
        return self.decode_video(x, show_progress_bar=show_progress_bar)


class CausalTAE(nn.Module):
    """Unified causal TAE for Wan 2.1: encoder + decoder in one module.

    Wraps :class:`CausalTAEEncoder` and :class:`CausalTAEDecoder` so a single
    object exposes the full video<->latent roundtrip. Both one-shot and
    streaming (chunked) paths are provided and are numerically equivalent.

    I/O convention:
        encode: [N, T=4n+1, C=3, H, W] RGB in [0, 1]
                -> [N, T_lat=n+1, latent_channels, H/8, W/8]
        decode: [N, T_lat=n+1, latent_channels, H, W]
                -> [N, T_pix=4n+1, 3, H*8, W*8] RGB in [0, 1]

    Streaming schedules (equivalent to the one-shot paths):
        encode: feed frames as 1, 4, 4, 4, ...   (encode_chunk)
        decode: feed latents one at a time       (decode_chunk)
    """

    def __init__(self,
                 encoder_time_downscale=(True, True, False),
                 decoder_time_upscale=(False, True, True),
                 decoder_space_upscale=(True, True, True),
                 latent_channels=16,
                 patch_size=1):
        super().__init__()
        self.latent_channels = latent_channels
        self.patch_size = patch_size
        self.image_channels = 3

        self.enc = CausalTAEEncoder(
            encoder_time_downscale=encoder_time_downscale,
            latent_channels=latent_channels,
            patch_size=patch_size,
        )
        self.dec = CausalTAEDecoder(
            decoder_time_upscale=decoder_time_upscale,
            decoder_space_upscale=decoder_space_upscale,
            latent_channels=latent_channels,
            patch_size=patch_size,
        )
        self.t_downscale = self.enc.t_downscale
        self.t_upscale = self.dec.t_upscale

    # ---- one-shot ----------------------------------------------------

    def encode_video(self, x, show_progress_bar=True):
        """One-shot encode. [N, 4n+1, 3, H, W] -> [N, n+1, C_z, H/8, W/8]."""
        return self.enc.encode_video(x, show_progress_bar=show_progress_bar)

    def decode_video(self, z, show_progress_bar=True):
        """One-shot decode. [N, n+1, C_z, H, W] -> [N, 4n+1, 3, H*8, W*8]."""
        return self.dec.decode_video(z, show_progress_bar=show_progress_bar)

    def forward(self, x, teacher_latent=None, show_progress_bar=False):
        """Two modes:

        * ``teacher_latent is None`` (default): full roundtrip encode->decode,
          returns reconstructed RGB.
        * ``teacher_latent`` given: TRAINING mode. Runs both the encoder
          (supervised by ``teacher_latent``) and the decoder (fed the teacher
          latent, reconstructing ``x``) through this single ``forward`` so that
          DistributedDataParallel sees the whole graph and syncs gradients
          correctly. Returns ``(encoded, decoded)``:
              encoded: [N, T_lat, C_z, H/8, W/8]   (compare to teacher_latent)
              decoded: [N, T_pix, 3, H*8, W*8]     (compare to x)
        """
        if teacher_latent is None:
            z = self.encode_video(x, show_progress_bar=show_progress_bar)
            return self.decode_video(z, show_progress_bar=show_progress_bar)
        encoded = self.encode_video(x, show_progress_bar=show_progress_bar)
        decoded = self.decode_video(teacher_latent, show_progress_bar=show_progress_bar)
        return encoded, decoded

    # ---- streaming: encode (1, 4, 4, 4, ...) -------------------------

    def init_encode_cache(self):
        return self.enc.init_cache()

    def encode_chunk(self, x, cache, is_first_chunk=False):
        """Streaming encode of one chunk (1 frame first, then 4 each)."""
        return self.enc.encode_chunk(x, cache, is_first_chunk=is_first_chunk)

    def encode_video_chunked(self, x, show_progress_bar=False):
        """Streaming encode over a full 4n+1 clip (== encode_video)."""
        return self.enc.encode_video_chunked(x, show_progress_bar=show_progress_bar)

    # ---- streaming: decode (one latent at a time) --------------------

    def init_decode_cache(self):
        return self.dec.init_cache()

    def decode_chunk(self, z, cache, is_first_chunk=False):
        """Streaming decode of one latent chunk (first -> 1 frame, then
        t_upscale frames each)."""
        return self.dec.decode_chunk(z, cache, is_first_chunk=is_first_chunk)

    def decode_video_streamed(self, z, show_progress_bar=False):
        """Streaming decode over a full latent sequence (== decode_video)."""
        return self.dec.decode_video_streamed(z, show_progress_bar=show_progress_bar)


# ===========================================================================
# Drop-in adapter exposing the SAME interface as ``worldplay2.vae.wan21.Wan2_1_VAE``
# so ``image2video_stage_two`` can switch VAEs without touching call sites.
# ===========================================================================

class CausalTAEVAE:
    r"""Adapter that wraps :class:`CausalTAE` behind the exact public surface
    of :class:`worldplay2.vae.wan21.Wan2_1_VAE`, so the stage-2/3 pipeline can
    use it as a drop-in replacement with NO changes to its call sites.

    Interface parity (all list-in / list-out, elements are ``[C, T, H, W]``
    tensors WITHOUT a batch dim, pixels in ``[-1, 1]``):
        * ``encode([clip])``               -> ``[z]``
        * ``decode([z])``                  -> ``[clip]``
        * ``encode_chunk([clip], is_first_chunk)`` -> ``[z]``
        * ``decode_chunk([z],   is_first_chunk)``  -> ``[clip]``
      plus the ``.scale`` / ``.dtype`` / ``.device`` attributes.

    Differences from ``CausalTAE`` that this adapter bridges:
      * LAYOUT: ``Wan2_1_VAE`` uses list of ``[C, T, H, W]``; ``CausalTAE``
        uses batched ``[N, T, C, H, W]``.  We add/remove the batch dim and
        move the channel axis.
      * VALUE RANGE: ``Wan2_1_VAE`` pixels are in ``[-1, 1]``; ``CausalTAE``
        pixels are in ``[0, 1]``.  We map ``(x+1)/2`` in and ``x*2-1`` out.
      * STREAMING CACHE: ``CausalTAE`` streaming keeps the per-block cache in
        an EXTERNAL list; ``Wan2_1_VAE`` keeps it INTERNAL and re-arms it when
        ``is_first_chunk=True``.  We hold the enc/dec caches on the adapter
        and rebuild them on ``is_first_chunk=True`` to reproduce the internal
        semantics exactly.

    Temporal convention matches ``Wan2_1_VAE`` (causal, first-frame aligned):
      ``T_pix = 4 * T_lat - 3`` (== ``CausalTAE``'s ``T_lat = n+1 <-> 4n+1``).
    """

    def __init__(self,
                 vae_pth: str,
                 z_dim: int = 16,
                 dtype=torch.float,
                 device="cuda",
                 **causal_tae_kwargs):
        self.dtype = dtype
        self.device = device
        # No mean/std normalisation: the CausalTAE latent space is
        # self-consistent (its own decoder inverts its own encoder), so we
        # expose a no-op ``scale`` only for attribute parity with Wan2_1_VAE.
        self.scale = [0.0, 1.0]

        self.model = CausalTAE(
            latent_channels=z_dim, **causal_tae_kwargs
        ).eval().requires_grad_(False)
        if vae_pth is not None:
            sd = torch.load(vae_pth, map_location="cpu", weights_only=True)
            # tolerate a wrapping dict (e.g. {"model": ...} / {"state_dict": ...})
            if isinstance(sd, dict):
                for key in ("state_dict", "model", "module"):
                    if key in sd and isinstance(sd[key], dict):
                        sd = sd[key]
                        break
            missing, unexpected = self.model.load_state_dict(sd, strict=False)
            if missing or unexpected:
                import logging
                logging.info(
                    f"CausalTAEVAE: loaded {vae_pth} "
                    f"(missing={len(missing)}, unexpected={len(unexpected)})")
        self.model = self.model.to(device)

        # streaming caches (internal, mirroring Wan2_1_VAE's internal state).
        self._enc_cache = None
        self._dec_cache = None

    # ---- layout / range helpers -------------------------------------------

    @staticmethod
    def _to_ntchw_01(clip):
        """``[C, T, H, W]`` in [-1,1]  ->  ``[1, T, C, H, W]`` in [0,1]."""
        x = clip.permute(1, 0, 2, 3).contiguous()   # [T, C, H, W]
        x = x.unsqueeze(0)                           # [1, T, C, H, W]
        return x.mul(0.5).add(0.5)                   # [-1,1] -> [0,1]

    @staticmethod
    def _from_ntchw_01(x):
        """``[1, T, C, H, W]`` in [0,1]  ->  ``[C, T, H, W]`` in [-1,1]."""
        x = x.squeeze(0)                             # [T, C, H, W]
        x = x.permute(1, 0, 2, 3).contiguous()       # [C, T, H, W]
        return x.mul(2.0).sub(1.0).clamp_(-1, 1)     # [0,1] -> [-1,1]

    @staticmethod
    def _lat_to_ntchw(z):
        """latent ``[C, T, H, W]``  ->  ``[1, T, C, H, W]`` (no range change)."""
        return z.permute(1, 0, 2, 3).contiguous().unsqueeze(0)

    @staticmethod
    def _lat_from_ntchw(z):
        """latent ``[1, T, C, H, W]``  ->  ``[C, T, H, W]``."""
        return z.squeeze(0).permute(1, 0, 2, 3).contiguous()

    # ---- one-shot ---------------------------------------------------------

    def encode(self, videos):
        """``videos``: list of ``[3, T_pix, H, W]`` in [-1,1] ->
        list of latent ``[16, T_lat, H/8, W/8]``.

        Uses the STREAMING encode (``encode_video_chunked``: the 1,4,4,4,...
        chunk schedule with a fresh cache per clip) rather than a one-shot
        ``encode_video`` -- numerically identical, but bounds peak memory."""
        with torch.amp.autocast('cuda', dtype=self.dtype), torch.no_grad():
            out = []
            for u in videos:
                x = self._to_ntchw_01(u).to(self.device)
                z = self.model.encode_video_chunked(x, show_progress_bar=False)
                out.append(self._lat_from_ntchw(z).float())
            return out

    def decode(self, zs):
        """``zs``: list of latent ``[16, T_lat, H, W]`` ->
        list of pixel ``[3, T_pix, H*8, W*8]`` in [-1,1].

        Uses the STREAMING decode (``decode_video_streamed``: one latent
        chunk at a time with a fresh cache per clip) rather than a one-shot
        ``decode_video`` -- numerically identical, but bounds peak memory."""
        with torch.amp.autocast('cuda', dtype=self.dtype), torch.no_grad():
            out = []
            for u in zs:
                z = self._lat_to_ntchw(u).to(self.device)
                x = self.model.decode_video_streamed(z, show_progress_bar=False)
                out.append(self._from_ntchw_01(x).float())
            return out

    # ---- streaming --------------------------------------------------------

    def encode_chunk(self, videos, is_first_chunk: bool = False):
        """Streaming encode.  Same list I/O as :meth:`encode`.  The adapter
        holds the encoder cache internally and re-arms it when
        ``is_first_chunk=True`` (mirroring ``Wan2_1_VAE.encode_chunk``)."""
        if is_first_chunk:
            self._enc_cache = self.model.init_encode_cache()
        with torch.amp.autocast('cuda', dtype=self.dtype), torch.no_grad():
            out = []
            for u in videos:
                x = self._to_ntchw_01(u).to(self.device)
                z = self.model.encode_chunk(
                    x, self._enc_cache, is_first_chunk=is_first_chunk)
                out.append(self._lat_from_ntchw(z).float())
            return out

    def decode_chunk(self, zs, is_first_chunk: bool = False):
        """Streaming decode.  Same list I/O as :meth:`decode`.  The adapter
        holds the decoder cache internally and re-arms it when
        ``is_first_chunk=True`` (mirroring ``Wan2_1_VAE.decode_chunk``)."""
        if is_first_chunk:
            self._dec_cache = self.model.init_decode_cache()
        with torch.amp.autocast('cuda', dtype=self.dtype), torch.no_grad():
            out = []
            for u in zs:
                z = self._lat_to_ntchw(u).to(self.device)
                x = self.model.decode_chunk(
                    z, self._dec_cache, is_first_chunk=is_first_chunk)
                out.append(self._from_ntchw_01(x).float())
            return out