Skip to content

[CVPR 2026] PersonaLive! : Expressive Portrait Image Animation for Live Streaming

8.7 KB
Raw Blame
import torch
import torch.nn as nn
import torch.nn.functional as F
from diffusers.models.embeddings import PatchEmbed

from einops import rearrange
from src.models.motion_module import TemporalTransformerBlock, zero_module, random_module

def conv3x3(in_planes, out_planes, strd=1, padding=1, bias=False):
    "3x3 convolution with padding"
    return nn.Conv2d(in_planes, out_planes, kernel_size=3,
                     stride=strd, padding=padding, bias=bias)


class ConvBlock(nn.Module):
    def __init__(self, in_planes, out_planes):
        super(ConvBlock, self).__init__()
        self.bn1 = nn.BatchNorm2d(in_planes)
        self.conv1 = conv3x3(in_planes, int(out_planes / 2))
        self.bn2 = nn.BatchNorm2d(int(out_planes / 2))
        self.conv2 = conv3x3(int(out_planes / 2), int(out_planes / 4))
        self.bn3 = nn.BatchNorm2d(int(out_planes / 4))
        self.conv3 = conv3x3(int(out_planes / 4), int(out_planes / 4))

        if in_planes != out_planes:
            self.downsample = nn.Sequential(
                nn.BatchNorm2d(in_planes),
                nn.ReLU(True),
                nn.Conv2d(in_planes, out_planes,
                          kernel_size=1, stride=1, bias=False),
            )
        else:
            self.downsample = None

    def forward(self, x):
        residual = x

        out1 = self.bn1(x)
        out1 = F.relu(out1, True)
        out1 = self.conv1(out1)

        out2 = self.bn2(out1)
        out2 = F.relu(out2, True)
        out2 = self.conv2(out2)

        out3 = self.bn3(out2)
        out3 = F.relu(out3, True)
        out3 = self.conv3(out3)

        out3 = torch.cat((out1, out2, out3), 1)

        if self.downsample is not None:
            residual = self.downsample(residual)

        out3 += residual

        return out3


class HourGlass(nn.Module):
    def __init__(self, num_modules, depth, num_features):
        super(HourGlass, self).__init__()
        self.num_modules = num_modules
        self.depth = depth
        self.features = num_features
        self.dropout = nn.Dropout(0.5)

        self._generate_network(self.depth)

    def _generate_network(self, level):
        self.add_module('b1_' + str(level), ConvBlock(256, 256))

        self.add_module('b2_' + str(level), ConvBlock(256, 256))

        if level > 1:
            self._generate_network(level - 1)
        else:
            self.add_module('b2_plus_' + str(level), ConvBlock(256, 256))

        self.add_module('b3_' + str(level), ConvBlock(256, 256))

    def _forward(self, level, inp):
        # Upper branch
        up1 = inp
        up1 = self._modules['b1_' + str(level)](up1)
        up1 = self.dropout(up1)
        # Lower branch
        low1 = F.max_pool2d(inp, 2, stride=2)
        low1 = self._modules['b2_' + str(level)](low1)

        if level > 1:
            low2 = self._forward(level - 1, low1)
        else:
            low2 = low1
            low2 = self._modules['b2_plus_' + str(level)](low2)

        low3 = low2
        low3 = self._modules['b3_' + str(level)](low3)
        up1size = up1.size()
        rescale_size = (up1size[2], up1size[3])
        up2 = F.upsample(low3, size=rescale_size, mode='bilinear')

        return up1 + up2

    def forward(self, x):
        return self._forward(self.depth, x)

class TemporalTransformer3DModel(nn.Module):
    def __init__(
        self,
        in_channels,
        num_attention_heads=8,
        attention_head_dim=128,
        num_layers=1,
        attention_block_types=("Temporal_Self",),
        sample_size=56,
        patch_size=2,
        dropout=0.0,
        norm_num_groups=32,
        cross_attention_dim=0,
        activation_fn="geglu",
        attention_bias=False,
        upcast_attention=False,
        cross_frame_attention_mode=None,
        temporal_position_encoding=True,
        temporal_position_encoding_max_len=24,
        zero_initialize=True
    ):
        super().__init__()
        self.in_channels = in_channels
        inner_dim = num_attention_heads * attention_head_dim

        self.norm = torch.nn.GroupNorm(
            num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True
        )
        # self.proj_in = nn.Linear(in_channels, inner_dim)  # patchEmbed中有
        if cross_attention_dim == 0 and any([block_name.endswith("_Cross") for block_name in attention_block_types]): 
            cross_attention_dim = inner_dim
        self.cross_frame_attention_mode = cross_frame_attention_mode
        if cross_frame_attention_mode is not None:
            assert cross_frame_attention_mode in ['Spatial', 'Temporal'], cross_attention_dim
            assert any([block_name.endswith("_Cross") for block_name in attention_block_types])
        self.transformer_blocks = nn.ModuleList(
            [
                TemporalTransformerBlock(
                    dim=inner_dim,
                    num_attention_heads=num_attention_heads,
                    attention_head_dim=attention_head_dim,
                    attention_block_types=attention_block_types,
                    dropout=dropout,
                    norm_num_groups=norm_num_groups,
                    cross_attention_dim=cross_attention_dim,
                    activation_fn=activation_fn,
                    attention_bias=attention_bias,
                    upcast_attention=upcast_attention,
                    cross_frame_attention_mode=None,
                    temporal_position_encoding=temporal_position_encoding,
                    temporal_position_encoding_max_len=temporal_position_encoding_max_len,
                )
                for _ in range(num_layers)
            ]
        )
        self.proj_out = nn.Linear(inner_dim // (patch_size**2), in_channels)

        self.patch_size = patch_size
        self.sample_size = sample_size
        self.pos_embed = PatchEmbed(
            height=sample_size,
            width=sample_size,
            patch_size=patch_size,
            in_channels=in_channels,
            embed_dim=inner_dim,
            interpolation_scale=1,
        )
        # 在转onnx模型时,需要设置为False
        self.pos_embed.pos_embed.requires_grad = True
        self.proj_out = zero_module(self.proj_out) if zero_initialize else random_module(self.proj_out)

    def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, skip=True):
        assert hidden_states.dim() == 5, f"Expected hidden_states to have ndim=5, but got ndim={hidden_states.dim()}."
        batch, _, video_length = hidden_states.shape[:3]
        residual = hidden_states
        hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w")

        _, _, height, width = hidden_states.shape

        hidden_states = self.norm(hidden_states)  # 需要先norm,再加posEmb,否则Emb偏执太大了
        grid_h, grid_w =  height // self.patch_size, width // self.patch_size,

        hidden_states = self.pos_embed(hidden_states)  # [(bf), l, c]  # 偏执太大了
        # print(
        #     round(torch.abs(hidden_states).mean().item(), 6),
        #     round(torch.abs(norm_hidden_states).mean().item(), 6),
        # )
        if self.cross_frame_attention_mode is not None:
            assert encoder_hidden_states is None
            encoder_hidden_states = rearrange(hidden_states, "(b f) d c -> b f d c", b=batch)

            hidden_states = encoder_hidden_states[:, -1] # [b, d, c]
            residual = residual[:, :, -1:]
            video_length = 1
            if self.cross_frame_attention_mode == 'Temporal': # 用所有帧来condition最后一帧(当前帧)
                encoder_hidden_states = rearrange(encoder_hidden_states, "b f d c -> (b d) f c")
            elif self.cross_frame_attention_mode == 'Spatial':
                encoder_hidden_states = rearrange(encoder_hidden_states, "b f d c -> b (f d) c")

        # Transformer Blocks
        for block in self.transformer_blocks:
            hidden_states = block(
                hidden_states,
                encoder_hidden_states=encoder_hidden_states,
                video_length=video_length,
                att_flag=False
            )

        hidden_states = hidden_states.reshape(
            shape=(batch * video_length, grid_h, grid_w, self.patch_size, self.patch_size, -1)
        )
        hidden_states = self.proj_out(hidden_states)
        hidden_states = torch.einsum("nhwpqc->nchpwq", hidden_states)
        hidden_states = hidden_states.reshape(shape=(batch * video_length, self.in_channels, height, width))

        hidden_states = rearrange(hidden_states, "(b f) c h w -> b c f h w", b=batch)
        # print(
        #     'out',
        #     round(torch.abs(hidden_states).mean().item(), 6),
        #     round(torch.abs(residual).mean().item(), 6),
        # )
        output = (hidden_states + residual) if skip else hidden_states

        return output