Skip to content

[CVPR 2026] PersonaLive! : Expressive Portrait Image Animation for Live Streaming

update src

Zhiyuan Li12c7f7d9f399
diff --git a/README.md b/README.md
index 0a6d66f..42d3792 100644
--- a/README.md
+++ b/README.md
@@ -31,4 +31,4 @@ If you find PersonaLive useful for your research, welcome to 🌟 this repo and
## ❤️ Acknowledgement
-This project is based on [X-NeMo](https://byteaigc.github.io/X-Portrait2/), [StreamDiffusion](https://github.com/cumulo-autumn/StreamDiffusion) and [RAIN](https://pscgylotti.github.io/pages/RAIN/), thanks to their invaluable contributions.
+This project is based on [Moore-AnimateAnyone](https://github.com/MooreThreads/Moore-AnimateAnyone), [X-NeMo](https://byteaigc.github.io/X-Portrait2/), [StreamDiffusion](https://github.com/cumulo-autumn/StreamDiffusion), [RAIN](https://pscgylotti.github.io/pages/RAIN/) and [LivePortrait](https://github.com/KlingTeam/LivePortrait), thanks to their invaluable contributions.
diff --git a/pretrained_weights/onnx/readme.txt b/pretrained_weights/onnx/readme.txt
new file mode 100644
index 0000000..84baf61
--- /dev/null
+++ b/pretrained_weights/onnx/readme.txt
@@ -0,0 +1 @@
+onnx models
\ No newline at end of file
diff --git a/pretrained_weights/personalive/readme.txt b/pretrained_weights/personalive/readme.txt
new file mode 100644
index 0000000..a78b831
--- /dev/null
+++ b/pretrained_weights/personalive/readme.txt
@@ -0,0 +1 @@
+pre_trained models
\ No newline at end of file
diff --git a/pretrained_weights/tensorrt/readme.txt b/pretrained_weights/tensorrt/readme.txt
new file mode 100644
index 0000000..33bdb41
--- /dev/null
+++ b/pretrained_weights/tensorrt/readme.txt
@@ -0,0 +1 @@
+tensorrt models
\ No newline at end of file
diff --git a/src/liveportrait/camera.py b/src/liveportrait/camera.py
new file mode 100644
index 0000000..a3dd942
--- /dev/null
+++ b/src/liveportrait/camera.py
@@ -0,0 +1,73 @@
+# coding: utf-8
+
+"""
+functions for processing and transforming 3D facial keypoints
+"""
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+PI = np.pi
+
+
+def headpose_pred_to_degree(pred):
+ """
+ pred: (bs, 66) or (bs, 1) or others
+ """
+ if pred.ndim > 1 and pred.shape[1] == 66:
+ # NOTE: note that the average is modified to 97.5
+ device = pred.device
+ idx_tensor = [idx for idx in range(0, 66)]
+ idx_tensor = torch.FloatTensor(idx_tensor).to(device)
+ pred = F.softmax(pred, dim=1)
+ degree = torch.sum(pred*idx_tensor, axis=1) * 3 - 97.5
+
+ return degree
+
+ return pred
+
+
+def get_rotation_matrix(pitch_, yaw_, roll_):
+ """ the input is in degree
+ """
+ # transform to radian
+ pitch = pitch_ / 180 * PI
+ yaw = yaw_ / 180 * PI
+ roll = roll_ / 180 * PI
+
+ device = pitch.device
+
+ if pitch.ndim == 1:
+ pitch = pitch.unsqueeze(1)
+ if yaw.ndim == 1:
+ yaw = yaw.unsqueeze(1)
+ if roll.ndim == 1:
+ roll = roll.unsqueeze(1)
+
+ # calculate the euler matrix
+ bs = pitch.shape[0]
+ ones = torch.ones([bs, 1]).to(device)
+ zeros = torch.zeros([bs, 1]).to(device)
+ x, y, z = pitch, yaw, roll
+
+ rot_x = torch.cat([
+ ones, zeros, zeros,
+ zeros, torch.cos(x), -torch.sin(x),
+ zeros, torch.sin(x), torch.cos(x)
+ ], dim=1).reshape([bs, 3, 3])
+
+ rot_y = torch.cat([
+ torch.cos(y), zeros, torch.sin(y),
+ zeros, ones, zeros,
+ -torch.sin(y), zeros, torch.cos(y)
+ ], dim=1).reshape([bs, 3, 3])
+
+ rot_z = torch.cat([
+ torch.cos(z), -torch.sin(z), zeros,
+ torch.sin(z), torch.cos(z), zeros,
+ zeros, zeros, ones
+ ], dim=1).reshape([bs, 3, 3])
+
+ rot = rot_z @ rot_y @ rot_x
+ return rot.permute(0, 2, 1) # transpose
diff --git a/src/liveportrait/convnextv2.py b/src/liveportrait/convnextv2.py
new file mode 100644
index 0000000..8cb7fc9
--- /dev/null
+++ b/src/liveportrait/convnextv2.py
@@ -0,0 +1,216 @@
+# coding: utf-8
+
+"""
+This moudle is adapted to the ConvNeXtV2 version for the extraction of implicit keypoints, poses, and expression deformation.
+"""
+
+import torch
+import torch.nn as nn
+# from timm.models.layers import trunc_normal_, DropPath
+from src.liveportrait.util import LayerNorm, DropPath, trunc_normal_, GRN
+from einops import rearrange
+
+__all__ = ['convnextv2_tiny']
+
+
+class Block(nn.Module):
+ """ ConvNeXtV2 Block.
+
+ Args:
+ dim (int): Number of input channels.
+ drop_path (float): Stochastic depth rate. Default: 0.0
+ """
+
+ def __init__(self, dim, drop_path=0.):
+ super().__init__()
+ self.dwconv = nn.Conv2d(dim, dim, kernel_size=7, padding=3, groups=dim) # depthwise conv
+ self.norm = LayerNorm(dim, eps=1e-6)
+ self.pwconv1 = nn.Linear(dim, 4 * dim) # pointwise/1x1 convs, implemented with linear layers
+ self.act = nn.GELU()
+ self.grn = GRN(4 * dim)
+ self.pwconv2 = nn.Linear(4 * dim, dim)
+ self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+
+ def forward(self, x):
+ input = x
+ x = self.dwconv(x)
+ x = x.permute(0, 2, 3, 1) # (N, C, H, W) -> (N, H, W, C)
+ x = self.norm(x)
+ x = self.pwconv1(x)
+ x = self.act(x)
+ x = self.grn(x)
+ x = self.pwconv2(x)
+ x = x.permute(0, 3, 1, 2) # (N, H, W, C) -> (N, C, H, W)
+ x = input + self.drop_path(x)
+ return x
+
+
+class ConvNeXtV2(nn.Module):
+ """ ConvNeXt V2
+
+ Args:
+ in_chans (int): Number of input image channels. Default: 3
+ num_classes (int): Number of classes for classification head. Default: 1000
+ depths (tuple(int)): Number of blocks at each stage. Default: [3, 3, 9, 3]
+ dims (int): Feature dimension at each stage. Default: [96, 192, 384, 768]
+ drop_path_rate (float): Stochastic depth rate. Default: 0.
+ head_init_scale (float): Init scaling value for classifier weights and biases. Default: 1.
+ """
+
+ def __init__(
+ self,
+ in_chans=3,
+ depths=[3, 3, 9, 3],
+ dims=[96, 192, 384, 768],
+ drop_path_rate=0.,
+ **kwargs
+ ):
+ super().__init__()
+ self.depths = depths
+ self.downsample_layers = nn.ModuleList() # stem and 3 intermediate downsampling conv layers
+ stem = nn.Sequential(
+ nn.Conv2d(in_chans, dims[0], kernel_size=4, stride=4),
+ LayerNorm(dims[0], eps=1e-6, data_format="channels_first")
+ )
+ self.downsample_layers.append(stem)
+ for i in range(3):
+ downsample_layer = nn.Sequential(
+ LayerNorm(dims[i], eps=1e-6, data_format="channels_first"),
+ nn.Conv2d(dims[i], dims[i+1], kernel_size=2, stride=2),
+ )
+ self.downsample_layers.append(downsample_layer)
+
+ self.stages = nn.ModuleList() # 4 feature resolution stages, each consisting of multiple residual blocks
+ dp_rates = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]
+ cur = 0
+ for i in range(4):
+ stage = nn.Sequential(
+ *[Block(dim=dims[i], drop_path=dp_rates[cur + j]) for j in range(depths[i])]
+ )
+ self.stages.append(stage)
+ cur += depths[i]
+
+ self.norm = nn.LayerNorm(dims[-1], eps=1e-6) # final norm layer
+
+ # NOTE: the output semantic items
+ num_bins = kwargs.get('num_bins', 66)
+ num_kp = kwargs.get('num_kp', 24) # the number of implicit keypoints
+ self.fc_kp = nn.Linear(dims[-1], 3 * num_kp) # implicit keypoints
+
+ # print('dims[-1]: ', dims[-1])
+ self.fc_scale = nn.Linear(dims[-1], 1) # scale
+ self.fc_pitch = nn.Linear(dims[-1], num_bins) # pitch bins
+ self.fc_yaw = nn.Linear(dims[-1], num_bins) # yaw bins
+ self.fc_roll = nn.Linear(dims[-1], num_bins) # roll bins
+ self.fc_t = nn.Linear(dims[-1], 3) # translation
+ self.fc_exp = nn.Linear(dims[-1], 3 * num_kp) # expression / delta
+
+ def _init_weights(self, m):
+ if isinstance(m, (nn.Conv2d, nn.Linear)):
+ trunc_normal_(m.weight, std=.02)
+ nn.init.constant_(m.bias, 0)
+
+ def forward_features(self, x):
+ for i in range(4):
+ x = self.downsample_layers[i](x)
+ x = self.stages[i](x)
+ return self.norm(x.mean([-2, -1])) # global average pooling, (N, C, H, W) -> (N, C)
+
+ def forward(self, x):
+ x = self.forward_features(x)
+
+ # implicit keypoints
+ kp = self.fc_kp(x)
+
+ # pose and expression deformation
+ pitch = self.fc_pitch(x)
+ yaw = self.fc_yaw(x)
+ roll = self.fc_roll(x)
+ t = self.fc_t(x)
+ # exp = self.fc_exp(x)
+ scale = self.fc_scale(x)
+
+ ret_dct = {
+ 'pitch': pitch,
+ 'yaw': yaw,
+ 'roll': roll,
+ 't': t,
+ # 'exp': exp,
+ 'scale': scale,
+
+ 'kp': kp, # canonical keypoint
+ }
+
+ return ret_dct
+
+
+def convnextv2_tiny(**kwargs):
+ model = ConvNeXtV2(depths=[3, 3, 9, 3], dims=[96, 192, 384, 768], **kwargs)
+ return model
+
+class ConvNeXt(nn.Module):
+ """ ConvNeXt V2
+
+ Args:
+ in_chans (int): Number of input image channels. Default: 3
+ num_classes (int): Number of classes for classification head. Default: 1000
+ depths (tuple(int)): Number of blocks at each stage. Default: [3, 3, 9, 3]
+ dims (int): Feature dimension at each stage. Default: [96, 192, 384, 768]
+ drop_path_rate (float): Stochastic depth rate. Default: 0.
+ head_init_scale (float): Init scaling value for classifier weights and biases. Default: 1.
+ """
+
+ def __init__(
+ self,
+ in_chans=3,
+ depths=[3, 3, 9, 3],
+ dims=[96, 192, 384, 768],
+ drop_path_rate=0.,
+ **kwargs
+ ):
+ super().__init__()
+ self.depths = depths
+ self.downsample_layers = nn.ModuleList() # stem and 3 intermediate downsampling conv layers
+ stem = nn.Sequential(
+ nn.Conv2d(in_chans, dims[0], kernel_size=4, stride=4),
+ LayerNorm(dims[0], eps=1e-6, data_format="channels_first")
+ )
+ self.downsample_layers.append(stem)
+ for i in range(3):
+ downsample_layer = nn.Sequential(
+ LayerNorm(dims[i], eps=1e-6, data_format="channels_first"),
+ nn.Conv2d(dims[i], dims[i+1], kernel_size=2, stride=2),
+ )
+ self.downsample_layers.append(downsample_layer)
+
+ self.stages = nn.ModuleList() # 4 feature resolution stages, each consisting of multiple residual blocks
+ dp_rates = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]
+ cur = 0
+ for i in range(4):
+ stage = nn.Sequential(
+ *[Block(dim=dims[i], drop_path=dp_rates[cur + j]) for j in range(depths[i])]
+ )
+ self.stages.append(stage)
+ cur += depths[i]
+
+ self.norm = nn.LayerNorm(dims[-1], eps=1e-6) # final norm layer
+
+ def _init_weights(self, m):
+ if isinstance(m, (nn.Conv2d, nn.Linear)):
+ trunc_normal_(m.weight, std=.02)
+ nn.init.constant_(m.bias, 0)
+
+ def forward_features(self, x):
+ for i in range(4):
+ x = self.downsample_layers[i](x)
+ x = self.stages[i](x)
+ return self.norm(x.mean([-2, -1])) # global average pooling, (N, C, H, W) -> (N, C)
+
+ def forward(self, x):
+ x = self.forward_features(x)
+ return x
+
+
+def convnextv2(**kwargs):
+ model = ConvNeXt(depths=[3, 3, 9, 3], dims=[96, 192, 384, 768], **kwargs)
+ return model
\ No newline at end of file
diff --git a/src/liveportrait/motion_extractor.py b/src/liveportrait/motion_extractor.py
new file mode 100644
index 0000000..ace8477
--- /dev/null
+++ b/src/liveportrait/motion_extractor.py
@@ -0,0 +1,212 @@
+# coding: utf-8
+
+"""
+Motion extractor(M), which directly predicts the canonical keypoints, head pose and expression deformation of the input image
+"""
+
+from torch import nn
+import torch
+from diffusers.models.modeling_utils import ModelMixin
+from src.liveportrait.convnextv2 import convnextv2_tiny
+from src.liveportrait.util import filter_state_dict
+from src.liveportrait.camera import headpose_pred_to_degree, get_rotation_matrix
+
+model_dict = {
+ 'convnextv2_tiny': convnextv2_tiny,
+}
+
+
+class MotionExtractor(ModelMixin):
+ def __init__(self, **kwargs):
+ super(MotionExtractor, self).__init__()
+
+ # default is convnextv2_base
+ backbone = kwargs.get('backbone', 'convnextv2_tiny')
+ self.detector = model_dict.get(backbone)(**kwargs)
+ self.register_buffer('idx_tensor', torch.arange(66, dtype=torch.float32))
+
+ def headpose_pred_to_degree(self, pred):
+ """
+ pred: (bs, 66) or (bs, 1) or others
+ """
+ if pred.ndim > 1 and pred.shape[1] == 66:
+ # NOTE: note that the average is modified to 97.5
+ prob = torch.nn.functional.softmax(pred, dim=1)
+ degree = torch.matmul(prob, self.idx_tensor)
+ degree = degree * 3 - 97.5
+
+ return degree
+
+ return pred
+
+ def load_pretrained(self, init_path: str):
+ if init_path not in (None, ''):
+ state_dict = torch.load(init_path, map_location=lambda storage, loc: storage)['model']
+ state_dict = filter_state_dict(state_dict, remove_name='head')
+ ret = self.detector.load_state_dict(state_dict, strict=False)
+ print(f'Load pretrained model from {init_path}, ret: {ret}')
+
+ def forward(self, x):
+ kp_info = self.detector(x)
+ return self.get_kp(kp_info)
+
+ def get_kp(self, kp_info):
+ bs = kp_info['kp'].shape[0]
+
+ angles_raw = torch.cat([kp_info['pitch'], kp_info['yaw'], kp_info['roll']], dim=0) # (3, 66)
+ angles_deg = self.headpose_pred_to_degree(angles_raw)[:, None] # (B, 3)
+ pitch, yaw, roll = torch.chunk(angles_deg, chunks=3, dim=0)
+
+
+ kp = kp_info['kp'].reshape(bs, -1, 3) # BxNx3
+ t, scale = kp_info['t'], kp_info['scale']
+
+ rot_mat = get_rotation_matrix(pitch, yaw, roll).to(self.dtype) # (bs, 3, 3)
+
+ if kp.ndim == 2:
+ num_kp = kp.shape[1] // 3 # Bx(num_kpx3)
+ else:
+ num_kp = kp.shape[1] # Bxnum_kpx3
+
+ # Eqn.2: s * (R * x_c,s) + t
+ kp_transformed = kp.view(bs, num_kp, 3) @ rot_mat# + exp.view(bs, num_kp, 3)
+ kp_transformed *= scale[..., None] # (bs, k, 3) * (bs, 1, 1) = (bs, k, 3)
+ kp_transformed[:, :, 0:2] += t[:, None, 0:2] # remove z, only apply tx ty
+
+ return kp_transformed
+
+ def interpolate_tensors(self, a: torch.Tensor, b: torch.Tensor, num: int = 10) -> torch.Tensor:
+ if a.shape != b.shape:
+ raise ValueError(f"Shape mismatch: a.shape={a.shape}, b.shape={b.shape}")
+
+ B, *rest = a.shape
+ alphas = torch.linspace(0, 1, num, device=a.device, dtype=a.dtype)
+ view_shape = (num,) + (1,) * len(rest)
+ alphas = alphas.view(view_shape) # (1, num, 1, 1, ...)
+
+ result = (1 - alphas) * a + alphas * b
+ return result[:-1]
+
+ def interpolate_kps(self, ref, motion, num_interp, t_scale=0.5, s_scale=0):
+ kp1 = self.detector(ref.to(self.dtype))
+ kp2_list = []
+ for i in range(0, motion.shape[0], 256):
+ motion_chunk = motion[i:i+256]
+ kp2_chunk = self.detector(motion_chunk.to(self.dtype))
+ kp2_list.append(kp2_chunk)
+ kp2 = {}
+ for key in kp2_list[0].keys():
+ kp2[key] = torch.cat([kp2_chunk[key] for kp2_chunk in kp2_list], dim=0)
+
+ angles_raw = torch.cat([kp1['pitch'], kp1['yaw'], kp1['roll']], dim=0) # (3, 66)
+ angles_deg = self.headpose_pred_to_degree(angles_raw) # (B, 3)
+ pitch_1, yaw_1, roll_1 = torch.chunk(angles_deg, chunks=3, dim=0)
+
+ angles_raw = torch.cat([kp2['pitch'], kp2['yaw'], kp2['roll']], dim=0) # (3, 66)
+ angles_deg = self.headpose_pred_to_degree(angles_raw) # (B, 3)
+ pitch_2, yaw_2, roll_2 = torch.chunk(angles_deg, chunks=3, dim=0)
+
+ pitch_interp = self.interpolate_tensors(pitch_1, pitch_2[:1], num_interp) # Bx(num_interp)x1
+ yaw_interp = self.interpolate_tensors(yaw_1, yaw_2[:1], num_interp) # Bx(num_interp)x1
+ roll_interp = self.interpolate_tensors(roll_1, roll_2[:1], num_interp) # Bx(num_interp)x1
+
+ t_1 = kp1['t']
+ t_2 = kp2['t']
+ t_2 = (t_2 - t_2[0]) * t_scale + t_1
+ t_interp = self.interpolate_tensors(t_1, t_2[:1], num_interp)
+
+ s_1 = kp1['scale']
+ s_2 = kp2['scale']
+ s_2 = s_2 * s_scale + s_1
+ s_interp = self.interpolate_tensors(s_1, s_2[:1], num_interp)
+
+ kp = kp1['kp'].repeat(num_interp+motion.shape[0]-1, 1)
+
+ kps_interp = {
+ 'pitch': torch.cat([pitch_interp, pitch_2], dim=0),
+ 'yaw': torch.cat([yaw_interp, yaw_2], dim=0),
+ 'roll': torch.cat([roll_interp, roll_2], dim=0),
+ 't': torch.cat([t_interp, t_2], dim=0),
+ 'scale': torch.cat([s_interp, s_2], dim=0),
+ 'kp': kp
+ }
+
+ kp_intrep = self.get_kp(kps_interp)
+
+ return kp_intrep
+
+
+ def interpolate_kps_online(self, ref, motion, num_interp, t_scale=0.5, s_scale=0):
+ kp1 = self.detector(ref.to(self.dtype))
+ kp_frame1 = self.detector(motion[:1].to(self.dtype))
+ kp2 = self.detector(motion.to(self.dtype))
+
+ angles_raw = torch.cat([kp1['pitch'], kp1['yaw'], kp1['roll']], dim=0) # (3, 66)
+ angles_deg = self.headpose_pred_to_degree(angles_raw) # (B, 3)
+ pitch_1, yaw_1, roll_1 = torch.chunk(angles_deg, chunks=3, dim=0)
+
+ angles_raw = torch.cat([kp2['pitch'], kp2['yaw'], kp2['roll']], dim=0) # (3, 66)
+ angles_deg = self.headpose_pred_to_degree(angles_raw) # (B, 3)
+ pitch_2, yaw_2, roll_2 = torch.chunk(angles_deg, chunks=3, dim=0)
+
+ pitch_interp = self.interpolate_tensors(pitch_1, pitch_2[:1], num_interp) # Bx(num_interp)x1
+ yaw_interp = self.interpolate_tensors(yaw_1, yaw_2[:1], num_interp) # Bx(num_interp)x1
+ roll_interp = self.interpolate_tensors(roll_1, roll_2[:1], num_interp) # Bx(num_interp)x1
+
+ t_1 = kp1['t']
+ t_2 = kp2['t']
+ t_2 = (t_2 - t_2[0]) * t_scale + t_1
+ t_interp = self.interpolate_tensors(t_1, t_2[:1], num_interp)
+
+ s_1 = kp1['scale']
+ s_2 = kp2['scale']
+ s_2 = s_2 * s_scale + s_1
+ s_interp = self.interpolate_tensors(s_1, s_2[:1], num_interp)
+
+ kp = kp1['kp'].repeat(num_interp+motion.shape[0]-1, 1)
+
+ kps_interp = {
+ 'pitch': torch.cat([pitch_interp, pitch_2], dim=0),
+ 'yaw': torch.cat([yaw_interp, yaw_2], dim=0),
+ 'roll': torch.cat([roll_interp, roll_2], dim=0),
+ 't': torch.cat([t_interp, t_2], dim=0),
+ 'scale': torch.cat([s_interp, s_2], dim=0),
+ 'kp': kp
+ }
+
+ kp_intrep = self.get_kp(kps_interp)
+
+ kp_dri = self.get_kp(kp2)
+
+ return kp_intrep, kp1, kp_frame1, kp_dri
+
+ def get_kps(self, kp_ref, kp_frame1, motion, t_scale=0.5, s_scale=0):
+ kps_motion = self.detector(motion.to(self.dtype))
+
+ kps_dri = self.get_kp(kps_motion)
+
+ t_ref = kp_ref['t']
+ t_frame1 = kp_frame1['t']
+ t_motion = kps_motion['t']
+ kps_motion['t'] = (t_motion - t_frame1) * t_scale + t_ref
+
+ s_ref = kp_ref['scale']
+ s_motion = kps_motion['scale']
+ kps_motion['scale'] = s_motion * s_scale + s_ref
+
+
+ kps_motion['kp'] = kp_ref['kp'].repeat(motion.shape[0], 1)
+
+ kps_motion = self.get_kp(kps_motion)
+
+ return kps_motion, kps_dri
+
+ def inference(self, ref, motion):
+ kps_ref = self.detector(ref.to(self.dtype))
+ kps_motion = self.detector(motion.to(self.dtype))
+ kps_motion['kp'] = kps_ref['kp']
+
+ kp_s = self.get_kp(kps_ref)
+ kp_d = self.get_kp(kps_motion)
+
+ return kp_s, kp_d
\ No newline at end of file
diff --git a/src/liveportrait/util.py b/src/liveportrait/util.py
new file mode 100644
index 0000000..a58ce58
--- /dev/null
+++ b/src/liveportrait/util.py
@@ -0,0 +1,492 @@
+# coding: utf-8
+
+"""
+This file defines various neural network modules and utility functions, including convolutional and residual blocks,
+normalizations, and functions for spatial transformation and tensor manipulation.
+"""
+
+from torch import nn
+import torch.nn.functional as F
+import torch
+import torch.nn.utils.spectral_norm as spectral_norm
+import math
+import warnings
+import collections.abc
+from itertools import repeat
+
+def kp2gaussian(kp, spatial_size, kp_variance):
+ """
+ Transform a keypoint into gaussian like representation
+ """
+ mean = kp
+
+ coordinate_grid = make_coordinate_grid(spatial_size, mean)
+ number_of_leading_dimensions = len(mean.shape) - 1
+ shape = (1,) * number_of_leading_dimensions + coordinate_grid.shape
+ coordinate_grid = coordinate_grid.view(*shape)
+ repeats = mean.shape[:number_of_leading_dimensions] + (1, 1, 1, 1)
+ coordinate_grid = coordinate_grid.repeat(*repeats)
+
+ # Preprocess kp shape
+ shape = mean.shape[:number_of_leading_dimensions] + (1, 1, 1, 3)
+ mean = mean.view(*shape)
+
+ mean_sub = (coordinate_grid - mean)
+
+ out = torch.exp(-0.5 * (mean_sub ** 2).sum(-1) / kp_variance)
+
+ return out
+
+
+def make_coordinate_grid(spatial_size, ref, **kwargs):
+ d, h, w = spatial_size
+ x = torch.arange(w).type(ref.dtype).to(ref.device)
+ y = torch.arange(h).type(ref.dtype).to(ref.device)
+ z = torch.arange(d).type(ref.dtype).to(ref.device)
+
+ # NOTE: must be right-down-in
+ x = (2 * (x / (w - 1)) - 1) # the x axis faces to the right
+ y = (2 * (y / (h - 1)) - 1) # the y axis faces to the bottom
+ z = (2 * (z / (d - 1)) - 1) # the z axis faces to the inner
+
+ yy = y.view(1, -1, 1).repeat(d, 1, w)
+ xx = x.view(1, 1, -1).repeat(d, h, 1)
+ zz = z.view(-1, 1, 1).repeat(1, h, w)
+
+ meshed = torch.cat([xx.unsqueeze_(3), yy.unsqueeze_(3), zz.unsqueeze_(3)], 3)
+
+ return meshed
+
+
+class ConvT2d(nn.Module):
+ """
+ Upsampling block for use in decoder.
+ """
+
+ def __init__(self, in_features, out_features, kernel_size=3, stride=2, padding=1, output_padding=1):
+ super(ConvT2d, self).__init__()
+
+ self.convT = nn.ConvTranspose2d(in_features, out_features, kernel_size=kernel_size, stride=stride,
+ padding=padding, output_padding=output_padding)
+ self.norm = nn.InstanceNorm2d(out_features)
+
+ def forward(self, x):
+ out = self.convT(x)
+ out = self.norm(out)
+ out = F.leaky_relu(out)
+ return out
+
+
+class ResBlock3d(nn.Module):
+ """
+ Res block, preserve spatial resolution.
+ """
+
+ def __init__(self, in_features, kernel_size, padding):
+ super(ResBlock3d, self).__init__()
+ self.conv1 = nn.Conv3d(in_channels=in_features, out_channels=in_features, kernel_size=kernel_size, padding=padding)
+ self.conv2 = nn.Conv3d(in_channels=in_features, out_channels=in_features, kernel_size=kernel_size, padding=padding)
+ self.norm1 = nn.BatchNorm3d(in_features, affine=True)
+ self.norm2 = nn.BatchNorm3d(in_features, affine=True)
+
+ def forward(self, x):
+ out = self.norm1(x)
+ out = F.relu(out)
+ out = self.conv1(out)
+ out = self.norm2(out)
+ out = F.relu(out)
+ out = self.conv2(out)
+ out += x
+ return out
+
+
+class UpBlock3d(nn.Module):
+ """
+ Upsampling block for use in decoder.
+ """
+
+ def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1):
+ super(UpBlock3d, self).__init__()
+
+ self.conv = nn.Conv3d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size,
+ padding=padding, groups=groups)
+ self.norm = nn.BatchNorm3d(out_features, affine=True)
+
+ def forward(self, x):
+ out = F.interpolate(x, scale_factor=(1, 2, 2))
+ out = self.conv(out)
+ out = self.norm(out)
+ out = F.relu(out)
+ return out
+
+
+class DownBlock2d(nn.Module):
+ """
+ Downsampling block for use in encoder.
+ """
+
+ def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1):
+ super(DownBlock2d, self).__init__()
+ self.conv = nn.Conv2d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size, padding=padding, groups=groups)
+ self.norm = nn.BatchNorm2d(out_features, affine=True)
+ self.pool = nn.AvgPool2d(kernel_size=(2, 2))
+
+ def forward(self, x):
+ out = self.conv(x)
+ out = self.norm(out)
+ out = F.relu(out)
+ out = self.pool(out)
+ return out
+
+
+class DownBlock3d(nn.Module):
+ """
+ Downsampling block for use in encoder.
+ """
+
+ def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1):
+ super(DownBlock3d, self).__init__()
+ '''
+ self.conv = nn.Conv3d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size,
+ padding=padding, groups=groups, stride=(1, 2, 2))
+ '''
+ self.conv = nn.Conv3d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size,
+ padding=padding, groups=groups)
+ self.norm = nn.BatchNorm3d(out_features, affine=True)
+ self.pool = nn.AvgPool3d(kernel_size=(1, 2, 2))
+
+ def forward(self, x):
+ out = self.conv(x)
+ out = self.norm(out)
+ out = F.relu(out)
+ out = self.pool(out)
+ return out
+
+
+class SameBlock2d(nn.Module):
+ """
+ Simple block, preserve spatial resolution.
+ """
+
+ def __init__(self, in_features, out_features, groups=1, kernel_size=3, padding=1, lrelu=False):
+ super(SameBlock2d, self).__init__()
+ self.conv = nn.Conv2d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size, padding=padding, groups=groups)
+ self.norm = nn.BatchNorm2d(out_features, affine=True)
+ if lrelu:
+ self.ac = nn.LeakyReLU()
+ else:
+ self.ac = nn.ReLU()
+
+ def forward(self, x):
+ out = self.conv(x)
+ out = self.norm(out)
+ out = self.ac(out)
+ return out
+
+
+class Encoder(nn.Module):
+ """
+ Hourglass Encoder
+ """
+
+ def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256):
+ super(Encoder, self).__init__()
+
+ down_blocks = []
+ for i in range(num_blocks):
+ down_blocks.append(DownBlock3d(in_features if i == 0 else min(max_features, block_expansion * (2 ** i)), min(max_features, block_expansion * (2 ** (i + 1))), kernel_size=3, padding=1))
+ self.down_blocks = nn.ModuleList(down_blocks)
+
+ def forward(self, x):
+ outs = [x]
+ for down_block in self.down_blocks:
+ outs.append(down_block(outs[-1]))
+ return outs
+
+
+class Decoder(nn.Module):
+ """
+ Hourglass Decoder
+ """
+
+ def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256):
+ super(Decoder, self).__init__()
+
+ up_blocks = []
+
+ for i in range(num_blocks)[::-1]:
+ in_filters = (1 if i == num_blocks - 1 else 2) * min(max_features, block_expansion * (2 ** (i + 1)))
+ out_filters = min(max_features, block_expansion * (2 ** i))
+ up_blocks.append(UpBlock3d(in_filters, out_filters, kernel_size=3, padding=1))
+
+ self.up_blocks = nn.ModuleList(up_blocks)
+ self.out_filters = block_expansion + in_features
+
+ self.conv = nn.Conv3d(in_channels=self.out_filters, out_channels=self.out_filters, kernel_size=3, padding=1)
+ self.norm = nn.BatchNorm3d(self.out_filters, affine=True)
+
+ def forward(self, x):
+ out = x.pop()
+ for up_block in self.up_blocks:
+ out = up_block(out)
+ skip = x.pop()
+ out = torch.cat([out, skip], dim=1)
+ out = self.conv(out)
+ out = self.norm(out)
+ out = F.relu(out)
+ return out
+
+
+class Hourglass(nn.Module):
+ """
+ Hourglass architecture.
+ """
+
+ def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256):
+ super(Hourglass, self).__init__()
+ self.encoder = Encoder(block_expansion, in_features, num_blocks, max_features)
+ self.decoder = Decoder(block_expansion, in_features, num_blocks, max_features)
+ self.out_filters = self.decoder.out_filters
+
+ def forward(self, x):
+ return self.decoder(self.encoder(x))
+
+
+class SPADE(nn.Module):
+ def __init__(self, norm_nc, label_nc):
+ super().__init__()
+
+ self.param_free_norm = nn.InstanceNorm2d(norm_nc, affine=False)
+ nhidden = 128
+
+ self.mlp_shared = nn.Sequential(
+ nn.Conv2d(label_nc, nhidden, kernel_size=3, padding=1),
+ nn.ReLU())
+ self.mlp_gamma = nn.Conv2d(nhidden, norm_nc, kernel_size=3, padding=1)
+ self.mlp_beta = nn.Conv2d(nhidden, norm_nc, kernel_size=3, padding=1)
+
+ def forward(self, x, segmap):
+ normalized = self.param_free_norm(x)
+ segmap = F.interpolate(segmap, size=x.size()[2:], mode='nearest')
+ actv = self.mlp_shared(segmap)
+ gamma = self.mlp_gamma(actv)
+ beta = self.mlp_beta(actv)
+ out = normalized * (1 + gamma) + beta
+ return out
+
+
+class SPADEResnetBlock(nn.Module):
+ def __init__(self, fin, fout, norm_G, label_nc, use_se=False, dilation=1):
+ super().__init__()
+ # Attributes
+ self.learned_shortcut = (fin != fout)
+ fmiddle = min(fin, fout)
+ self.use_se = use_se
+ # create conv layers
+ self.conv_0 = nn.Conv2d(fin, fmiddle, kernel_size=3, padding=dilation, dilation=dilation)
+ self.conv_1 = nn.Conv2d(fmiddle, fout, kernel_size=3, padding=dilation, dilation=dilation)
+ if self.learned_shortcut:
+ self.conv_s = nn.Conv2d(fin, fout, kernel_size=1, bias=False)
+ # apply spectral norm if specified
+ if 'spectral' in norm_G:
+ self.conv_0 = spectral_norm(self.conv_0)
+ self.conv_1 = spectral_norm(self.conv_1)
+ if self.learned_shortcut:
+ self.conv_s = spectral_norm(self.conv_s)
+ # define normalization layers
+ self.norm_0 = SPADE(fin, label_nc)
+ self.norm_1 = SPADE(fmiddle, label_nc)
+ if self.learned_shortcut:
+ self.norm_s = SPADE(fin, label_nc)
+
+ def forward(self, x, seg1):
+ x_s = self.shortcut(x, seg1)
+ dx = self.conv_0(self.actvn(self.norm_0(x, seg1)))
+ dx = self.conv_1(self.actvn(self.norm_1(dx, seg1)))
+ out = x_s + dx
+ return out
+
+ def shortcut(self, x, seg1):
+ if self.learned_shortcut:
+ x_s = self.conv_s(self.norm_s(x, seg1))
+ else:
+ x_s = x
+ return x_s
+
+ def actvn(self, x):
+ return F.leaky_relu(x, 2e-1)
+
+
+def filter_state_dict(state_dict, remove_name='fc'):
+ new_state_dict = {}
+ for key in state_dict:
+ if remove_name in key:
+ continue
+ new_state_dict[key] = state_dict[key]
+ return new_state_dict
+
+
+class GRN(nn.Module):
+ """ GRN (Global Response Normalization) layer
+ """
+
+ def __init__(self, dim):
+ super().__init__()
+ self.gamma = nn.Parameter(torch.zeros(1, 1, 1, dim))
+ self.beta = nn.Parameter(torch.zeros(1, 1, 1, dim))
+
+ def forward(self, x):
+ Gx = torch.norm(x, p=2, dim=(1, 2), keepdim=True)
+ Nx = Gx / (Gx.mean(dim=-1, keepdim=True) + 1e-6)
+ return self.gamma * (x * Nx) + self.beta + x
+
+
+class LayerNorm(nn.Module):
+ r""" LayerNorm that supports two data formats: channels_last (default) or channels_first.
+ The ordering of the dimensions in the inputs. channels_last corresponds to inputs with
+ shape (batch_size, height, width, channels) while channels_first corresponds to inputs
+ with shape (batch_size, channels, height, width).
+ """
+
+ def __init__(self, normalized_shape, eps=1e-6, data_format="channels_last"):
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(normalized_shape, dtype=torch.float32))
+ self.bias = nn.Parameter(torch.zeros(normalized_shape, dtype=torch.float32))
+ self.eps = float(eps)
+ self.data_format = data_format
+ if self.data_format not in ["channels_last", "channels_first"]:
+ raise NotImplementedError
+ self.normalized_shape = (normalized_shape, )
+
+ def _apply(self, fn):
+ """
+ 重写 _apply,完全接管参数的转换逻辑。
+ 拦截所有 .cuda(), .cpu(), .half(), .to() 操作。
+ """
+
+ for name, param in self._parameters.items():
+ if param is not None:
+ dummy_probe = param.data.view(-1)[:1]
+
+ try:
+ target_tensor = fn(dummy_probe)
+
+ target_device = target_tensor.device
+ target_dtype = target_tensor.dtype
+ except:
+ target_device = param.device
+ target_dtype = param.dtype
+
+ if name in ['weight', 'bias']:
+ # 核心逻辑:如果是 weight/bias,且目标是半精度,则强制保持 FP32
+ if target_dtype in [torch.float16, torch.bfloat16]:
+ new_data = param.data.to(device=target_device, dtype=torch.float32)
+ else:
+ new_data = fn(param.data)
+ else:
+ new_data = fn(param.data)
+
+ param.data = new_data
+
+ if param.grad is not None:
+ param.grad.data = param.grad.data.to(device=new_data.device, dtype=new_data.dtype)
+
+ for name, buf in self._buffers.items():
+ if buf is not None:
+ self._buffers[name] = fn(buf)
+
+ return self
+
+ def forward(self, x):
+ dtype = x.dtype
+ x = x.float()
+ if self.data_format == "channels_last":
+ x = F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)
+ elif self.data_format == "channels_first":
+ x = x.permute(0, 2, 3, 1) # BCHW → BHWC
+ x = F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)
+ x = x.permute(0, 3, 1, 2) # BHWC → BCHW
+ return x.to(dtype)
+
+
+def _no_grad_trunc_normal_(tensor, mean, std, a, b):
+ # Cut & paste from PyTorch official master until it's in a few official releases - RW
+ # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
+ def norm_cdf(x):
+ # Computes standard normal cumulative distribution function
+ return (1. + math.erf(x / math.sqrt(2.))) / 2.
+
+ if (mean < a - 2 * std) or (mean > b + 2 * std):
+ warnings.warn("mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
+ "The distribution of values may be incorrect.",
+ stacklevel=2)
+
+ with torch.no_grad():
+ # Values are generated by using a truncated uniform distribution and
+ # then using the inverse CDF for the normal distribution.
+ # Get upper and lower cdf values
+ l = norm_cdf((a - mean) / std)
+ u = norm_cdf((b - mean) / std)
+
+ # Uniformly fill tensor with values from [l, u], then translate to
+ # [2l-1, 2u-1].
+ tensor.uniform_(2 * l - 1, 2 * u - 1)
+
+ # Use inverse cdf transform for normal distribution to get truncated
+ # standard normal
+ tensor.erfinv_()
+
+ # Transform to proper mean, std
+ tensor.mul_(std * math.sqrt(2.))
+ tensor.add_(mean)
+
+ # Clamp to ensure it's in the proper range
+ tensor.clamp_(min=a, max=b)
+ return tensor
+
+
+def drop_path(x, drop_prob=0., training=False, scale_by_keep=True):
+ """ Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
+
+ This is the same as the DropConnect impl I created for EfficientNet, etc networks, however,
+ the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
+ See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for
+ changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use
+ 'survival rate' as the argument.
+
+ """
+ if drop_prob == 0. or not training:
+ return x
+ keep_prob = 1 - drop_prob
+ shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets
+ random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
+ if keep_prob > 0.0 and scale_by_keep:
+ random_tensor.div_(keep_prob)
+ return x * random_tensor
+
+
+class DropPath(nn.Module):
+ """ Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
+ """
+
+ def __init__(self, drop_prob=None, scale_by_keep=True):
+ super(DropPath, self).__init__()
+ self.drop_prob = drop_prob
+ self.scale_by_keep = scale_by_keep
+
+ def forward(self, x):
+ return drop_path(x, self.drop_prob, self.training, self.scale_by_keep)
+
+
+def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):
+ return _no_grad_trunc_normal_(tensor, mean, std, a, b)
+
+# From PyTorch internals
+def _ntuple(n):
+ def parse(x):
+ if isinstance(x, collections.abc.Iterable) and not isinstance(x, str):
+ return tuple(x)
+ return tuple(repeat(x, n))
+ return parse
+
+to_2tuple = _ntuple(2)
diff --git a/src/modeling/engine_model.py b/src/modeling/engine_model.py
new file mode 100644
index 0000000..ac41ba4
--- /dev/null
+++ b/src/modeling/engine_model.py
@@ -0,0 +1,308 @@
+import tensorrt as trt
+import pycuda.driver as cuda
+import pycuda.autoinit
+import numpy as np
+import torch
+import traceback
+import os
+from PIL import Image
+
+TRT_LOGGER = trt.Logger()
+SKIP_ENGINE_MODEL_CHECK = True
+
+def get_engine(engine_file_path):
+ if os.path.exists(engine_file_path):
+ print(f"Loading engine from file {engine_file_path}...")
+ with open(engine_file_path, "rb") as f, trt.Runtime(TRT_LOGGER) as runtime:
+ return runtime.deserialize_cuda_engine(f.read())
+ else:
+ print(f"No file named {engine_file_path}! Please check the input.")
+ return None
+
+
+def numpy_to_torch_dtype(np_dtype):
+ mapping = {
+ np.float32: torch.float,
+ np.float64: torch.double,
+ np.float16: torch.half,
+ np.int32: torch.int32,
+ np.int64: torch.int64,
+ np.int16: torch.int16,
+ np.int8: torch.int8,
+ np.uint8: torch.uint8,
+ np.bool_: torch.bool
+ }
+ return mapping.get(np_dtype, None)
+
+def match_shape(a, b):
+ if(len(a) == len(b)):
+ return tuple(a) == tuple(b)
+ elif len(a) > len(b):
+ if(a[0] == 1):
+ return match_shape(a[1:], b)
+ else:
+ if(b[0] == 1):
+ return match_shape(a, b[1:])
+ return False
+
+def match_dtype(a, b):
+ if(a.__class__ == torch.dtype):
+ a = torch.tensor(0,dtype=a).numpy().dtype
+ return a == b
+
+
+class EngineModel:
+ def __init__(self, engine_file_path, stream = None, device_int = 0, extra_lock = None):
+ self.device_int = device_int
+ self.extra_lock = extra_lock
+ if not(self.extra_lock is None):
+ self.extra_lock.acquire()
+ assert os.path.exists(engine_file_path), "Engine model path not exists!"
+ self.ctx = cuda.Device(self.device_int).make_context()
+ try:
+ self.engine = get_engine(engine_file_path) # 载入TensorRT引擎
+ input_nvars = 0
+ output_nvars = 0
+ self.input_names = []
+ self.output_names = []
+
+ # 【辅助函数】用于获取安全的 Shape (消除 -1)
+ def get_safe_shape(engine, name):
+ shape = engine.get_tensor_shape(name)
+ # 如果形状里包含 -1 (动态维度)
+ if -1 in shape:
+ # 获取 Profile 0 的 (min, opt, max)
+ # 取下标 [2] 即 Max Shape,确保分配足够大的显存
+ profile = engine.get_tensor_profile_shape(name, 0)
+ if profile:
+ print(f"[EngineModel] Detected dynamic shape for {name}: {shape} -> Using Max Profile: {profile[2]}")
+ return profile[2]
+ else:
+ # 如果获取不到 Profile (通常发生在 Output),这是一个风险点
+ # 这里为了防止报错,可以尝试打印警告
+ print(f"[EngineModel] Warning: Dynamic output {name} has no profile. Mem alloc might fail.")
+ return shape
+
+ for binding in self.engine: # 遍历所有tensor,区分Input/Output
+ mode = self.engine.get_tensor_mode(binding)
+ if(mode== trt.TensorIOMode.INPUT):
+ input_nvars += 1
+ self.input_names.append(binding)
+ elif(mode == trt.TensorIOMode.OUTPUT):
+ output_nvars += 1
+ self.output_names.append(binding)
+
+ self.input_nvars = input_nvars # input的数量
+ self.output_nvars = output_nvars # output的数量
+
+ self.input_shapes = {name : get_safe_shape(self.engine, name) for name in self.input_names} # 获取每个 I/O 的 shape 和 dtype
+ self.input_dtypes = {name : self.engine.get_tensor_dtype(name) for name in self.input_names}
+ self.input_nbytes = {
+ name : trt.volume(self.input_shapes[name]) * trt.nptype(self.input_dtypes[name])().itemsize
+ for name in self.input_names
+ } # nbytes = tensor 占多少 CUDA 内存(字节数)
+ self.output_shapes = {name : get_safe_shape(self.engine, name) for name in self.output_names}
+ self.output_dtypes = {name : self.engine.get_tensor_dtype(name) for name in self.output_names}
+ self.output_nbytes = {
+ name : trt.volume(self.output_shapes[name]) * trt.nptype(self.output_dtypes[name])().itemsize
+ for name in self.output_names
+ }
+ self.dinputs = {name : cuda.mem_alloc(self.input_nbytes[name]) for name in self.input_names} # 为每个输入/输出分配 CUDA 设备内存
+ self.doutputs = {name :cuda.mem_alloc(self.output_nbytes[name]) for name in self.output_names}
+ self.context = self.engine.create_execution_context() # 创建 ExecutionContext(执行上下文)
+ if stream is None:
+ self.stream = cuda.Stream()
+ else:
+ self.stream = stream
+ for name in self.input_names: # 绑定 tensor 到 context
+ self.context.set_tensor_address(name, int(self.dinputs[name]))
+ for name in self.output_names:
+ self.context.set_tensor_address(name, int(self.doutputs[name]))
+ self.houtputs = {
+ name :
+ cuda.pagelocked_empty(
+ trt.volume(self.output_shapes[name]), dtype=trt.nptype(self.output_dtypes[name])
+ ) for name in self.output_names
+ } # 分配 page-locked host 内存以存储输出
+ except:
+ self.ctx.pop()
+ raise Exception("CUDA Initialization Failed!")
+ self.ctx.pop()
+ if not(self.extra_lock is None):
+ self.extra_lock.release()
+
+ def __call__(self, skip_check=SKIP_ENGINE_MODEL_CHECK, output_list=[], return_tensor=False, **inputs):
+ if not skip_check:
+ for name in inputs:
+ assert name in self.input_names
+ assert match_shape(inputs[name].shape, self.input_shapes[name])
+ assert match_dtype(inputs[name].dtype, trt.nptype(self.input_dtypes[name]))
+ if not(self.extra_lock is None):
+ self.extra_lock.acquire()
+ self.ctx.push()
+ r = {}
+ try:
+
+ for name in inputs:
+ hinput = inputs[name]
+ if (isinstance(hinput,torch.Tensor) and hinput.device.type=="cuda" and hinput.device.index==self.device_int):
+ hinput_con = hinput.contiguous()
+ ptr = hinput_con.data_ptr()
+ cuda.memcpy_dtod_async(self.dinputs[name], ptr, self.input_nbytes[name], self.stream)
+ else:
+ hinput_con = np.ascontiguousarray(hinput)
+ cuda.memcpy_htod_async(self.dinputs[name], hinput_con, self.stream)
+ for name in self.input_names:
+ if name not in inputs:
+ self.context.set_input_shape(name, self.input_shapes[name])
+ self.context.execute_async_v3(self.stream.handle)
+ if(return_tensor):
+ for name in output_list:
+ t = torch.zeros(trt.volume(self.output_shapes[name]), device=f"cuda:{self.device_int}", dtype=numpy_to_torch_dtype(trt.nptype(self.output_dtypes[name])))
+ ptr = t.data_ptr()
+ cuda.memcpy_dtod_async(ptr, self.doutputs[name], self.output_nbytes[name], self.stream)
+ t = t.reshape(tuple(self.output_shapes[name]))
+ r[name] = t
+ else:
+ for name in output_list:
+ cuda.memcpy_dtoh_async(self.houtputs[name], self.doutputs[name], self.stream)
+ r[name] = self.houtputs[name]
+ self.stream.synchronize()
+ except Exception as e:
+ print("TensorRT Execution Failed!")
+ traceback.print_exc()
+ self.ctx.pop()
+ if not(self.extra_lock is None):
+ self.extra_lock.release()
+ return None
+ self.ctx.pop()
+ if not(self.extra_lock is None):
+ self.extra_lock.release()
+ return r
+
+
+ def prefill(self, skip_check=SKIP_ENGINE_MODEL_CHECK, **inputs):
+ if not (skip_check):
+ for name in inputs:
+ in_input = (name in self.input_names)
+ assert in_input or (name in self.output_names)
+ assert match_shape(inputs[name].shape, self.input_shapes[name] if in_input else self.output_shapes[name])
+ assert match_dtype(inputs[name].dtype, trt.nptype(self.input_dtypes[name] if in_input else self.output_dtypes[name]))
+ if not(self.extra_lock is None):
+ self.extra_lock.acquire()
+ self.ctx.push()
+ try:
+ for name in inputs:
+ in_input = (name in self.input_names)
+ hinput = inputs[name]
+
+ dst_ptr = self.dinputs[name] if in_input else self.doutputs[name]
+ real_nbytes = 0
+ if isinstance(hinput, torch.Tensor):
+ real_nbytes = hinput.numel() * hinput.element_size()
+ else:
+ # 假设是 numpy
+ real_nbytes = hinput.nbytes
+
+ if (isinstance(hinput,torch.Tensor) and hinput.device.type=="cuda" and hinput.device.index==self.device_int):
+ hinput_con = hinput.contiguous()
+ ptr = hinput_con.data_ptr()
+ cuda.memcpy_dtod_async(dst_ptr, ptr, real_nbytes, self.stream)
+ else:
+ hinput_con = np.ascontiguousarray(hinput)
+ cuda.memcpy_htod_async(dst_ptr, hinput, self.stream)
+ self.stream.synchronize()
+ except Exception as e:
+ traceback.print_exc()
+ self.ctx.pop()
+ if not(self.extra_lock is None):
+ self.extra_lock.release()
+ return False
+ self.ctx.pop()
+ if not(self.extra_lock is None):
+ self.extra_lock.release()
+ return True
+
+ def __repr__(self):
+ r = "TensorRTEngineModel(\n\tInput=[\n"
+ for name in self.input_names:
+ r += f"\t\t{name}: \t{trt.nptype(self.input_dtypes[name]).__name__}{self.input_shapes[name]},\n"
+ r += "\t],Output=[\n"
+ for name in self.output_names:
+ r += f"\t\t{name}: \t{trt.nptype(self.output_dtypes[name]).__name__}{self.output_shapes[name]},\n"
+ r+="\t]\n)"
+ return r
+
+ def link(self, other, var_map, skip_check=SKIP_ENGINE_MODEL_CHECK):
+ assert self.device_int == other.device_int
+ if not (skip_check):
+ for source in var_map:
+ assert source in other.output_names
+ target = var_map[source]
+ assert target in self.input_names
+ assert match_shape(other.output_shapes[source], self.input_shapes[target])
+ assert match_dtype(other.output_dtypes[source], self.input_dtypes[target])
+
+ if not(self.extra_lock is None):
+ self.extra_lock.acquire()
+ self.ctx.push()
+ try:
+ for source in var_map:
+ target = var_map[source]
+ self.context.set_tensor_address(target, int(other.doutputs[source]))
+ except Exception as e:
+ traceback.print_exc()
+ self.ctx.pop()
+ if not(self.extra_lock is None):
+ self.extra_lock.release()
+ return False
+ self.ctx.pop()
+ if not(self.extra_lock is None):
+ self.extra_lock.release()
+ return True
+
+ def bind(self, var_map, skip_check=SKIP_ENGINE_MODEL_CHECK):
+ if not (skip_check):
+ for source in var_map:
+ assert source in self.output_names
+ target = var_map[source]
+ assert target in self.input_names
+ assert match_shape(self.output_shapes[source], self.input_shapes[target])
+ assert match_dtype(self.output_dtypes[source], self.input_dtypes[target])
+
+ if not(self.extra_lock is None):
+ self.extra_lock.acquire()
+ self.ctx.push()
+ try:
+ for source in var_map:
+ target = var_map[source]
+ self.context.set_tensor_address(target, int(self.doutputs[source]))
+ except Exception as e:
+ traceback.print_exc()
+ self.ctx.pop()
+ if not(self.extra_lock is None):
+ self.extra_lock.release()
+ return False
+ self.ctx.pop()
+ if not(self.extra_lock is None):
+ self.extra_lock.release()
+ return True
+
+ def unlink(self):
+
+ if not(self.extra_lock is None):
+ self.extra_lock.acquire()
+ self.ctx.push()
+ try:
+ for name in self.input_names:
+ self.context.set_tensor_address(name, int(self.dinputs[name]))
+ except:
+ self.ctx.pop()
+ if not(self.extra_lock is None):
+ self.extra_lock.release()
+ return False
+ self.ctx.pop()
+ if not(self.extra_lock is None):
+ self.extra_lock.release()
+ return True
+
diff --git a/src/modeling/framed_models.py b/src/modeling/framed_models.py
new file mode 100644
index 0000000..a0508f8
--- /dev/null
+++ b/src/modeling/framed_models.py
@@ -0,0 +1,177 @@
+import torch
+from torch import nn
+from einops import rearrange
+from polygraphy.backend.trt import Profile
+
+class unet_work(nn.Module): # Ugly Power Strip
+ def __init__(self, pose_guider, motion_encoder, unet, vae, scheduler, timestep):
+ super().__init__()
+ self.pose_guider = pose_guider
+ self.motion_encoder = motion_encoder
+ self.unet = unet
+ self.vae = vae
+ self.scheduler = scheduler
+ self.timesteps = timestep
+
+ def decode_slice(self, vae, x):
+ x = x / 0.18215
+ x = vae.decode(x).sample
+ x = rearrange(x, "b c h w -> b h w c")
+ x = (x / 2 + 0.5).clamp(0, 1)
+ return x
+
+ def forward(self, sample, encoder_hidden_states, motion_hidden_states, motion, pose_cond_fea, pose, new_noise,
+ d00, d01, d10, d11, d20, d21, m, u10, u11, u12, u20, u21, u22, u30, u31, u32
+ ):
+ new_pose_cond_fea = self.pose_guider(pose)
+ pose_cond_fea = torch.cat([pose_cond_fea, new_pose_cond_fea], dim=2)
+ new_motion_hidden_states = self.motion_encoder(motion)
+ motion_hidden_states = torch.cat([motion_hidden_states, new_motion_hidden_states], dim=1)
+ encoder_hidden_states = [encoder_hidden_states, motion_hidden_states]
+ score = self.unet(sample, self.timesteps, encoder_hidden_states, pose_cond_fea, d00, d01, d10, d11, d20, d21, m, u10, u11, u12, u20, u21, u22, u30, u31, u32)
+ score = rearrange(score, 'b c f h w -> (b f) c h w')
+ sample = rearrange(sample, 'b c f h w -> (b f) c h w')
+ latents_model_input, pred_original_sample = self.scheduler.step(
+ score, self.timesteps, sample, return_dict=False
+ )
+ latents_model_input = latents_model_input.to(sample.dtype)
+ pred_original_sample = pred_original_sample.to(sample.dtype)
+ latents_model_input = rearrange(latents_model_input, '(b f) c h w -> b c f h w', f=16)
+ pred_video = self.decode_slice(self.vae, pred_original_sample[:4])
+ latents = torch.cat([latents_model_input[:, :, 4:, :, :], new_noise], dim=2)
+ pose_cond_fea_out = pose_cond_fea[:, :, 4:, :, :]
+ motion_hidden_states_out = motion_hidden_states[:, 4:, :, :]
+ motion_out = motion_hidden_states[:, :1, :, :]
+ return pred_video, latents, pose_cond_fea_out, motion_hidden_states_out, motion_out, pred_original_sample[:1]
+
+ def get_sample_input(self, batchsize, height, width, dtype, device):
+ tw, ts, tb = 4, 4, 16 # temporal window size| temporal adaptive steps | temporal batch size
+ ml, mc, mh, mw= 32, 16, 224, 224 # motion latent size | motion channels
+ b, h, w = batchsize, height, width
+ lh, lw = height // 8, width // 8 # latent height | width
+ cd0, cd1, cd2, cm, cu1, cu2, cu3 = 320, 640, 1280, 1280, 1280, 640, 320 # unet channels
+ emb = 768 # CLIP Embedding Dims | TAESDV Channels
+ lc, ic = 4, 3 # latent | image channels
+ profile = {
+ "sample" : [b, lc, tb, lh, lw],
+ "encoder_hidden_states" : [b, 1, emb],
+ "motion_hidden_states" : [b, tw * (ts - 1), ml, mc],
+ "motion": [b, ic, tw, mh, mw],
+ "pose_cond_fea" : [b, cd0, tw * (ts - 1), lh, lw],
+ "pose" : [b, ic, tw, h, w],
+ "new_noise" : [b, lc, tw, lh, lw],
+ "d00" : [b, lh * lw, cd0],
+ "d01" : [b, lh * lw, cd0],
+ "d10" : [b, lh * lw // 4, cd1],
+ "d11" : [b, lh * lw // 4, cd1],
+ "d20" : [b, lh * lw // 16, cd2],
+ "d21" : [b, lh * lw // 16, cd2],
+ "m" : [b, lh * lw // 64, cm],
+ "u10" : [b, lh * lw // 16, cu1],
+ "u11" : [b, lh * lw // 16, cu1],
+ "u12" : [b, lh * lw // 16, cu1],
+ "u20" : [b, lh * lw // 4, cu2],
+ "u21" : [b, lh * lw // 4, cu2],
+ "u22" : [b, lh * lw // 4, cu2],
+ "u30" : [b, lh * lw, cu3],
+ "u31" : [b, lh * lw, cu3],
+ "u32" : [b, lh * lw, cu3],
+ }
+ return {k: torch.randn(profile[k], dtype=dtype, device=device) for k in profile}
+
+ def get_input_names(self):
+ return ["sample", "encoder_hidden_states", "motion_hidden_states",
+ "motion", "pose_cond_fea", "pose", "new_noise",
+ "d00", "d01", "d10", "d11", "d20", "d21", "m", "u10", "u11", "u12",
+ "u20", "u21", "u22", "u30", "u31", "u32"]
+
+ def get_output_names(self):
+ return ["pred_video", "latents", "pose_cond_fea_out",
+ "motion_hidden_states_out", "motion_out", "latent_first"]
+
+ def get_dynamic_axes(self):
+ dynamic_axes = {
+ "sample": {3:"h_64", 4:"w_64"},
+ "pose_cond_fea": {3:"h_64", 4:"w_64"},
+ "pose": {3:"h_512", 4:"h_512"},
+ "new_noise": {3: "h_64", 4: "w_64"},
+ "d00" : {1: "len_4096"},
+ "d01" : {1: "len_4096"},
+ "u30" : {1: "len_4096"},
+ "u31" : {1: "len_4096"},
+ "u32" : {1: "len_4096"},
+ "d10" : {1: "len_1024"},
+ "d11" : {1: "len_1024"},
+ "u20" : {1: "len_1024"},
+ "u21" : {1: "len_1024"},
+ "u22" : {1: "len_1024"},
+ "d20" : {1: "len_256"},
+ "d21" : {1: "len_256"},
+ "u10" : {1: "len_256"},
+ "u11" : {1: "len_256"},
+ "u12" : {1: "len_256"},
+ "m" : {1: "len_64"},
+ }
+ return dynamic_axes
+
+ def get_dynamic_map(self, batchsize, height, width):
+ tw, ts, tb = 4, 4, 16 # temporal window size| temporal adaptive steps | temporal batch size
+ ml, mc, mh, mw= 32, 16, 224, 224 # motion latent size | motion channels
+ b, h, w = batchsize, height, width
+ lh, lw = height // 8, width // 8 # latent height | width
+ cd0, cd1, cd2, cm, cu1, cu2, cu3 = 320, 640, 1280, 1280, 1280, 640, 320 # unet channels
+ emb = 768 # CLIP Embedding Dims | TAESDV Channels
+ lc, ic = 4, 3 # latent | image channels
+
+ fixed_inputs_map = {
+ "sample": (b, lc, tb, lh, lw),
+ "encoder_hidden_states": (b, 1, emb),
+ "motion_hidden_states": (b, tw * (ts - 1), ml, mc),
+ "motion": (b, ic, tw, mh, mw),
+ "pose_cond_fea": (b, cd0, tw * (ts - 1), lh, lw),
+ "pose": (b, ic, tw, h, w),
+ "new_noise": (b, lc, tw, lh, lw),
+ }
+
+ dynamic_inputs_map = {
+ "d00": (b, lh * lw, cd0),
+ "d01": (b, lh * lw, cd0),
+ "d10": (b, lh * lw // 4, cd1),
+ "d11": (b, lh * lw // 4, cd1),
+ "d20": (b, lh * lw // 16, cd2),
+ "d21": (b, lh * lw // 16, cd2),
+ "m": (b, lh * lw // 64, cm),
+ "u10": (b, lh * lw // 16, cu1),
+ "u11": (b, lh * lw // 16, cu1),
+ "u12": (b, lh * lw // 16, cu1),
+ "u20": (b, lh * lw // 4, cu2),
+ "u21": (b, lh * lw // 4, cu2),
+ "u22": (b, lh * lw // 4, cu2),
+ "u30": (b, lh * lw, cu3),
+ "u31": (b, lh * lw, cu3),
+ "u32": (b, lh * lw, cu3),
+ }
+
+ profile = Profile()
+
+ for name, shape in fixed_inputs_map.items():
+ shape_tuple = tuple(shape)
+ profile.add(name, min=shape_tuple, opt=shape_tuple, max=shape_tuple)
+
+ for name, base_shape in dynamic_inputs_map.items():
+
+ dim0, dim1_base, dim2 = base_shape
+
+ val_1x = dim1_base * 1
+ val_2x = dim1_base * 2
+ val_4x = dim1_base * 4
+
+ min_shape = (dim0, val_1x, dim2)
+ opt_shape = (dim0, val_2x, dim2)
+ max_shape = (dim0, val_4x, dim2)
+
+ profile.add(name, min=min_shape, opt=opt_shape, max=max_shape)
+
+ print(f"Dynamic: {name:<5} | Base(1x): {dim1_base:<5} | Range: {val_1x} ~ {val_4x} | Opt: {val_2x}")
+
+ return profile
\ No newline at end of file
diff --git a/src/modeling/onnx_export.py b/src/modeling/onnx_export.py
new file mode 100644
index 0000000..c8b2069
--- /dev/null
+++ b/src/modeling/onnx_export.py
@@ -0,0 +1,102 @@
+# adapted from https://github.com/NVIDIA/TensorRT/blob/main/demo/Diffusion/utilities.py
+#
+# Copyright 2022 The HuggingFace Inc. team.
+# SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import onnx
+import gc
+import onnx_graphsurgeon as gs
+import torch
+from onnx import shape_inference
+from polygraphy.backend.onnx.loader import fold_constants
+import os
+from onnxsim import simplify
+
+def export_onnx(
+ model,
+ onnx_path: str,
+ opt_image_height: int,
+ opt_image_width: int,
+ opt_batch_size: int,
+ onnx_opset: int,
+ dtype,
+ device,
+ auto_cast: bool = True,
+):
+ from contextlib import contextmanager
+
+ @contextmanager
+ def auto_cast_manager(enabled):
+ if enabled:
+ with torch.inference_mode(), torch.autocast("cuda"):
+ yield
+ else:
+ yield
+
+ # 确保父目录存在
+ os.makedirs(os.path.dirname(onnx_path), exist_ok=True)
+
+ with auto_cast_manager(auto_cast):
+ inputs = model.get_sample_input(opt_batch_size, opt_image_height, opt_image_width, dtype, device)
+
+ print(model.get_output_names())
+ print(f"开始导出 ONNX 模型到: {onnx_path} ...")
+ torch.onnx.utils.export(
+ model,
+ inputs,
+ onnx_path,
+ export_params=True,
+ opset_version=onnx_opset,
+ do_constant_folding=True,
+ input_names=model.get_input_names(),
+ output_names=model.get_output_names(),
+ dynamic_axes=model.get_dynamic_axes(),
+ )
+
+ del model
+ gc.collect()
+ torch.cuda.empty_cache()
+
+def optimize_onnx(onnx_path, onnx_opt_path):
+ model = onnx.load(onnx_path)
+ name = os.path.splitext(os.path.basename(onnx_opt_path))[0]
+ model_opt = model
+
+ print(f"Saving to {onnx_opt_path}...")
+ onnx.save(
+ model_opt,
+ onnx_opt_path,
+ save_as_external_data=True,
+ all_tensors_to_one_file=True,
+ location=f"{name}.onnx.data",
+ size_threshold=1024
+ )
+ print("Optimization done.")
+
+def handle_onnx_batch_norm(onnx_path: str):
+ onnx_model = onnx.load(onnx_path)
+ for node in onnx_model.graph.node:
+ if node.op_type == "BatchNormalization":
+ for attribute in node.attribute:
+ if attribute.name == "training_mode":
+ if attribute.i == 1:
+ node.output.remove(node.output[1])
+ node.output.remove(node.output[1])
+ attribute.i = 0
+
+ onnx.save_model(onnx_model, onnx_path)
\ No newline at end of file
diff --git a/src/models/attention.py b/src/models/attention.py
new file mode 100644
index 0000000..fd8b435
--- /dev/null
+++ b/src/models/attention.py
@@ -0,0 +1,463 @@
+from typing import Any, Dict, Optional
+
+import torch
+from diffusers.models.attention import AdaLayerNorm, Attention, FeedForward
+from diffusers.models.embeddings import SinusoidalPositionalEmbedding
+from einops import rearrange
+from torch import nn
+
+from diffusers.models.attention import *
+from diffusers.models.attention_processor import *
+
+class BasicTransformerBlock(nn.Module):
+ r"""
+ A basic Transformer block.
+
+ Parameters:
+ dim (`int`): The number of channels in the input and output.
+ num_attention_heads (`int`): The number of heads to use for multi-head attention.
+ attention_head_dim (`int`): The number of channels in each head.
+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
+ cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention.
+ activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward.
+ num_embeds_ada_norm (:
+ obj: `int`, *optional*): The number of diffusion steps used during training. See `Transformer2DModel`.
+ attention_bias (:
+ obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter.
+ only_cross_attention (`bool`, *optional*):
+ Whether to use only cross-attention layers. In this case two cross attention layers are used.
+ double_self_attention (`bool`, *optional*):
+ Whether to use two self-attention layers. In this case no cross attention layers are used.
+ upcast_attention (`bool`, *optional*):
+ Whether to upcast the attention computation to float32. This is useful for mixed precision training.
+ norm_elementwise_affine (`bool`, *optional*, defaults to `True`):
+ Whether to use learnable elementwise affine parameters for normalization.
+ norm_type (`str`, *optional*, defaults to `"layer_norm"`):
+ The normalization layer to use. Can be `"layer_norm"`, `"ada_norm"` or `"ada_norm_zero"`.
+ final_dropout (`bool` *optional*, defaults to False):
+ Whether to apply a final dropout after the last feed-forward layer.
+ attention_type (`str`, *optional*, defaults to `"default"`):
+ The type of attention to use. Can be `"default"` or `"gated"` or `"gated-text-image"`.
+ positional_embeddings (`str`, *optional*, defaults to `None`):
+ The type of positional embeddings to apply to.
+ num_positional_embeddings (`int`, *optional*, defaults to `None`):
+ The maximum number of positional embeddings to apply.
+ """
+
+ def __init__(
+ self,
+ dim: int,
+ num_attention_heads: int,
+ attention_head_dim: int,
+ dropout=0.0,
+ cross_attention_dim: Optional[int] = None,
+ activation_fn: str = "geglu",
+ num_embeds_ada_norm: Optional[int] = None,
+ attention_bias: bool = False,
+ only_cross_attention: bool = False,
+ double_self_attention: bool = False,
+ upcast_attention: bool = False,
+ norm_elementwise_affine: bool = True,
+ norm_type: str = "layer_norm", # 'layer_norm', 'ada_norm', 'ada_norm_zero', 'ada_norm_single'
+ norm_eps: float = 1e-5,
+ final_dropout: bool = False,
+ attention_type: str = "default",
+ positional_embeddings: Optional[str] = None,
+ num_positional_embeddings: Optional[int] = None,
+ ):
+ super().__init__()
+ self.only_cross_attention = only_cross_attention
+
+ self.use_ada_layer_norm_zero = (
+ num_embeds_ada_norm is not None
+ ) and norm_type == "ada_norm_zero"
+ self.use_ada_layer_norm = (
+ num_embeds_ada_norm is not None
+ ) and norm_type == "ada_norm"
+ self.use_ada_layer_norm_single = norm_type == "ada_norm_single"
+ self.use_layer_norm = norm_type == "layer_norm"
+
+ if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
+ raise ValueError(
+ f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
+ f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."
+ )
+
+ if positional_embeddings and (num_positional_embeddings is None):
+ raise ValueError(
+ "If `positional_embedding` type is defined, `num_positition_embeddings` must also be defined."
+ )
+
+ if positional_embeddings == "sinusoidal":
+ self.pos_embed = SinusoidalPositionalEmbedding(
+ dim, max_seq_length=num_positional_embeddings
+ )
+ else:
+ self.pos_embed = None
+
+ # Define 3 blocks. Each block has its own normalization layer.
+ # 1. Self-Attn
+ if self.use_ada_layer_norm:
+ self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm)
+ elif self.use_ada_layer_norm_zero:
+ self.norm1 = AdaLayerNormZero(dim, num_embeds_ada_norm)
+ else:
+ self.norm1 = nn.LayerNorm(
+ dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps
+ )
+
+ self.attn1 = Attention(
+ query_dim=dim,
+ heads=num_attention_heads,
+ dim_head=attention_head_dim,
+ dropout=dropout,
+ bias=attention_bias,
+ cross_attention_dim=cross_attention_dim if only_cross_attention else None,
+ upcast_attention=upcast_attention,
+ )
+
+ # 2. Cross-Attn
+ if cross_attention_dim is not None or double_self_attention:
+ # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
+ # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
+ # the second cross attention block.
+ self.norm2 = (
+ AdaLayerNorm(dim, num_embeds_ada_norm)
+ if self.use_ada_layer_norm
+ else nn.LayerNorm(
+ dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps
+ )
+ )
+ self.attn2 = Attention(
+ query_dim=dim,
+ cross_attention_dim=cross_attention_dim
+ if not double_self_attention
+ else None,
+ heads=num_attention_heads,
+ dim_head=attention_head_dim,
+ dropout=dropout,
+ bias=attention_bias,
+ upcast_attention=upcast_attention,
+ ) # is self-attn if encoder_hidden_states is none
+ else:
+ self.norm2 = None
+ self.attn2 = None
+
+ # 3. Feed-forward
+ if not self.use_ada_layer_norm_single:
+ self.norm3 = nn.LayerNorm(
+ dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps
+ )
+
+ self.ff = FeedForward(
+ dim,
+ dropout=dropout,
+ activation_fn=activation_fn,
+ final_dropout=final_dropout,
+ )
+
+ # 4. Fuser
+ if attention_type == "gated" or attention_type == "gated-text-image":
+ self.fuser = GatedSelfAttentionDense(
+ dim, cross_attention_dim, num_attention_heads, attention_head_dim
+ )
+
+ # 5. Scale-shift for PixArt-Alpha.
+ if self.use_ada_layer_norm_single:
+ self.scale_shift_table = nn.Parameter(torch.randn(6, dim) / dim**0.5)
+
+ # let chunk size default to None
+ self._chunk_size = None
+ self._chunk_dim = 0
+
+ def set_chunk_feed_forward(self, chunk_size: Optional[int], dim: int = 0):
+ # Sets chunk feed-forward
+ self._chunk_size = chunk_size
+ self._chunk_dim = dim
+
+ def forward(
+ self,
+ hidden_states: torch.FloatTensor,
+ attention_mask: Optional[torch.FloatTensor] = None,
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
+ timestep: Optional[torch.LongTensor] = None,
+ cross_attention_kwargs: Dict[str, Any] = None,
+ class_labels: Optional[torch.LongTensor] = None,
+ ) -> torch.FloatTensor:
+ # Notice that normalization is always applied before the real computation in the following blocks.
+ # 0. Self-Attention
+ batch_size = hidden_states.shape[0]
+
+ if self.use_ada_layer_norm:
+ norm_hidden_states = self.norm1(hidden_states, timestep)
+ elif self.use_ada_layer_norm_zero:
+ norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(
+ hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype
+ )
+ elif self.use_layer_norm:
+ norm_hidden_states = self.norm1(hidden_states)
+ elif self.use_ada_layer_norm_single:
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
+ self.scale_shift_table[None] + timestep.reshape(batch_size, 6, -1)
+ ).chunk(6, dim=1)
+ norm_hidden_states = self.norm1(hidden_states)
+ norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa
+ norm_hidden_states = norm_hidden_states.squeeze(1)
+ else:
+ raise ValueError("Incorrect norm used")
+
+ if self.pos_embed is not None:
+ norm_hidden_states = self.pos_embed(norm_hidden_states)
+
+ # 1. Retrieve lora scale.
+ lora_scale = (
+ cross_attention_kwargs.get("scale", 1.0)
+ if cross_attention_kwargs is not None
+ else 1.0
+ )
+
+ # 2. Prepare GLIGEN inputs
+ cross_attention_kwargs = (
+ cross_attention_kwargs.copy() if cross_attention_kwargs is not None else {}
+ )
+ gligen_kwargs = cross_attention_kwargs.pop("gligen", None)
+
+ attn_output = self.attn1(
+ norm_hidden_states,
+ encoder_hidden_states=encoder_hidden_states
+ if self.only_cross_attention
+ else None,
+ attention_mask=attention_mask,
+ **cross_attention_kwargs,
+ )
+ if self.use_ada_layer_norm_zero:
+ attn_output = gate_msa.unsqueeze(1) * attn_output
+ elif self.use_ada_layer_norm_single:
+ attn_output = gate_msa * attn_output
+
+ hidden_states = attn_output + hidden_states
+ if hidden_states.ndim == 4:
+ hidden_states = hidden_states.squeeze(1)
+
+ # 2.5 GLIGEN Control
+ if gligen_kwargs is not None:
+ hidden_states = self.fuser(hidden_states, gligen_kwargs["objs"])
+
+ # 3. Cross-Attention
+ if self.attn2 is not None:
+ if self.use_ada_layer_norm:
+ norm_hidden_states = self.norm2(hidden_states, timestep)
+ elif self.use_ada_layer_norm_zero or self.use_layer_norm:
+ norm_hidden_states = self.norm2(hidden_states)
+ elif self.use_ada_layer_norm_single:
+ # For PixArt norm2 isn't applied here:
+ # https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L70C1-L76C103
+ norm_hidden_states = hidden_states
+ else:
+ raise ValueError("Incorrect norm")
+
+ if self.pos_embed is not None and self.use_ada_layer_norm_single is False:
+ norm_hidden_states = self.pos_embed(norm_hidden_states)
+
+ attn_output = self.attn2(
+ norm_hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ attention_mask=encoder_attention_mask,
+ **cross_attention_kwargs,
+ )
+ hidden_states = attn_output + hidden_states
+
+ # 4. Feed-forward
+ if not self.use_ada_layer_norm_single:
+ norm_hidden_states = self.norm3(hidden_states)
+
+ if self.use_ada_layer_norm_zero:
+ norm_hidden_states = (
+ norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
+ )
+
+ if self.use_ada_layer_norm_single:
+ norm_hidden_states = self.norm2(hidden_states)
+ norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp
+
+ ff_output = self.ff(norm_hidden_states, scale=lora_scale)
+
+ if self.use_ada_layer_norm_zero:
+ ff_output = gate_mlp.unsqueeze(1) * ff_output
+ elif self.use_ada_layer_norm_single:
+ ff_output = gate_mlp * ff_output
+
+ hidden_states = ff_output + hidden_states
+ if hidden_states.ndim == 4:
+ hidden_states = hidden_states.squeeze(1)
+
+ return hidden_states
+
+
+class TemporalBasicTransformerBlock(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ num_attention_heads: int,
+ attention_head_dim: int,
+ dropout=0.0,
+ cross_attention_dim: Optional[int] = None,
+ activation_fn: str = "geglu",
+ num_embeds_ada_norm: Optional[int] = None,
+ attention_bias: bool = False,
+ only_cross_attention: bool = False,
+ upcast_attention: bool = False,
+ unet_use_cross_frame_attention=None,
+ unet_use_temporal_attention=None,
+ ):
+ super().__init__()
+ self.only_cross_attention = only_cross_attention
+ self.use_ada_layer_norm = num_embeds_ada_norm is not None
+ self.unet_use_cross_frame_attention = unet_use_cross_frame_attention
+ self.unet_use_temporal_attention = unet_use_temporal_attention
+
+ # SC-Attn
+ self.attn1 = Attention(
+ query_dim=dim,
+ heads=num_attention_heads,
+ dim_head=attention_head_dim,
+ dropout=dropout,
+ bias=attention_bias,
+ upcast_attention=upcast_attention,
+ )
+ self.norm1 = (
+ AdaLayerNorm(dim, num_embeds_ada_norm)
+ if self.use_ada_layer_norm
+ else nn.LayerNorm(dim)
+ )
+
+ # Cross-Attn
+ if cross_attention_dim is not None:
+ self.attn2 = Attention(
+ query_dim=dim,
+ cross_attention_dim=cross_attention_dim,
+ heads=num_attention_heads,
+ dim_head=attention_head_dim,
+ dropout=dropout,
+ bias=attention_bias,
+ upcast_attention=upcast_attention,
+ )
+ else:
+ self.attn2 = None
+
+ if cross_attention_dim is not None:
+ self.norm2 = (
+ AdaLayerNorm(dim, num_embeds_ada_norm)
+ if self.use_ada_layer_norm
+ else nn.LayerNorm(dim)
+ )
+ else:
+ self.norm2 = None
+
+ # Feed-forward
+ self.ff = FeedForward(dim, dropout=dropout, activation_fn=activation_fn)
+ self.norm3 = nn.LayerNorm(dim)
+ self.use_ada_layer_norm_zero = False
+
+ # Temp-Attn
+ assert unet_use_temporal_attention is not None
+ if unet_use_temporal_attention:
+ self.attn_temp = Attention(
+ query_dim=dim,
+ heads=num_attention_heads,
+ dim_head=attention_head_dim,
+ dropout=dropout,
+ bias=attention_bias,
+ upcast_attention=upcast_attention,
+ )
+ nn.init.zeros_(self.attn_temp.to_out[0].weight.data)
+ self.norm_temp = (
+ AdaLayerNorm(dim, num_embeds_ada_norm)
+ if self.use_ada_layer_norm
+ else nn.LayerNorm(dim)
+ )
+
+ def set_use_cross_frame_attention(self, value):
+ self.unet_use_cross_frame_attention = value
+
+ def forward(
+ self,
+ hidden_states,
+ encoder_hidden_states=None,
+ timestep=None,
+ attention_mask=None,
+ video_length=None,
+ reference=None,
+ ):
+ norm_hidden_states = (
+ self.norm1(hidden_states, timestep)
+ if self.use_ada_layer_norm
+ else self.norm1(hidden_states)
+ )
+
+ if self.unet_use_cross_frame_attention:
+ hidden_states = (
+ self.attn1(
+ norm_hidden_states,
+ attention_mask=attention_mask,
+ video_length=video_length,
+ )
+ + hidden_states
+ )
+ else:
+ if reference is not None:
+ bank_fea = [
+ rearrange(
+ reference.unsqueeze(1).repeat(1, video_length, 1, 1),
+ "b t l c -> (b t) l c",
+ )
+ ]
+ modify_norm_hidden_states = torch.cat(
+ [norm_hidden_states] + bank_fea, dim=1
+ )
+
+ hidden_states = (
+ self.attn1(norm_hidden_states, encoder_hidden_states=modify_norm_hidden_states, attention_mask=attention_mask)
+ + hidden_states
+ )
+ else:
+ hidden_states = (
+ self.attn1(norm_hidden_states, attention_mask=attention_mask)
+ + hidden_states
+ )
+
+ if self.attn2 is not None:
+ # Cross-Attention
+ norm_hidden_states = (
+ self.norm2(hidden_states, timestep)
+ if self.use_ada_layer_norm
+ else self.norm2(hidden_states)
+ )
+ hidden_states = (
+ self.attn2(
+ norm_hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ attention_mask=attention_mask,
+ )
+ + hidden_states
+ )
+
+ # Feed-forward
+ hidden_states = self.ff(self.norm3(hidden_states)) + hidden_states
+
+ # Temporal-Attention
+ if self.unet_use_temporal_attention:
+ d = hidden_states.shape[1]
+ hidden_states = rearrange(
+ hidden_states, "(b f) d c -> (b d) f c", f=video_length
+ )
+ norm_hidden_states = (
+ self.norm_temp(hidden_states, timestep)
+ if self.use_ada_layer_norm
+ else self.norm_temp(hidden_states)
+ )
+ hidden_states = self.attn_temp(norm_hidden_states) + hidden_states
+ hidden_states = rearrange(hidden_states, "(b d) f c -> (b f) d c", d=d)
+
+ return hidden_states
diff --git a/src/models/motion_encoder/FAN_feature_extractor.py b/src/models/motion_encoder/FAN_feature_extractor.py
new file mode 100644
index 0000000..1fca87d
--- /dev/null
+++ b/src/models/motion_encoder/FAN_feature_extractor.py
@@ -0,0 +1,334 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import Dict, Union
+from diffusers.models.attention_processor import AttentionProcessor
+
+
+def conv3x3(in_planes, out_planes, strd=1, padding=1, bias=False):
+ "3x3 convolution with padding"
+ return nn.Conv2d(in_planes, out_planes, kernel_size=3,
+ stride=strd, padding=padding, bias=bias)
+
+
+class ConvBlock(nn.Module):
+ def __init__(self, in_planes, out_planes):
+ super(ConvBlock, self).__init__()
+ self.bn1 = nn.BatchNorm2d(in_planes)
+ self.conv1 = conv3x3(in_planes, int(out_planes / 2))
+ self.bn2 = nn.BatchNorm2d(int(out_planes / 2))
+ self.conv2 = conv3x3(int(out_planes / 2), int(out_planes / 4))
+ self.bn3 = nn.BatchNorm2d(int(out_planes / 4))
+ self.conv3 = conv3x3(int(out_planes / 4), int(out_planes / 4))
+
+ if in_planes != out_planes:
+ self.downsample = nn.Sequential(
+ nn.BatchNorm2d(in_planes, eps=1e-4),
+ nn.ReLU(True),
+ nn.Conv2d(in_planes, out_planes,
+ kernel_size=1, stride=1, bias=False),
+ )
+ else:
+ self.downsample = None
+
+ def forward(self, x):
+ residual = x
+
+ out1 = self.bn1(x)
+ out1 = F.relu(out1, True)
+ out1 = self.conv1(out1)
+
+ out2 = self.bn2(out1)
+ out2 = F.relu(out2, True)
+ out2 = self.conv2(out2)
+
+ out3 = self.bn3(out2)
+ out3 = F.relu(out3, True)
+ out3 = self.conv3(out3)
+
+ out3 = torch.cat((out1, out2, out3), 1)
+
+ if self.downsample is not None:
+ residual = self.downsample(residual)
+
+ out3 += residual
+
+ return out3
+
+
+class HourGlass(nn.Module):
+ def __init__(self, num_modules, depth, num_features):
+ super(HourGlass, self).__init__()
+ self.num_modules = num_modules
+ self.depth = depth
+ self.features = num_features
+ self.dropout = nn.Dropout(0.5)
+
+ self._generate_network(self.depth)
+
+ def _generate_network(self, level):
+ self.add_module('b1_' + str(level), ConvBlock(256, 256))
+
+ self.add_module('b2_' + str(level), ConvBlock(256, 256))
+
+ if level > 1:
+ self._generate_network(level - 1)
+ else:
+ self.add_module('b2_plus_' + str(level), ConvBlock(256, 256))
+
+ self.add_module('b3_' + str(level), ConvBlock(256, 256))
+
+ def _forward(self, level, inp):
+ # Upper branch
+ up1 = inp
+ up1 = self._modules['b1_' + str(level)](up1)
+ up1 = self.dropout(up1)
+ # Lower branch
+ low1 = F.max_pool2d(inp, 2, stride=2)
+ low1 = self._modules['b2_' + str(level)](low1)
+
+ if level > 1:
+ low2 = self._forward(level - 1, low1)
+ else:
+ low2 = low1
+ low2 = self._modules['b2_plus_' + str(level)](low2)
+
+ low3 = low2
+ low3 = self._modules['b3_' + str(level)](low3)
+ up1size = up1.size()
+ rescale_size = (up1size[2], up1size[3])
+ up2 = F.upsample(low3, size=rescale_size, mode='bilinear')
+
+ return up1 + up2
+
+ def forward(self, x):
+ return self._forward(self.depth, x)
+
+
+class FAN_use(nn.Module):
+ def __init__(self):
+ super(FAN_use, self).__init__()
+ self.num_modules = 1
+
+ # Base part
+ self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3)
+ self.bn1 = nn.BatchNorm2d(64)
+ self.conv2 = ConvBlock(64, 128)
+ self.conv3 = ConvBlock(128, 128)
+ self.conv4 = ConvBlock(128, 256)
+
+ # Stacking part
+ hg_module = 0
+ self.add_module('m' + str(hg_module), HourGlass(1, 4, 256))
+ self.add_module('top_m_' + str(hg_module), ConvBlock(256, 256))
+ self.add_module('conv_last' + str(hg_module), nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0))
+ self.add_module('l' + str(hg_module), nn.Conv2d(256, 68, kernel_size=1, stride=1, padding=0))
+ self.add_module('bn_end' + str(hg_module), nn.BatchNorm2d(256))
+
+ if hg_module < self.num_modules - 1:
+ self.add_module('bl' + str(hg_module), nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0))
+ self.add_module('al' + str(hg_module), nn.Conv2d(68, 256, kernel_size=1, stride=1, padding=0))
+
+ self.avgpool = nn.MaxPool2d((2, 2), 2)
+ self.conv6 = nn.Conv2d(68, 1, 3, 2, 1)
+ self.fc = nn.Linear(28 * 28, 512)
+ self.bn5 = nn.BatchNorm2d(68)
+ self.relu = nn.ReLU(True)
+
+ def forward(self, x, return_featmap=False):
+ x = F.relu(self.bn1(self.conv1(x)), True) # 112
+ x = F.max_pool2d(self.conv2(x), 2) # 56 # [B, 128, 112, 112]
+ x = self.conv3(x)
+ x = self.conv4(x) # [B, 256, 56, 56]
+
+ previous = x
+
+ i = 0
+ hg = self._modules['m' + str(i)](previous)
+
+ ll = hg
+ ll = self._modules['top_m_' + str(i)](ll)
+
+ ll = self._modules['bn_end' + str(i)](self._modules['conv_last' + str(i)](ll)) # [B, 256, 56, 56]
+ if return_featmap:
+ return ll
+ tmp_out = self._modules['l' + str(i)](F.relu(ll))
+
+ net = self.relu(self.bn5(tmp_out)) # [B, 68, 56, 56]
+ net = self.conv6(net) # 28 # [B, 1, 28, 28]
+ net = net.view(-1, net.shape[-2] * net.shape[-1])
+ net = self.relu(net)
+ net = self.fc(net)
+ return net
+
+from .FAN_temporal_feature_extractor import TemporalTransformer3DModel
+from einops import rearrange
+
+
+class FAN_SA(nn.Module):
+ def __init__(self):
+ super(FAN_SA, self).__init__()
+ self.num_modules = 1
+
+ # Base part
+ self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3)
+ self.bn1 = nn.BatchNorm2d(64)
+ self.conv2 = ConvBlock(64, 128)
+ self.conv3 = ConvBlock(128, 128)
+ self.conv4 = ConvBlock(128, 256)
+
+ # Stacking part
+ hg_module = 0
+ self.add_module('m' + str(hg_module), HourGlass(1, 4, 256))
+ self.add_module('top_m_' + str(hg_module), ConvBlock(256, 256))
+ self.add_module(
+ 'conv_last' + str(hg_module),
+ nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0),
+ )
+ self.add_module(
+ 'l' + str(hg_module), nn.Conv2d(256, 68, kernel_size=1, stride=1, padding=0)
+ )
+ self.add_module('bn_end' + str(hg_module), nn.BatchNorm2d(256))
+
+ if hg_module < self.num_modules - 1:
+ self.add_module(
+ 'bl' + str(hg_module),
+ nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0),
+ )
+ self.add_module(
+ 'al' + str(hg_module),
+ nn.Conv2d(68, 256, kernel_size=1, stride=1, padding=0),
+ )
+
+ self.avgpool = nn.MaxPool2d((2, 2), 2)
+ self.conv6 = nn.Conv2d(68, 1, 3, 2, 1)
+ self.fc = nn.Linear(28 * 28, 512)
+ # self.conv6 = nn.Conv2d(68, 2, 3, 2, 1)
+ # self.fc = nn.Linear(28 * 28 * 2, 1024)
+ self.bn5 = nn.BatchNorm2d(68)
+ self.relu = nn.ReLU(True)
+
+ # Add by zxc
+ self.att_1 = TemporalTransformer3DModel(
+ in_channels=128,
+ sample_size=112,
+ patch_size=4,
+ attention_block_types=("Spatial_Self",),
+ zero_initialize=True,
+ )
+ self.att_2 = TemporalTransformer3DModel(
+ in_channels=256,
+ sample_size=56,
+ patch_size=2,
+ attention_block_types=("Spatial_Self",),
+ zero_initialize=True,
+ )
+ self.att_3 = TemporalTransformer3DModel(
+ in_channels=256,
+ sample_size=56,
+ patch_size=2,
+ attention_block_types=("Spatial_Self",),
+ zero_initialize=True,
+ )
+
+ @property
+ # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors
+ def attn_processors(self) -> Dict[str, AttentionProcessor]:
+ r"""
+ Returns:
+ `dict` of attention processors: A dictionary containing all attention processors used in the model with
+ indexed by its weight name.
+ """
+ # set recursively
+ processors = {}
+
+ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]):
+ if hasattr(module, "get_processor"):
+ processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True)
+
+ for sub_name, child in module.named_children():
+ fn_recursive_add_processors(f"{name}.{sub_name}", child, processors)
+
+ return processors
+
+ for name, module in self.named_children():
+ fn_recursive_add_processors(name, module, processors)
+
+ return processors
+
+ def set_attn_processor(
+ self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]
+ ):
+ r"""
+ Sets the attention processor to use to compute attention.
+
+ Parameters:
+ processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):
+ The instantiated processor class or a dictionary of processor classes that will be set as the processor
+ for **all** `Attention` layers.
+
+ If `processor` is a dict, the key needs to define the path to the corresponding cross attention
+ processor. This is strongly recommended when setting trainable attention processors.
+
+ """
+ count = len(self.attn_processors.keys())
+
+ if isinstance(processor, dict) and len(processor) != count:
+ raise ValueError(
+ f"A dict of processors was passed, but the number of processors {len(processor)} does not match the"
+ f" number of attention layers: {count}. Please make sure to pass {count} processor classes."
+ )
+
+ def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor):
+ if hasattr(module, "set_processor"):
+ if not isinstance(processor, dict):
+ module.set_processor(processor)
+ else:
+ module.set_processor(processor.pop(f"{name}.processor"))
+
+ for sub_name, child in module.named_children():
+ fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor)
+
+ for name, module in self.named_children():
+ fn_recursive_attn_processor(name, module, processor)
+
+ def forward(self, x):
+ x = F.relu(self.bn1(self.conv1(x)), True) # 112
+ x = self.conv2(x) # [B, 128, 112, 112]
+
+ # Temp Self-Att: [B*h*w, T, 128*p*p] [B*28*28, T, 1024]
+ x = rearrange(x, "(b f) c h w -> b c f h w", f=1)
+ x = self.att_1(x, skip=True)[:, :, 0]
+
+ x = F.max_pool2d(x, 2) # 56
+ x = self.conv3(x)
+ x = self.conv4(x) # [B, 256, 56, 56]
+
+ # Temp Self-Att: [B*h*w, T, 256*p*p] [B*28*28, T, 1024]
+ x = rearrange(x, "(b f) c h w -> b c f h w", f=1)
+ x = self.att_2(x, skip=True)[:, :, 0]
+
+ previous = x
+
+ i = 0
+ hg = self._modules['m' + str(i)](previous)
+
+ ll = hg
+ ll = self._modules['top_m_' + str(i)](ll)
+
+ ll = self._modules['bn_end' + str(i)](
+ self._modules['conv_last' + str(i)](ll)
+ ) # [B, 256, 56, 56]
+ # Temp Cross-Att: [B*28*28, 1, 1024]*[B*28*28, T, 1024]
+ ll = rearrange(ll, "(b f) c h w -> b c f h w", f=1)
+ ll = self.att_3(ll, skip=True)[:, :, 0] # "b c 1 h w -> b c h w"
+ # print('att3', torch.abs(ll).mean().item())
+
+ tmp_out = self._modules['l' + str(i)](F.relu(ll))
+
+ net = self.relu(self.bn5(tmp_out)) # [B, 68, 56, 56]
+ net = self.conv6(net) # 28 # [B, 1, 28, 28]
+ net = net.view(-1, net.shape[-2] * net.shape[-1] * net.shape[1])
+ net = self.relu(net)
+ net = self.fc(net)
+ return net
diff --git a/src/models/motion_encoder/FAN_temporal_feature_extractor.py b/src/models/motion_encoder/FAN_temporal_feature_extractor.py
new file mode 100644
index 0000000..01676dd
--- /dev/null
+++ b/src/models/motion_encoder/FAN_temporal_feature_extractor.py
@@ -0,0 +1,231 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from diffusers.models.embeddings import PatchEmbed
+
+from einops import rearrange
+from src.models.motion_module import TemporalTransformerBlock, zero_module, random_module
+
+def conv3x3(in_planes, out_planes, strd=1, padding=1, bias=False):
+ "3x3 convolution with padding"
+ return nn.Conv2d(in_planes, out_planes, kernel_size=3,
+ stride=strd, padding=padding, bias=bias)
+
+
+class ConvBlock(nn.Module):
+ def __init__(self, in_planes, out_planes):
+ super(ConvBlock, self).__init__()
+ self.bn1 = nn.BatchNorm2d(in_planes)
+ self.conv1 = conv3x3(in_planes, int(out_planes / 2))
+ self.bn2 = nn.BatchNorm2d(int(out_planes / 2))
+ self.conv2 = conv3x3(int(out_planes / 2), int(out_planes / 4))
+ self.bn3 = nn.BatchNorm2d(int(out_planes / 4))
+ self.conv3 = conv3x3(int(out_planes / 4), int(out_planes / 4))
+
+ if in_planes != out_planes:
+ self.downsample = nn.Sequential(
+ nn.BatchNorm2d(in_planes),
+ nn.ReLU(True),
+ nn.Conv2d(in_planes, out_planes,
+ kernel_size=1, stride=1, bias=False),
+ )
+ else:
+ self.downsample = None
+
+ def forward(self, x):
+ residual = x
+
+ out1 = self.bn1(x)
+ out1 = F.relu(out1, True)
+ out1 = self.conv1(out1)
+
+ out2 = self.bn2(out1)
+ out2 = F.relu(out2, True)
+ out2 = self.conv2(out2)
+
+ out3 = self.bn3(out2)
+ out3 = F.relu(out3, True)
+ out3 = self.conv3(out3)
+
+ out3 = torch.cat((out1, out2, out3), 1)
+
+ if self.downsample is not None:
+ residual = self.downsample(residual)
+
+ out3 += residual
+
+ return out3
+
+
+class HourGlass(nn.Module):
+ def __init__(self, num_modules, depth, num_features):
+ super(HourGlass, self).__init__()
+ self.num_modules = num_modules
+ self.depth = depth
+ self.features = num_features
+ self.dropout = nn.Dropout(0.5)
+
+ self._generate_network(self.depth)
+
+ def _generate_network(self, level):
+ self.add_module('b1_' + str(level), ConvBlock(256, 256))
+
+ self.add_module('b2_' + str(level), ConvBlock(256, 256))
+
+ if level > 1:
+ self._generate_network(level - 1)
+ else:
+ self.add_module('b2_plus_' + str(level), ConvBlock(256, 256))
+
+ self.add_module('b3_' + str(level), ConvBlock(256, 256))
+
+ def _forward(self, level, inp):
+ # Upper branch
+ up1 = inp
+ up1 = self._modules['b1_' + str(level)](up1)
+ up1 = self.dropout(up1)
+ # Lower branch
+ low1 = F.max_pool2d(inp, 2, stride=2)
+ low1 = self._modules['b2_' + str(level)](low1)
+
+ if level > 1:
+ low2 = self._forward(level - 1, low1)
+ else:
+ low2 = low1
+ low2 = self._modules['b2_plus_' + str(level)](low2)
+
+ low3 = low2
+ low3 = self._modules['b3_' + str(level)](low3)
+ up1size = up1.size()
+ rescale_size = (up1size[2], up1size[3])
+ up2 = F.upsample(low3, size=rescale_size, mode='bilinear')
+
+ return up1 + up2
+
+ def forward(self, x):
+ return self._forward(self.depth, x)
+
+class TemporalTransformer3DModel(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ num_attention_heads=8,
+ attention_head_dim=128,
+ num_layers=1,
+ attention_block_types=("Temporal_Self",),
+ sample_size=56,
+ patch_size=2,
+ dropout=0.0,
+ norm_num_groups=32,
+ cross_attention_dim=0,
+ activation_fn="geglu",
+ attention_bias=False,
+ upcast_attention=False,
+ cross_frame_attention_mode=None,
+ temporal_position_encoding=True,
+ temporal_position_encoding_max_len=24,
+ zero_initialize=True
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ inner_dim = num_attention_heads * attention_head_dim
+
+ self.norm = torch.nn.GroupNorm(
+ num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True
+ )
+ # self.proj_in = nn.Linear(in_channels, inner_dim) # patchEmbed中有
+ if cross_attention_dim == 0 and any([block_name.endswith("_Cross") for block_name in attention_block_types]):
+ cross_attention_dim = inner_dim
+ self.cross_frame_attention_mode = cross_frame_attention_mode
+ if cross_frame_attention_mode is not None:
+ assert cross_frame_attention_mode in ['Spatial', 'Temporal'], cross_attention_dim
+ assert any([block_name.endswith("_Cross") for block_name in attention_block_types])
+ self.transformer_blocks = nn.ModuleList(
+ [
+ TemporalTransformerBlock(
+ dim=inner_dim,
+ num_attention_heads=num_attention_heads,
+ attention_head_dim=attention_head_dim,
+ attention_block_types=attention_block_types,
+ dropout=dropout,
+ norm_num_groups=norm_num_groups,
+ cross_attention_dim=cross_attention_dim,
+ activation_fn=activation_fn,
+ attention_bias=attention_bias,
+ upcast_attention=upcast_attention,
+ cross_frame_attention_mode=None,
+ temporal_position_encoding=temporal_position_encoding,
+ temporal_position_encoding_max_len=temporal_position_encoding_max_len,
+ )
+ for _ in range(num_layers)
+ ]
+ )
+ self.proj_out = nn.Linear(inner_dim // (patch_size**2), in_channels)
+
+ self.patch_size = patch_size
+ self.sample_size = sample_size
+ self.pos_embed = PatchEmbed(
+ height=sample_size,
+ width=sample_size,
+ patch_size=patch_size,
+ in_channels=in_channels,
+ embed_dim=inner_dim,
+ interpolation_scale=1,
+ )
+ # 在转onnx模型时,需要设置为False
+ self.pos_embed.pos_embed.requires_grad = False
+ self.proj_out = zero_module(self.proj_out) if zero_initialize else random_module(self.proj_out)
+
+ def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, skip=True):
+ assert hidden_states.dim() == 5, f"Expected hidden_states to have ndim=5, but got ndim={hidden_states.dim()}."
+ batch, _, video_length = hidden_states.shape[:3]
+ residual = hidden_states
+ hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w")
+
+ _, _, height, width = hidden_states.shape
+
+ hidden_states = self.norm(hidden_states) # 需要先norm,再加posEmb,否则Emb偏执太大了
+ grid_h, grid_w = height // self.patch_size, width // self.patch_size,
+
+ hidden_states = self.pos_embed(hidden_states) # [(bf), l, c] # 偏执太大了
+ # print(
+ # round(torch.abs(hidden_states).mean().item(), 6),
+ # round(torch.abs(norm_hidden_states).mean().item(), 6),
+ # )
+ if self.cross_frame_attention_mode is not None:
+ assert encoder_hidden_states is None
+ encoder_hidden_states = rearrange(hidden_states, "(b f) d c -> b f d c", b=batch)
+
+ hidden_states = encoder_hidden_states[:, -1] # [b, d, c]
+ residual = residual[:, :, -1:]
+ video_length = 1
+ if self.cross_frame_attention_mode == 'Temporal': # 用所有帧来condition最后一帧(当前帧)
+ encoder_hidden_states = rearrange(encoder_hidden_states, "b f d c -> (b d) f c")
+ elif self.cross_frame_attention_mode == 'Spatial':
+ encoder_hidden_states = rearrange(encoder_hidden_states, "b f d c -> b (f d) c")
+
+ # Transformer Blocks
+ for block in self.transformer_blocks:
+ hidden_states = block(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ video_length=video_length,
+ att_flag=False
+ )
+
+ hidden_states = hidden_states.reshape(
+ shape=(batch * video_length, grid_h, grid_w, self.patch_size, self.patch_size, -1)
+ )
+ hidden_states = self.proj_out(hidden_states)
+ hidden_states = torch.einsum("nhwpqc->nchpwq", hidden_states)
+ hidden_states = hidden_states.reshape(shape=(batch * video_length, self.in_channels, height, width))
+
+ hidden_states = rearrange(hidden_states, "(b f) c h w -> b c f h w", b=batch)
+ # print(
+ # 'out',
+ # round(torch.abs(hidden_states).mean().item(), 6),
+ # round(torch.abs(residual).mean().item(), 6),
+ # )
+ output = (hidden_states + residual) if skip else hidden_states
+
+ return output
diff --git a/src/models/motion_encoder/encoder.py b/src/models/motion_encoder/encoder.py
new file mode 100644
index 0000000..55efbcd
--- /dev/null
+++ b/src/models/motion_encoder/encoder.py
@@ -0,0 +1,41 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+from .FAN_feature_extractor import FAN_SA
+from einops import rearrange
+from diffusers.models.embeddings import get_1d_sincos_pos_embed_from_grid
+from diffusers.models.modeling_utils import ModelMixin
+
+def zero_module(module):
+ # Zero out the parameters of a module and return it.
+ assert isinstance(module, nn.Conv2d) or isinstance(module, nn.Linear), type(module)
+ for p in module.parameters():
+ p.detach().zero_()
+ return module
+
+class MotEncoder(ModelMixin):
+ def __init__(self, out_ch=16):
+ super().__init__()
+ self.model = FAN_SA()
+ self.out_drop = None #nn.Dropout(p=0.4)
+ self.out_ch = out_ch
+ expr_dim = 512
+ extra_pos_embed = get_1d_sincos_pos_embed_from_grid(out_ch, np.arange(expr_dim//out_ch))
+ self.register_buffer("pe", torch.from_numpy(extra_pos_embed).float().unsqueeze(0))
+ self.final_proj = nn.Linear(expr_dim, expr_dim)
+ self.out_bn = None
+
+ def change_out_dim(self, out_ch):
+ self.out_proj = nn.Linear(self.out_ch, out_ch)
+
+ def set_attn_processor(self, processor):
+ self.model.set_attn_processor(processor)
+
+ def forward(self, x):
+ x = x.to(self.dtype)
+ latent = self.model(rearrange(x, "b c f h w -> (b f) c h w"))
+ latent = self.final_proj(latent)
+ latent = rearrange(latent, "b (l c) -> b l c", c=self.out_ch) + self.pe
+ latent = rearrange(latent, "(b f) l c -> b f l c", f=x.shape[2])
+ return latent
\ No newline at end of file
diff --git a/src/models/motion_module.py b/src/models/motion_module.py
new file mode 100644
index 0000000..f3f0ba7
--- /dev/null
+++ b/src/models/motion_module.py
@@ -0,0 +1,472 @@
+import math
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+import torch
+from diffusers.models.attention import FeedForward
+from diffusers.models.attention_processor import Attention, AttnProcessor
+from diffusers.utils import BaseOutput
+from diffusers.utils.import_utils import is_xformers_available
+from einops import rearrange, repeat
+from torch import nn
+
+
+def zero_module(module):
+ # Zero out the parameters of a module and return it.
+ assert isinstance(module, nn.Conv2d) or isinstance(module, nn.Linear), type(module)
+ for p in module.parameters():
+ p.detach().zero_()
+ return module
+
+def random_module(m):
+ assert isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear), type(m)
+ # Initialize weights with He initialization and zero out the biases
+ n = (m.kernel_size[0] * m.kernel_size[1] * m.in_channels) if isinstance(m, nn.Conv2d) else m.in_features
+ nn.init.normal_(m.weight, mean=0.0, std=math.sqrt(2. / n))
+ if m.bias is not None:
+ nn.init.zeros_(m.bias)
+ return m
+
+
+@dataclass
+class TemporalTransformer3DModelOutput(BaseOutput):
+ sample: torch.FloatTensor
+
+
+if is_xformers_available():
+ import xformers
+ import xformers.ops
+else:
+ xformers = None
+
+
+def get_motion_module(in_channels, motion_module_type: str, motion_module_kwargs: dict):
+ if motion_module_type == "Vanilla":
+ return VanillaTemporalModule(
+ in_channels=in_channels,
+ **motion_module_kwargs,
+ )
+ elif motion_module_type == "RefImage_Vanilla":
+ return VanillaTemporalModule(
+ in_channels=in_channels,
+ skip_ref_image=True,
+ **motion_module_kwargs,
+ )
+ elif motion_module_type == "RefImageCond_Vanilla":
+ return VanillaTemporalModule(
+ in_channels=in_channels,
+ cond_ref_image=True,
+ **motion_module_kwargs,
+ )
+ else:
+ raise ValueError
+
+
+class VanillaTemporalModule(nn.Module):
+
+ def __init__(
+ self,
+ in_channels,
+ num_attention_heads=8,
+ num_transformer_block=2,
+ attention_block_types=("Temporal_Self", "Temporal_Self"),
+ cross_attention_dim=768,
+ cross_frame_attention_mode=None,
+ temporal_position_encoding=False,
+ temporal_position_encoding_max_len=24,
+ temporal_attention_dim_div=1,
+ zero_initialize=True,
+ skip_ref_image=False,
+ cond_ref_image=False,
+ ):
+ super().__init__()
+ self.skip_ref_image = skip_ref_image
+ self.cond_ref_image = cond_ref_image
+
+ self.temporal_transformer = TemporalTransformer3DModel(
+ in_channels=in_channels,
+ num_attention_heads=num_attention_heads,
+ attention_head_dim=in_channels
+ // num_attention_heads
+ // temporal_attention_dim_div,
+ num_layers=num_transformer_block,
+ attention_block_types=attention_block_types,
+ cross_attention_dim=cross_attention_dim,
+ cross_frame_attention_mode=cross_frame_attention_mode,
+ temporal_position_encoding=temporal_position_encoding,
+ temporal_position_encoding_max_len=temporal_position_encoding_max_len,
+ )
+
+ if zero_initialize:
+ self.temporal_transformer.proj_out = zero_module(
+ self.temporal_transformer.proj_out
+ )
+
+ def set_use_cross_frame_attention(self, value):
+ self.skip_ref_image = value
+
+ def forward(
+ self,
+ input_tensor,
+ temb,
+ encoder_hidden_states,
+ attention_mask=None,
+ anchor_frame_idx=None,
+ debug=False
+ ):
+ hidden_states = input_tensor
+ if self.skip_ref_image:
+ # if input_tensor.shape[2] > 1:
+ hidden_states, ref_hidden_states = input_tensor[:, :, :-1], input_tensor[:, :, -1:]
+
+ hidden_states = self.temporal_transformer(
+ hidden_states, encoder_hidden_states, attention_mask, debug=debug
+ )
+
+ output = hidden_states
+ if self.skip_ref_image:
+ # if input_tensor.shape[2] > 1:
+ output = torch.cat([output, ref_hidden_states], dim=2)
+ elif self.cond_ref_image:
+ output = torch.cat([output[:, :, :-1], input_tensor[:, :, -1:]], dim=2)
+ return output
+
+
+class TemporalTransformer3DModel(nn.Module):
+ def __init__(
+ self,
+ in_channels,
+ num_attention_heads,
+ attention_head_dim,
+ num_layers,
+ attention_block_types=(
+ "Temporal_Self",
+ "Temporal_Self",
+ ),
+ dropout=0.0,
+ norm_num_groups=32,
+ cross_attention_dim=768,
+ activation_fn="geglu",
+ attention_bias=False,
+ upcast_attention=False,
+ cross_frame_attention_mode=None,
+ temporal_position_encoding=False,
+ temporal_position_encoding_max_len=24,
+ ):
+ super().__init__()
+
+ inner_dim = num_attention_heads * attention_head_dim
+
+ self.norm = torch.nn.GroupNorm(
+ num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True
+ )
+ self.proj_in = nn.Linear(in_channels, inner_dim)
+
+ self.transformer_blocks = nn.ModuleList(
+ [
+ TemporalTransformerBlock(
+ dim=inner_dim,
+ num_attention_heads=num_attention_heads,
+ attention_head_dim=attention_head_dim,
+ attention_block_types=attention_block_types,
+ dropout=dropout,
+ norm_num_groups=norm_num_groups,
+ cross_attention_dim=cross_attention_dim,
+ activation_fn=activation_fn,
+ attention_bias=attention_bias,
+ upcast_attention=upcast_attention,
+ cross_frame_attention_mode=cross_frame_attention_mode,
+ temporal_position_encoding=temporal_position_encoding,
+ temporal_position_encoding_max_len=temporal_position_encoding_max_len,
+ )
+ for d in range(num_layers)
+ ]
+ )
+ self.proj_out = nn.Linear(inner_dim, in_channels)
+
+ def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, debug=False):
+ assert (
+ hidden_states.dim() == 5
+ ), f"Expected hidden_states to have ndim=5, but got ndim={hidden_states.dim()}."
+ video_length = hidden_states.shape[2]
+ hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w")
+
+ if encoder_hidden_states is not None and encoder_hidden_states.ndim == 4:
+ assert encoder_hidden_states.shape[1] == video_length, (video_length, encoder_hidden_states.shape)
+ encoder_hidden_states = rearrange(encoder_hidden_states, "b d n c -> (b d) n c",)
+
+ batch, channel, height, weight = hidden_states.shape
+ residual = hidden_states
+
+ hidden_states = self.norm(hidden_states)
+ inner_dim = hidden_states.shape[1]
+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(
+ batch, height * weight, inner_dim
+ )
+ hidden_states = self.proj_in(hidden_states)
+
+ # Transformer Blocks
+ for block in self.transformer_blocks:
+ hidden_states = block(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ video_length=video_length,
+ )
+
+ # output
+ hidden_states = self.proj_out(hidden_states)
+ hidden_states = (
+ hidden_states.reshape(batch, height, weight, inner_dim)
+ .permute(0, 3, 1, 2)
+ .contiguous()
+ )
+ if False:
+ print(
+ 'TemporalModule',
+ hidden_states.shape,
+ # round(torch.abs(residual).mean().item(), 6),
+ # round(torch.abs(residual).max().item(), 6),
+ # round(torch.abs(hidden_states).mean().item(), 6),
+ # round(torch.abs(hidden_states).max().item(), 6),
+ )
+ # hidden_states *= 0
+ output = hidden_states + residual
+ output = rearrange(output, "(b f) c h w -> b c f h w", f=video_length)
+
+ return output
+
+
+class TemporalTransformerBlock(nn.Module):
+
+ def __init__(
+ self,
+ dim,
+ num_attention_heads,
+ attention_head_dim,
+ attention_block_types=(
+ "Temporal_Self",
+ "Temporal_Self",
+ ),
+ dropout=0.0,
+ norm_num_groups=32,
+ cross_attention_dim=768,
+ activation_fn="geglu",
+ attention_bias=False,
+ upcast_attention=False,
+ cross_frame_attention_mode=None,
+ temporal_position_encoding=False,
+ temporal_position_encoding_max_len=24,
+ proj_out_dim=None,
+ ):
+ super().__init__()
+
+ attention_blocks = []
+ norms = []
+
+ for block_name in attention_block_types:
+ attention_blocks.append(
+ VersatileAttention(
+ attention_mode=block_name.split("_")[0],
+ cross_attention_dim=cross_attention_dim
+ if block_name.endswith("_Cross")
+ else None,
+ query_dim=dim,
+ heads=num_attention_heads,
+ dim_head=attention_head_dim,
+ dropout=dropout,
+ bias=attention_bias,
+ upcast_attention=upcast_attention,
+ cross_frame_attention_mode=cross_frame_attention_mode,
+ temporal_position_encoding=temporal_position_encoding,
+ temporal_position_encoding_max_len=temporal_position_encoding_max_len,
+ )
+ )
+ norms.append(nn.LayerNorm(dim))
+
+ self.attention_blocks = nn.ModuleList(attention_blocks)
+ self.norms = nn.ModuleList(norms)
+
+ self.ff = FeedForward(dim, dropout=dropout, activation_fn=activation_fn)
+ self.ff_norm = nn.LayerNorm(dim)
+
+ self.proj_out = nn.Linear(dim, proj_out_dim) if proj_out_dim is not None else None
+
+ def forward(
+ self,
+ hidden_states,
+ encoder_hidden_states=None,
+ attention_mask=None,
+ video_length=None,
+ att_flag=False
+ ):
+ for attention_block, norm in zip(self.attention_blocks, self.norms):
+ norm_hidden_states = norm(hidden_states)
+ if att_flag:
+ print(
+ 'block',
+ round(torch.abs(hidden_states).mean().item(), 6),
+ round(torch.abs(norm_hidden_states).mean().item(), 6),
+ )
+ hidden_states = (
+ attention_block(
+ norm_hidden_states,
+ encoder_hidden_states=encoder_hidden_states
+ if attention_block.is_cross_attention
+ else None,
+ video_length=video_length,
+ att_flag=att_flag
+ )
+ + hidden_states
+ )
+
+ hidden_states = self.ff(self.ff_norm(hidden_states)) + hidden_states
+
+ output = hidden_states if self.proj_out is None else self.proj_out(hidden_states)
+ return output
+
+
+class PositionalEncoding(nn.Module):
+ def __init__(self, d_model, dropout=0.0, max_len=24):
+ super().__init__()
+ self.dropout = nn.Dropout(p=dropout)
+ position = torch.arange(max_len).unsqueeze(1)
+ div_term = torch.exp(
+ torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model)
+ )
+ pe = torch.zeros(1, max_len, d_model)
+ pe[0, :, 0::2] = torch.sin(position * div_term)
+ pe[0, :, 1::2] = torch.cos(position * div_term)
+ self.register_buffer("pe", pe)
+
+ def forward(self, x):
+ x = x + self.pe[:, : x.size(1)]
+ return self.dropout(x)
+
+
+class VersatileAttention(Attention):
+ def __init__(
+ self,
+ attention_mode=None,
+ cross_frame_attention_mode=None,
+ temporal_position_encoding=False,
+ temporal_position_encoding_max_len=24,
+ *args,
+ **kwargs,
+ ):
+ super().__init__(*args, **kwargs)
+ assert attention_mode in ["Temporal", "Spatial"], attention_mode
+
+ self.attention_mode = attention_mode
+ self.is_cross_attention = kwargs["cross_attention_dim"] is not None
+
+ self.pos_encoder = (
+ PositionalEncoding(
+ kwargs["query_dim"],
+ dropout=0.0,
+ max_len=temporal_position_encoding_max_len,
+ )
+ if (temporal_position_encoding and attention_mode == "Temporal")
+ else None
+ )
+
+ def extra_repr(self):
+ return f"(Module Info) Attention_Mode: {self.attention_mode}, Is_Cross_Attention: {self.is_cross_attention}"
+
+ def set_use_memory_efficient_attention_xformers(
+ self,
+ use_memory_efficient_attention_xformers: bool,
+ attention_op: Optional[Callable] = None,
+ ):
+ if use_memory_efficient_attention_xformers:
+ if not is_xformers_available():
+ raise ModuleNotFoundError(
+ (
+ "Refer to https://github.com/facebookresearch/xformers for more information on how to install"
+ " xformers"
+ ),
+ name="xformers",
+ )
+ elif not torch.cuda.is_available():
+ raise ValueError(
+ "torch.cuda.is_available() should be True but is False. xformers' memory efficient attention is"
+ " only available for GPU "
+ )
+ else:
+ try:
+ # Make sure we can run the memory efficient attention
+ _ = xformers.ops.memory_efficient_attention(
+ torch.randn((1, 2, 40), device="cuda"),
+ torch.randn((1, 2, 40), device="cuda"),
+ torch.randn((1, 2, 40), device="cuda"),
+ )
+ except Exception as e:
+ raise e
+
+ # XFormersAttnProcessor corrupts video generation and work with Pytorch 1.13.
+ # Pytorch 2.0.1 AttnProcessor works the same as XFormersAttnProcessor in Pytorch 1.13.
+ # You don't need XFormersAttnProcessor here.
+ # processor = XFormersAttnProcessor(
+ # attention_op=attention_op,
+ # )
+ processor = AttnProcessor()
+ else:
+ processor = AttnProcessor()
+
+ self.set_processor(processor)
+
+ def forward(
+ self,
+ hidden_states,
+ encoder_hidden_states=None,
+ attention_mask=None,
+ video_length=None,
+ bank=None,
+ att_flag=False,
+ **cross_attention_kwargs,
+ ):
+ if self.attention_mode == "Temporal":
+ d = hidden_states.shape[1] # d means HxW
+ hidden_states = rearrange(hidden_states, "(b f) d c -> (b d) f c", f=video_length)
+
+ if encoder_hidden_states is not None:
+ if not encoder_hidden_states.shape[0] == hidden_states.shape[0]:
+ encoder_hidden_states = repeat(encoder_hidden_states, "b n c -> (b d) n c", d=d)
+
+ if bank is not None and self.attention_mode == "Temporal" and not self.is_cross_attention:
+ # motion_frames作为之前的帧,引入motion module进行condition
+ modify_norm_hidden_states = torch.cat(bank + [hidden_states], dim=1)
+
+ if self.pos_encoder is not None:
+ modify_norm_hidden_states = self.pos_encoder(modify_norm_hidden_states)
+
+ hidden_states = self.processor(
+ self,
+ hidden_states,
+ encoder_hidden_states=modify_norm_hidden_states,
+ attention_mask=attention_mask,
+ **cross_attention_kwargs,
+ ) # 改为cross-att
+
+ else:
+ if self.pos_encoder is not None:
+ hidden_states = self.pos_encoder(hidden_states)
+ inp = hidden_states
+ hidden_states = self.processor(
+ self,
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ attention_mask=attention_mask,
+ **cross_attention_kwargs,
+ )
+ if att_flag:
+ print(
+ 'ver_att',
+ round(torch.abs(inp).mean().item(), 6),
+ round(torch.abs(encoder_hidden_states).mean().item(), 6),
+ round(torch.abs(hidden_states).mean().item(), 6),
+ )
+
+ if self.attention_mode == "Temporal":
+ hidden_states = rearrange(hidden_states, "(b d) f c -> (b f) d c", d=d)
+
+ return hidden_states
\ No newline at end of file
diff --git a/src/models/mutual_self_attention.py b/src/models/mutual_self_attention.py
new file mode 100644
index 0000000..81781a7
--- /dev/null
+++ b/src/models/mutual_self_attention.py
@@ -0,0 +1,465 @@
+from typing import Any, Dict, Optional
+
+import torch
+from einops import rearrange
+
+from src.models.attention import TemporalBasicTransformerBlock
+
+from .attention import BasicTransformerBlock
+
+
+def torch_dfs(model: torch.nn.Module):
+ result = [model]
+ for child in model.children():
+ result += torch_dfs(child)
+ return result
+
+
+class ReferenceAttentionControl:
+ def __init__(
+ self,
+ unet,
+ mode="write",
+ do_classifier_free_guidance=False,
+ attention_auto_machine_weight=float("inf"),
+ gn_auto_machine_weight=1.0,
+ style_fidelity=1.0,
+ reference_attn=True,
+ reference_adain=False,
+ fusion_blocks="midup",
+ batch_size=1,
+ cache_kv=False,
+ ) -> None:
+ # 10. Modify self attention and group norm
+ self.unet = unet
+ assert mode in ["read", "write"]
+ assert fusion_blocks in ["midup", "full"]
+ self.reference_attn = reference_attn
+ self.reference_adain = reference_adain
+ self.fusion_blocks = fusion_blocks
+ self.cache_kv = cache_kv
+ self.register_reference_hooks(
+ mode,
+ do_classifier_free_guidance,
+ attention_auto_machine_weight,
+ gn_auto_machine_weight,
+ style_fidelity,
+ reference_attn,
+ reference_adain,
+ fusion_blocks,
+ batch_size=batch_size,
+ cache_kv=self.cache_kv,
+ )
+
+ def register_reference_hooks(
+ self,
+ mode,
+ do_classifier_free_guidance,
+ attention_auto_machine_weight,
+ gn_auto_machine_weight,
+ style_fidelity,
+ reference_attn,
+ reference_adain,
+ dtype=torch.float16,
+ batch_size=1,
+ num_images_per_prompt=1,
+ device=torch.device("cpu"),
+ fusion_blocks="midup",
+ cache_kv=False,
+ ):
+ MODE = mode
+ do_classifier_free_guidance = do_classifier_free_guidance
+ attention_auto_machine_weight = attention_auto_machine_weight
+ gn_auto_machine_weight = gn_auto_machine_weight
+ style_fidelity = style_fidelity
+ reference_attn = reference_attn
+ reference_adain = reference_adain
+ fusion_blocks = fusion_blocks
+ num_images_per_prompt = num_images_per_prompt
+ cache_kv = cache_kv
+ dtype = dtype
+ if do_classifier_free_guidance:
+ uc_mask = (
+ torch.Tensor(
+ [1] * batch_size * num_images_per_prompt * 16
+ + [0] * batch_size * num_images_per_prompt * 16
+ )
+ .to(device)
+ .bool()
+ )
+ else:
+ uc_mask = (
+ torch.Tensor([0] * batch_size * num_images_per_prompt * 2)
+ .to(device)
+ .bool()
+ )
+
+ def hacked_basic_transformer_inner_forward(
+ self,
+ hidden_states: torch.FloatTensor,
+ attention_mask: Optional[torch.FloatTensor] = None,
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
+ timestep: Optional[torch.LongTensor] = None,
+ cross_attention_kwargs: Dict[str, Any] = None,
+ class_labels: Optional[torch.LongTensor] = None,
+ video_length=None,
+ ):
+ if self.use_ada_layer_norm: # False
+ norm_hidden_states = self.norm1(hidden_states, timestep)
+ elif self.use_ada_layer_norm_zero:
+ (
+ norm_hidden_states,
+ gate_msa,
+ shift_mlp,
+ scale_mlp,
+ gate_mlp,
+ ) = self.norm1(
+ hidden_states,
+ timestep,
+ class_labels,
+ hidden_dtype=hidden_states.dtype,
+ )
+ else:
+ norm_hidden_states = self.norm1(hidden_states)
+
+ # 1. Self-Attention
+ # self.only_cross_attention = False
+ cross_attention_kwargs = (
+ cross_attention_kwargs if cross_attention_kwargs is not None else {}
+ )
+ if self.only_cross_attention:
+ attn_output = self.attn1(
+ norm_hidden_states,
+ encoder_hidden_states=encoder_hidden_states
+ if self.only_cross_attention
+ else None,
+ attention_mask=attention_mask,
+ **cross_attention_kwargs,
+ )
+ else:
+ if MODE == "write":
+ self.bank.append(norm_hidden_states.clone())
+ attn_output = self.attn1(
+ norm_hidden_states,
+ encoder_hidden_states=encoder_hidden_states
+ if self.only_cross_attention
+ else None,
+ attention_mask=attention_mask,
+ **cross_attention_kwargs,
+ )
+ if MODE == "read":
+ kv_cache = norm_hidden_states.clone()
+ kv_cache = rearrange(
+ kv_cache, "(b t) l c -> b t l c", t=video_length
+ )
+ self.kv_cache = kv_cache[:,:2,:,:]
+
+ bank_fea = [
+ rearrange(
+ d.unsqueeze(1).repeat(1, video_length, 1, 1),
+ "b t l c -> (b t) l c",
+ )
+ for d in self.bank
+ ]
+
+ if self.kv_bank is not None and cache_kv:
+ ahead_fea = self.kv_bank.unsqueeze(1).repeat(1, video_length, 1, 1, 1)
+ ahead_fea = rearrange(ahead_fea, "b t n l c -> (b t) (n l) c")
+ bank_fea.append(ahead_fea)
+
+ modify_norm_hidden_states = torch.cat(
+ [norm_hidden_states] + bank_fea, dim=1
+ )
+
+ hidden_states_uc = (
+ self.attn1(
+ norm_hidden_states,
+ encoder_hidden_states=modify_norm_hidden_states,
+ attention_mask=attention_mask,
+ )
+ # self.attn1(
+ # modify_norm_hidden_states,
+ # encoder_hidden_states=modify_norm_hidden_states,
+ # attention_mask=attention_mask,
+ # )[:, : hidden_states.shape[-2], :]
+ + hidden_states
+ )
+ if do_classifier_free_guidance:
+ hidden_states_c = hidden_states_uc.clone()
+ _uc_mask = uc_mask.clone()
+ if hidden_states.shape[0] != _uc_mask.shape[0]:
+ _uc_mask = (
+ torch.Tensor(
+ [1] * (hidden_states.shape[0] // 2)
+ + [0] * (hidden_states.shape[0] // 2)
+ )
+ .to(device)
+ .bool()
+ )
+ if self.kv_bank is not None:
+ # if False:
+ modify_norm_hidden_states = torch.cat(
+ [norm_hidden_states, ahead_fea], dim=1
+ )
+ else:
+ modify_norm_hidden_states = norm_hidden_states
+ hidden_states_c[_uc_mask] = (
+ self.attn1(
+ norm_hidden_states[_uc_mask],
+ encoder_hidden_states=modify_norm_hidden_states[_uc_mask],
+ attention_mask=attention_mask,
+ )
+ + hidden_states[_uc_mask]
+ )
+ hidden_states = hidden_states_c.clone()
+ else:
+ hidden_states = hidden_states_uc
+
+ # self.bank.clear()
+ if self.attn2 is not None:
+ # Cross-Attention
+ norm_hidden_states = (
+ self.norm2(hidden_states, timestep)
+ if self.use_ada_layer_norm
+ else self.norm2(hidden_states)
+ )
+ hidden_states = (
+ self.attn2(
+ norm_hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ attention_mask=attention_mask,
+ )
+ + hidden_states
+ )
+
+ # Feed-forward
+ hidden_states = self.ff(self.norm3(hidden_states)) + hidden_states
+
+ # Temporal-Attention
+ if self.unet_use_temporal_attention:
+ d = hidden_states.shape[1]
+ hidden_states = rearrange(
+ hidden_states, "(b f) d c -> (b d) f c", f=video_length
+ )
+ norm_hidden_states = (
+ self.norm_temp(hidden_states, timestep)
+ if self.use_ada_layer_norm
+ else self.norm_temp(hidden_states)
+ )
+ hidden_states = (
+ self.attn_temp(norm_hidden_states) + hidden_states
+ )
+ hidden_states = rearrange(
+ hidden_states, "(b d) f c -> (b f) d c", d=d
+ )
+
+ return hidden_states
+
+ if self.use_ada_layer_norm_zero:
+ attn_output = gate_msa.unsqueeze(1) * attn_output
+ hidden_states = attn_output + hidden_states
+
+ if self.attn2 is not None:
+ norm_hidden_states = (
+ self.norm2(hidden_states, timestep)
+ if self.use_ada_layer_norm
+ else self.norm2(hidden_states)
+ )
+
+ # 2. Cross-Attention
+ attn_output = self.attn2(
+ norm_hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ attention_mask=encoder_attention_mask,
+ **cross_attention_kwargs,
+ )
+ hidden_states = attn_output + hidden_states
+
+ # 3. Feed-forward
+ norm_hidden_states = self.norm3(hidden_states)
+
+ if self.use_ada_layer_norm_zero:
+ norm_hidden_states = (
+ norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
+ )
+
+ ff_output = self.ff(norm_hidden_states)
+
+ if self.use_ada_layer_norm_zero:
+ ff_output = gate_mlp.unsqueeze(1) * ff_output
+
+ hidden_states = ff_output + hidden_states
+
+ return hidden_states
+
+ if self.reference_attn:
+ if self.fusion_blocks == "midup":
+ attn_modules = [
+ module
+ for module in (
+ torch_dfs(self.unet.mid_block) + torch_dfs(self.unet.up_blocks)
+ )
+ if isinstance(module, BasicTransformerBlock)
+ or isinstance(module, TemporalBasicTransformerBlock)
+ ]
+ elif self.fusion_blocks == "full":
+ attn_modules = [
+ module
+ for module in torch_dfs(self.unet)
+ if isinstance(module, BasicTransformerBlock)
+ or isinstance(module, TemporalBasicTransformerBlock)
+ ]
+ attn_modules = sorted(
+ attn_modules, key=lambda x: -x.norm1.normalized_shape[0]
+ )
+
+ for i, module in enumerate(attn_modules):
+ module._original_inner_forward = module.forward
+ if isinstance(module, BasicTransformerBlock):
+ module.forward = hacked_basic_transformer_inner_forward.__get__(
+ module, BasicTransformerBlock
+ )
+ if isinstance(module, TemporalBasicTransformerBlock):
+ module.forward = hacked_basic_transformer_inner_forward.__get__(
+ module, TemporalBasicTransformerBlock
+ )
+
+ module.bank = []
+ if(self.cache_kv):
+ module.kv_bank = None
+ module.kv_cache = None
+ module.attn_weight = float(i) / float(len(attn_modules))
+
+ def update(self, writer, dtype=torch.float16, drop_ratio=0.):
+ if self.reference_attn:
+ if self.fusion_blocks == "midup":
+ reader_attn_modules = [
+ module
+ for module in (
+ torch_dfs(self.unet.mid_block) + torch_dfs(self.unet.up_blocks)
+ )
+ if isinstance(module, TemporalBasicTransformerBlock)
+ ]
+ writer_attn_modules = [
+ module
+ for module in (
+ torch_dfs(writer.unet.mid_block)
+ + torch_dfs(writer.unet.up_blocks)
+ )
+ if isinstance(module, BasicTransformerBlock)
+ ]
+ elif self.fusion_blocks == "full":
+ reader_attn_modules = [
+ module
+ for module in torch_dfs(self.unet)
+ if isinstance(module, TemporalBasicTransformerBlock)
+ ]
+ writer_attn_modules = [
+ module
+ for module in torch_dfs(writer.unet)
+ if isinstance(module, BasicTransformerBlock)
+ ]
+
+ reader_attn_modules = sorted(
+ reader_attn_modules, key=lambda x: -x.norm1.normalized_shape[0]
+ )
+ writer_attn_modules = sorted(
+ writer_attn_modules, key=lambda x: -x.norm1.normalized_shape[0]
+ )
+ for r, w in zip(reader_attn_modules, writer_attn_modules):
+ if drop_ratio > 0:
+ r.bank = []
+ for v in w.bank:
+ N, L, D = v.shape # batch, length, dim
+ len_keep = int(L * (1 - drop_ratio))
+
+ noise = torch.rand(N, L) # noise in [0, 1]
+ ids_shuffle = torch.argsort(noise, dim=1) # ascend: small is keep, large is remove
+ ids_keep = ids_shuffle[:, :len_keep].to(v.device)
+ visible_tokens = torch.gather(v.clone(), dim=1, index=ids_keep.unsqueeze(-1).repeat(1, 1, D))
+ r.bank.append((visible_tokens).to(dtype))
+
+ else:
+ r.bank = [v.clone().to(dtype) for v in w.bank]
+ # w.bank.clear()
+
+ def update_hkf(self, writer, dtype=torch.float16, drop_ratio=0.):
+ if self.reference_attn:
+ if self.fusion_blocks == "midup":
+ reader_attn_modules = [
+ module
+ for module in (
+ torch_dfs(self.unet.mid_block) + torch_dfs(self.unet.up_blocks)
+ )
+ if isinstance(module, TemporalBasicTransformerBlock)
+ ]
+ writer_attn_modules = [
+ module
+ for module in (
+ torch_dfs(writer.unet.mid_block)
+ + torch_dfs(writer.unet.up_blocks)
+ )
+ if isinstance(module, BasicTransformerBlock)
+ ]
+ elif self.fusion_blocks == "full":
+ reader_attn_modules = [
+ module
+ for module in torch_dfs(self.unet)
+ if isinstance(module, TemporalBasicTransformerBlock)
+ ]
+ writer_attn_modules = [
+ module
+ for module in torch_dfs(writer.unet)
+ if isinstance(module, BasicTransformerBlock)
+ ]
+
+ reader_attn_modules = sorted(
+ reader_attn_modules, key=lambda x: -x.norm1.normalized_shape[0]
+ )
+ writer_attn_modules = sorted(
+ writer_attn_modules, key=lambda x: -x.norm1.normalized_shape[0]
+ )
+ for r, w in zip(reader_attn_modules, writer_attn_modules):
+ if r.kv_bank is None:
+ r.kv_bank = torch.cat([v.clone().unsqueeze(1).to(dtype) for v in w.bank], dim=1)
+ else:
+ r.kv_bank = torch.cat([r.kv_bank] + [v.clone().unsqueeze(1).to(dtype) for v in w.bank], dim=1).to(dtype)
+
+ def output(self, dtype=torch.float16):
+ res = {}
+ for i in range(3):
+ for j in range(2):
+ res[f"d{i}{j}"] = torch.cat([v.clone().to(dtype=dtype, device=self.unet.device) for v in self.unet.down_blocks[i].attentions[j].transformer_blocks[0].bank], dim=1)
+ res["m"] = torch.cat([v.clone().to(dtype=dtype, device=self.unet.device) for v in self.unet.mid_block.attentions[0].transformer_blocks[0].bank], dim=1)
+ for i in range(1, 4):
+ for j in range(3):
+ res[f"u{i}{j}"] = torch.cat([v.clone().to(dtype=dtype, device=self.unet.device) for v in self.unet.up_blocks[i].attentions[j].transformer_blocks[0].bank], dim=1)
+ return res
+
+ def clear(self):
+ if self.reference_attn:
+ if self.fusion_blocks == "midup":
+ reader_attn_modules = [
+ module
+ for module in (
+ torch_dfs(self.unet.mid_block) + torch_dfs(self.unet.up_blocks)
+ )
+ if isinstance(module, BasicTransformerBlock)
+ or isinstance(module, TemporalBasicTransformerBlock)
+ ]
+ elif self.fusion_blocks == "full":
+ reader_attn_modules = [
+ module
+ for module in torch_dfs(self.unet)
+ if isinstance(module, BasicTransformerBlock)
+ or isinstance(module, TemporalBasicTransformerBlock)
+ ]
+ reader_attn_modules = sorted(
+ reader_attn_modules, key=lambda x: -x.norm1.normalized_shape[0]
+ )
+ for r in reader_attn_modules:
+ r.bank.clear()
+ if self.cache_kv:
+ r.kv_bank=None
+ r.kv_cache=None
\ No newline at end of file
diff --git a/src/models/pose_guider.py b/src/models/pose_guider.py
new file mode 100644
index 0000000..f079429
--- /dev/null
+++ b/src/models/pose_guider.py
@@ -0,0 +1,57 @@
+from typing import Tuple
+
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.nn.init as init
+from diffusers.models.modeling_utils import ModelMixin
+
+from src.models.motion_module import zero_module
+from src.models.resnet import InflatedConv3d
+
+
+class PoseGuider(ModelMixin):
+ def __init__(
+ self,
+ conditioning_embedding_channels: int = 320,
+ conditioning_channels: int = 3,
+ block_out_channels: Tuple[int] = (16, 32, 96, 256),
+ ):
+ super().__init__()
+ self.conv_in = InflatedConv3d(
+ conditioning_channels, block_out_channels[0], kernel_size=3, padding=1
+ )
+
+ self.blocks = nn.ModuleList([])
+
+ for i in range(len(block_out_channels) - 1):
+ channel_in = block_out_channels[i]
+ channel_out = block_out_channels[i + 1]
+ self.blocks.append(
+ InflatedConv3d(channel_in, channel_in, kernel_size=3, padding=1)
+ )
+ self.blocks.append(
+ InflatedConv3d(
+ channel_in, channel_out, kernel_size=3, padding=1, stride=2
+ )
+ )
+
+ self.conv_out_modify = zero_module(
+ InflatedConv3d(
+ block_out_channels[-1],
+ conditioning_embedding_channels,
+ kernel_size=3,
+ padding=1,
+ )
+ )
+
+ def forward(self, conditioning):
+ embedding = self.conv_in(conditioning)
+ embedding = F.silu(embedding)
+
+ for block in self.blocks:
+ embedding = block(embedding)
+ embedding = F.silu(embedding)
+
+ embedding = self.conv_out_modify(embedding)
+
+ return embedding
diff --git a/src/models/resnet.py b/src/models/resnet.py
new file mode 100644
index 0000000..ea73144
--- /dev/null
+++ b/src/models/resnet.py
@@ -0,0 +1,255 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+from typing import Dict, Optional
+
+
+class InflatedConv3d(nn.Conv2d):
+ def forward(self, x):
+ video_length = x.shape[2]
+
+ x = rearrange(x, "b c f h w -> (b f) c h w")
+ x = super().forward(x)
+ x = rearrange(x, "(b f) c h w -> b c f h w", f=video_length)
+
+ return x
+
+
+class InflatedGroupNorm(nn.GroupNorm):
+ def forward(self, x):
+ video_length = x.shape[2]
+
+ x = rearrange(x, "b c f h w -> (b f) c h w")
+ x = super().forward(x)
+ x = rearrange(x, "(b f) c h w -> b c f h w", f=video_length)
+
+ return x
+
+
+class Upsample3D(nn.Module):
+ def __init__(
+ self,
+ channels,
+ use_conv=False,
+ use_conv_transpose=False,
+ out_channels=None,
+ name="conv",
+ ):
+ super().__init__()
+ self.channels = channels
+ self.out_channels = out_channels or channels
+ self.use_conv = use_conv
+ self.use_conv_transpose = use_conv_transpose
+ self.name = name
+
+ conv = None
+ if use_conv_transpose:
+ raise NotImplementedError
+ elif use_conv:
+ self.conv = InflatedConv3d(self.channels, self.out_channels, 3, padding=1)
+
+ def forward(self, hidden_states, output_size=None):
+ assert hidden_states.shape[1] == self.channels
+
+ if self.use_conv_transpose:
+ raise NotImplementedError
+
+ # Cast to float32 to as 'upsample_nearest2d_out_frame' op does not support bfloat16
+ dtype = hidden_states.dtype
+ if dtype == torch.bfloat16:
+ hidden_states = hidden_states.to(torch.float32)
+
+ # upsample_nearest_nhwc fails with large batch sizes. see https://github.com/huggingface/diffusers/issues/984
+ if hidden_states.shape[0] >= 64:
+ hidden_states = hidden_states.contiguous()
+
+ # if `output_size` is passed we force the interpolation output
+ # size and do not make use of `scale_factor=2`
+ if output_size is None:
+ hidden_states = F.interpolate(
+ hidden_states, scale_factor=[1.0, 2.0, 2.0], mode="nearest"
+ )
+ else:
+ hidden_states = F.interpolate(
+ hidden_states, size=output_size, mode="nearest"
+ )
+
+ # If the input is bfloat16, we cast back to bfloat16
+ if dtype == torch.bfloat16:
+ hidden_states = hidden_states.to(dtype)
+
+ # if self.use_conv:
+ # if self.name == "conv":
+ # hidden_states = self.conv(hidden_states)
+ # else:
+ # hidden_states = self.Conv2d_0(hidden_states)
+ hidden_states = self.conv(hidden_states)
+
+ return hidden_states
+
+
+class Downsample3D(nn.Module):
+ def __init__(
+ self, channels, use_conv=False, out_channels=None, padding=1, name="conv"
+ ):
+ super().__init__()
+ self.channels = channels
+ self.out_channels = out_channels or channels
+ self.use_conv = use_conv
+ self.padding = padding
+ stride = 2
+ self.name = name
+
+ if use_conv:
+ self.conv = InflatedConv3d(
+ self.channels, self.out_channels, 3, stride=stride, padding=padding
+ )
+ else:
+ raise NotImplementedError
+
+ def forward(self, hidden_states):
+ assert hidden_states.shape[1] == self.channels
+ if self.use_conv and self.padding == 0:
+ raise NotImplementedError
+
+ assert hidden_states.shape[1] == self.channels
+ hidden_states = self.conv(hidden_states)
+
+ return hidden_states
+
+
+class ResnetBlock3D(nn.Module):
+ def __init__(
+ self,
+ *,
+ in_channels,
+ out_channels=None,
+ conv_shortcut=False,
+ dropout=0.0,
+ temb_channels=512,
+ groups=32,
+ groups_out=None,
+ pre_norm=True,
+ eps=1e-6,
+ non_linearity="swish",
+ time_embedding_norm="default",
+ output_scale_factor=1.0,
+ use_in_shortcut=None,
+ use_inflated_groupnorm=None,
+ ):
+ super().__init__()
+ self.pre_norm = pre_norm
+ self.pre_norm = True
+ self.in_channels = in_channels
+ out_channels = in_channels if out_channels is None else out_channels
+ self.out_channels = out_channels
+ self.use_conv_shortcut = conv_shortcut
+ self.time_embedding_norm = time_embedding_norm
+ self.output_scale_factor = output_scale_factor
+
+ if groups_out is None:
+ groups_out = groups
+
+ assert use_inflated_groupnorm != None
+ if use_inflated_groupnorm:
+ self.norm1 = InflatedGroupNorm(
+ num_groups=groups, num_channels=in_channels, eps=eps, affine=True
+ )
+ else:
+ self.norm1 = torch.nn.GroupNorm(
+ num_groups=groups, num_channels=in_channels, eps=eps, affine=True
+ )
+
+ self.conv1 = InflatedConv3d(
+ in_channels, out_channels, kernel_size=3, stride=1, padding=1
+ )
+
+ if temb_channels is not None:
+ if self.time_embedding_norm == "default":
+ time_emb_proj_out_channels = out_channels
+ elif self.time_embedding_norm == "scale_shift":
+ time_emb_proj_out_channels = out_channels * 2
+ else:
+ raise ValueError(
+ f"unknown time_embedding_norm : {self.time_embedding_norm} "
+ )
+
+ self.time_emb_proj = torch.nn.Linear(
+ temb_channels, time_emb_proj_out_channels
+ )
+ else:
+ self.time_emb_proj = None
+
+ if use_inflated_groupnorm:
+ self.norm2 = InflatedGroupNorm(
+ num_groups=groups_out, num_channels=out_channels, eps=eps, affine=True
+ )
+ else:
+ self.norm2 = torch.nn.GroupNorm(
+ num_groups=groups_out, num_channels=out_channels, eps=eps, affine=True
+ )
+ self.dropout = torch.nn.Dropout(dropout)
+ self.conv2 = InflatedConv3d(
+ out_channels, out_channels, kernel_size=3, stride=1, padding=1
+ )
+
+ if non_linearity == "swish":
+ self.nonlinearity = lambda x: F.silu(x)
+ elif non_linearity == "mish":
+ self.nonlinearity = Mish()
+ elif non_linearity == "silu":
+ self.nonlinearity = nn.SiLU()
+
+ self.use_in_shortcut = (
+ self.in_channels != self.out_channels
+ if use_in_shortcut is None
+ else use_in_shortcut
+ )
+
+ self.conv_shortcut = None
+ if self.use_in_shortcut:
+ self.conv_shortcut = InflatedConv3d(
+ in_channels, out_channels, kernel_size=1, stride=1, padding=0
+ )
+
+ def forward(self, input_tensor, temb):
+ hidden_states = input_tensor
+
+ hidden_states = self.norm1(hidden_states)
+ hidden_states = self.nonlinearity(hidden_states)
+
+ hidden_states = self.conv1(hidden_states)
+
+ if temb is not None:
+ video_length = hidden_states.shape[2]
+ temb = self.time_emb_proj(self.nonlinearity(temb))
+ temb = rearrange(temb, "(b f) c -> b c f", f=video_length)[:, :, :, None, None]
+
+ # if temb is not None:
+ # temb = self.time_emb_proj(self.nonlinearity(temb))[:, :, None, None, None]
+
+ if temb is not None and self.time_embedding_norm == "default":
+ hidden_states = hidden_states + temb
+
+ hidden_states = self.norm2(hidden_states)
+
+ if temb is not None and self.time_embedding_norm == "scale_shift":
+ scale, shift = torch.chunk(temb, 2, dim=1)
+ hidden_states = hidden_states * (1 + scale) + shift
+
+ hidden_states = self.nonlinearity(hidden_states)
+
+ hidden_states = self.dropout(hidden_states)
+ hidden_states = self.conv2(hidden_states)
+
+ if self.conv_shortcut is not None:
+ input_tensor = self.conv_shortcut(input_tensor)
+
+ output_tensor = (input_tensor + hidden_states) / self.output_scale_factor
+
+ return output_tensor
+
+class Mish(torch.nn.Module):
+ def forward(self, hidden_states):
+ return hidden_states * torch.tanh(torch.nn.functional.softplus(hidden_states))
diff --git a/src/models/transformer_2d.py b/src/models/transformer_2d.py
new file mode 100644
index 0000000..4c0995b
--- /dev/null
+++ b/src/models/transformer_2d.py
@@ -0,0 +1,395 @@
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+import torch
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.models.embeddings import PixArtAlphaTextProjection as CaptionProjection
+from diffusers.models.lora import LoRACompatibleConv, LoRACompatibleLinear
+from diffusers.models.modeling_utils import ModelMixin
+from diffusers.models.normalization import AdaLayerNormSingle
+from diffusers.utils import USE_PEFT_BACKEND, BaseOutput, deprecate, is_torch_version
+from torch import nn
+
+from .attention import BasicTransformerBlock
+
+
+@dataclass
+class Transformer2DModelOutput(BaseOutput):
+ """
+ The output of [`Transformer2DModel`].
+
+ Args:
+ sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `(batch size, num_vector_embeds - 1, num_latent_pixels)` if [`Transformer2DModel`] is discrete):
+ The hidden states output conditioned on the `encoder_hidden_states` input. If discrete, returns probability
+ distributions for the unnoised latent pixels.
+ """
+
+ sample: torch.FloatTensor
+ ref_feature: torch.FloatTensor
+
+
+class Transformer2DModel(ModelMixin, ConfigMixin):
+ """
+ A 2D Transformer model for image-like data.
+
+ Parameters:
+ num_attention_heads (`int`, *optional*, defaults to 16): The number of heads to use for multi-head attention.
+ attention_head_dim (`int`, *optional*, defaults to 88): The number of channels in each head.
+ in_channels (`int`, *optional*):
+ The number of channels in the input and output (specify if the input is **continuous**).
+ num_layers (`int`, *optional*, defaults to 1): The number of layers of Transformer blocks to use.
+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
+ cross_attention_dim (`int`, *optional*): The number of `encoder_hidden_states` dimensions to use.
+ sample_size (`int`, *optional*): The width of the latent images (specify if the input is **discrete**).
+ This is fixed during training since it is used to learn a number of position embeddings.
+ num_vector_embeds (`int`, *optional*):
+ The number of classes of the vector embeddings of the latent pixels (specify if the input is **discrete**).
+ Includes the class for the masked latent pixel.
+ activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to use in feed-forward.
+ num_embeds_ada_norm ( `int`, *optional*):
+ The number of diffusion steps used during training. Pass if at least one of the norm_layers is
+ `AdaLayerNorm`. This is fixed during training since it is used to learn a number of embeddings that are
+ added to the hidden states.
+
+ During inference, you can denoise for up to but not more steps than `num_embeds_ada_norm`.
+ attention_bias (`bool`, *optional*):
+ Configure if the `TransformerBlocks` attention should contain a bias parameter.
+ """
+
+ _supports_gradient_checkpointing = True
+
+ @register_to_config
+ def __init__(
+ self,
+ num_attention_heads: int = 16,
+ attention_head_dim: int = 88,
+ in_channels: Optional[int] = None,
+ out_channels: Optional[int] = None,
+ num_layers: int = 1,
+ dropout: float = 0.0,
+ norm_num_groups: int = 32,
+ cross_attention_dim: Optional[int] = None,
+ attention_bias: bool = False,
+ sample_size: Optional[int] = None,
+ num_vector_embeds: Optional[int] = None,
+ patch_size: Optional[int] = None,
+ activation_fn: str = "geglu",
+ num_embeds_ada_norm: Optional[int] = None,
+ use_linear_projection: bool = False,
+ only_cross_attention: bool = False,
+ double_self_attention: bool = False,
+ upcast_attention: bool = False,
+ norm_type: str = "layer_norm",
+ norm_elementwise_affine: bool = True,
+ norm_eps: float = 1e-5,
+ attention_type: str = "default",
+ caption_channels: int = None,
+ ):
+ super().__init__()
+ self.use_linear_projection = use_linear_projection
+ self.num_attention_heads = num_attention_heads
+ self.attention_head_dim = attention_head_dim
+ inner_dim = num_attention_heads * attention_head_dim
+
+ conv_cls = nn.Conv2d if USE_PEFT_BACKEND else LoRACompatibleConv
+ linear_cls = nn.Linear if USE_PEFT_BACKEND else LoRACompatibleLinear
+
+ # 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
+ # Define whether input is continuous or discrete depending on configuration
+ self.is_input_continuous = (in_channels is not None) and (patch_size is None)
+ self.is_input_vectorized = num_vector_embeds is not None
+ self.is_input_patches = in_channels is not None and patch_size is not None
+
+ if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
+ deprecation_message = (
+ f"The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or"
+ " incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config."
+ " Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect"
+ " results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it"
+ " would be very nice if you could open a Pull request for the `transformer/config.json` file"
+ )
+ deprecate(
+ "norm_type!=num_embeds_ada_norm",
+ "1.0.0",
+ deprecation_message,
+ standard_warn=False,
+ )
+ norm_type = "ada_norm"
+
+ if self.is_input_continuous and self.is_input_vectorized:
+ raise ValueError(
+ f"Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make"
+ " sure that either `in_channels` or `num_vector_embeds` is None."
+ )
+ elif self.is_input_vectorized and self.is_input_patches:
+ raise ValueError(
+ f"Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make"
+ " sure that either `num_vector_embeds` or `num_patches` is None."
+ )
+ elif (
+ not self.is_input_continuous
+ and not self.is_input_vectorized
+ and not self.is_input_patches
+ ):
+ raise ValueError(
+ f"Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:"
+ f" {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None."
+ )
+
+ # 2. Define input layers
+ self.in_channels = in_channels
+
+ self.norm = torch.nn.GroupNorm(
+ num_groups=norm_num_groups,
+ num_channels=in_channels,
+ eps=1e-6,
+ affine=True,
+ )
+ if use_linear_projection:
+ self.proj_in = linear_cls(in_channels, inner_dim)
+ else:
+ self.proj_in = conv_cls(
+ in_channels, inner_dim, kernel_size=1, stride=1, padding=0
+ )
+
+ # 3. Define transformers blocks
+ self.transformer_blocks = nn.ModuleList(
+ [
+ BasicTransformerBlock(
+ inner_dim,
+ num_attention_heads,
+ attention_head_dim,
+ dropout=dropout,
+ cross_attention_dim=cross_attention_dim,
+ activation_fn=activation_fn,
+ num_embeds_ada_norm=num_embeds_ada_norm,
+ attention_bias=attention_bias,
+ only_cross_attention=only_cross_attention,
+ double_self_attention=double_self_attention,
+ upcast_attention=upcast_attention,
+ norm_type=norm_type,
+ norm_elementwise_affine=norm_elementwise_affine,
+ norm_eps=norm_eps,
+ attention_type=attention_type,
+ )
+ for d in range(num_layers)
+ ]
+ )
+
+ # 4. Define output layers
+ self.out_channels = in_channels if out_channels is None else out_channels
+ # TODO: should use out_channels for continuous projections
+ if use_linear_projection:
+ self.proj_out = linear_cls(inner_dim, in_channels)
+ else:
+ self.proj_out = conv_cls(
+ inner_dim, in_channels, kernel_size=1, stride=1, padding=0
+ )
+
+ # 5. PixArt-Alpha blocks.
+ self.adaln_single = None
+ self.use_additional_conditions = False
+ if norm_type == "ada_norm_single":
+ self.use_additional_conditions = self.config.sample_size == 128
+ # TODO(Sayak, PVP) clean this, for now we use sample size to determine whether to use
+ # additional conditions until we find better name
+ self.adaln_single = AdaLayerNormSingle(
+ inner_dim, use_additional_conditions=self.use_additional_conditions
+ )
+
+ self.caption_projection = None
+ if caption_channels is not None:
+ self.caption_projection = CaptionProjection(
+ in_features=caption_channels, hidden_size=inner_dim
+ )
+
+ self.gradient_checkpointing = False
+
+ def _set_gradient_checkpointing(self, module, value=False):
+ if hasattr(module, "gradient_checkpointing"):
+ module.gradient_checkpointing = value
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ encoder_hidden_states: Optional[torch.Tensor] = None,
+ timestep: Optional[torch.LongTensor] = None,
+ added_cond_kwargs: Dict[str, torch.Tensor] = None,
+ class_labels: Optional[torch.LongTensor] = None,
+ cross_attention_kwargs: Dict[str, Any] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ encoder_attention_mask: Optional[torch.Tensor] = None,
+ return_dict: bool = True,
+ ):
+ """
+ The [`Transformer2DModel`] forward method.
+
+ Args:
+ hidden_states (`torch.LongTensor` of shape `(batch size, num latent pixels)` if discrete, `torch.FloatTensor` of shape `(batch size, channel, height, width)` if continuous):
+ Input `hidden_states`.
+ encoder_hidden_states ( `torch.FloatTensor` of shape `(batch size, sequence len, embed dims)`, *optional*):
+ Conditional embeddings for cross attention layer. If not given, cross-attention defaults to
+ self-attention.
+ timestep ( `torch.LongTensor`, *optional*):
+ Used to indicate denoising step. Optional timestep to be applied as an embedding in `AdaLayerNorm`.
+ class_labels ( `torch.LongTensor` of shape `(batch size, num classes)`, *optional*):
+ Used to indicate class labels conditioning. Optional class labels to be applied as an embedding in
+ `AdaLayerZeroNorm`.
+ cross_attention_kwargs ( `Dict[str, Any]`, *optional*):
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
+ `self.processor` in
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
+ attention_mask ( `torch.Tensor`, *optional*):
+ An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask
+ is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large
+ negative values to the attention scores corresponding to "discard" tokens.
+ encoder_attention_mask ( `torch.Tensor`, *optional*):
+ Cross-attention mask applied to `encoder_hidden_states`. Two formats supported:
+
+ * Mask `(batch, sequence_length)` True = keep, False = discard.
+ * Bias `(batch, 1, sequence_length)` 0 = keep, -10000 = discard.
+
+ If `ndim == 2`: will be interpreted as a mask, then converted into a bias consistent with the format
+ above. This bias will be added to the cross-attention scores.
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether or not to return a [`~models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain
+ tuple.
+
+ Returns:
+ If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a
+ `tuple` where the first element is the sample tensor.
+ """
+ # ensure attention_mask is a bias, and give it a singleton query_tokens dimension.
+ # we may have done this conversion already, e.g. if we came here via UNet2DConditionModel#forward.
+ # we can tell by counting dims; if ndim == 2: it's a mask rather than a bias.
+ # expects mask of shape:
+ # [batch, key_tokens]
+ # adds singleton query_tokens dimension:
+ # [batch, 1, key_tokens]
+ # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
+ # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn)
+ # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
+ if attention_mask is not None and attention_mask.ndim == 2:
+ # assume that mask is expressed as:
+ # (1 = keep, 0 = discard)
+ # convert mask into a bias that can be added to attention scores:
+ # (keep = +0, discard = -10000.0)
+ attention_mask = (1 - attention_mask.to(hidden_states.dtype)) * -10000.0
+ attention_mask = attention_mask.unsqueeze(1)
+
+ # convert encoder_attention_mask to a bias the same way we do for attention_mask
+ if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
+ encoder_attention_mask = (
+ 1 - encoder_attention_mask.to(hidden_states.dtype)
+ ) * -10000.0
+ encoder_attention_mask = encoder_attention_mask.unsqueeze(1)
+
+ # Retrieve lora scale.
+ lora_scale = (
+ cross_attention_kwargs.get("scale", 1.0)
+ if cross_attention_kwargs is not None
+ else 1.0
+ )
+
+ # 1. Input
+ batch, _, height, width = hidden_states.shape
+ residual = hidden_states
+
+ hidden_states = self.norm(hidden_states)
+ if not self.use_linear_projection:
+ hidden_states = (
+ self.proj_in(hidden_states, scale=lora_scale)
+ if not USE_PEFT_BACKEND
+ else self.proj_in(hidden_states)
+ )
+ inner_dim = hidden_states.shape[1]
+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(
+ batch, height * width, inner_dim
+ )
+ else:
+ inner_dim = hidden_states.shape[1]
+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(
+ batch, height * width, inner_dim
+ )
+ hidden_states = (
+ self.proj_in(hidden_states, scale=lora_scale)
+ if not USE_PEFT_BACKEND
+ else self.proj_in(hidden_states)
+ )
+
+ # 2. Blocks
+ if self.caption_projection is not None:
+ batch_size = hidden_states.shape[0]
+ encoder_hidden_states = self.caption_projection(encoder_hidden_states)
+ encoder_hidden_states = encoder_hidden_states.view(
+ batch_size, -1, hidden_states.shape[-1]
+ )
+
+ ref_feature = hidden_states.reshape(batch, height, width, inner_dim)
+ for block in self.transformer_blocks:
+ if self.training and self.gradient_checkpointing:
+
+ def create_custom_forward(module, return_dict=None):
+ def custom_forward(*inputs):
+ if return_dict is not None:
+ return module(*inputs, return_dict=return_dict)
+ else:
+ return module(*inputs)
+
+ return custom_forward
+
+ ckpt_kwargs: Dict[str, Any] = (
+ {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {}
+ )
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(block),
+ hidden_states,
+ attention_mask,
+ encoder_hidden_states,
+ encoder_attention_mask,
+ timestep,
+ cross_attention_kwargs,
+ class_labels,
+ **ckpt_kwargs,
+ )
+ else:
+ hidden_states = block(
+ hidden_states,
+ attention_mask=attention_mask,
+ encoder_hidden_states=encoder_hidden_states,
+ encoder_attention_mask=encoder_attention_mask,
+ timestep=timestep,
+ cross_attention_kwargs=cross_attention_kwargs,
+ class_labels=class_labels,
+ )
+
+ # 3. Output
+ if self.is_input_continuous:
+ if not self.use_linear_projection:
+ hidden_states = (
+ hidden_states.reshape(batch, height, width, inner_dim)
+ .permute(0, 3, 1, 2)
+ .contiguous()
+ )
+ hidden_states = (
+ self.proj_out(hidden_states, scale=lora_scale)
+ if not USE_PEFT_BACKEND
+ else self.proj_out(hidden_states)
+ )
+ else:
+ hidden_states = (
+ self.proj_out(hidden_states, scale=lora_scale)
+ if not USE_PEFT_BACKEND
+ else self.proj_out(hidden_states)
+ )
+ hidden_states = (
+ hidden_states.reshape(batch, height, width, inner_dim)
+ .permute(0, 3, 1, 2)
+ .contiguous()
+ )
+
+ output = hidden_states + residual
+ if not return_dict:
+ return (output, ref_feature)
+
+ return Transformer2DModelOutput(sample=output, ref_feature=ref_feature)
diff --git a/src/models/transformer_3d.py b/src/models/transformer_3d.py
new file mode 100644
index 0000000..d10a3a4
--- /dev/null
+++ b/src/models/transformer_3d.py
@@ -0,0 +1,182 @@
+from dataclasses import dataclass
+from typing import Optional, Dict
+
+import torch
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.models import ModelMixin
+from diffusers.utils import BaseOutput
+from diffusers.utils.import_utils import is_xformers_available
+from einops import rearrange, repeat
+from torch import nn
+
+from .attention import TemporalBasicTransformerBlock
+
+
+@dataclass
+class Transformer3DModelOutput(BaseOutput):
+ sample: torch.FloatTensor
+
+
+if is_xformers_available():
+ import xformers
+ import xformers.ops
+else:
+ xformers = None
+
+
+class Transformer3DModel(ModelMixin, ConfigMixin):
+ _supports_gradient_checkpointing = True
+
+ @register_to_config
+ def __init__(
+ self,
+ num_attention_heads: int = 16,
+ attention_head_dim: int = 88,
+ in_channels: Optional[int] = None,
+ num_layers: int = 1,
+ dropout: float = 0.0,
+ norm_num_groups: int = 32,
+ cross_attention_dim: Optional[int] = None,
+ attention_bias: bool = False,
+ activation_fn: str = "geglu",
+ num_embeds_ada_norm: Optional[int] = None,
+ use_linear_projection: bool = False,
+ only_cross_attention: bool = False,
+ upcast_attention: bool = False,
+ unet_use_cross_frame_attention=None,
+ unet_use_temporal_attention=None,
+ ):
+ super().__init__()
+ self.use_linear_projection = use_linear_projection
+ self.num_attention_heads = num_attention_heads
+ self.attention_head_dim = attention_head_dim
+ inner_dim = num_attention_heads * attention_head_dim
+
+ # Define input layers
+ self.in_channels = in_channels
+
+ self.norm = torch.nn.GroupNorm(
+ num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True
+ )
+ if use_linear_projection:
+ self.proj_in = nn.Linear(in_channels, inner_dim)
+ else:
+ self.proj_in = nn.Conv2d(
+ in_channels, inner_dim, kernel_size=1, stride=1, padding=0
+ )
+
+ # Define transformers blocks
+ self.transformer_blocks = nn.ModuleList(
+ [
+ TemporalBasicTransformerBlock(
+ inner_dim,
+ num_attention_heads,
+ attention_head_dim,
+ dropout=dropout,
+ cross_attention_dim=cross_attention_dim,
+ activation_fn=activation_fn,
+ num_embeds_ada_norm=num_embeds_ada_norm,
+ attention_bias=attention_bias,
+ only_cross_attention=only_cross_attention,
+ upcast_attention=upcast_attention,
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
+ unet_use_temporal_attention=unet_use_temporal_attention,
+ )
+ for d in range(num_layers)
+ ]
+ )
+
+ # 4. Define output layers
+ if use_linear_projection:
+ self.proj_out = nn.Linear(in_channels, inner_dim)
+ else:
+ self.proj_out = nn.Conv2d(
+ inner_dim, in_channels, kernel_size=1, stride=1, padding=0
+ )
+
+ self.gradient_checkpointing = False
+
+ def _set_gradient_checkpointing(self, module, value=False):
+ if hasattr(module, "gradient_checkpointing"):
+ module.gradient_checkpointing = value
+
+ def forward(
+ self,
+ hidden_states,
+ encoder_hidden_states=None,
+ timestep=None,
+ return_dict: bool = True,
+ reference=None,
+ **block_kwargs,
+ ):
+ # Input
+ assert (
+ hidden_states.dim() == 5
+ ), f"Expected hidden_states to have ndim=5, but got ndim={hidden_states.dim()}."
+ video_length = hidden_states.shape[2]
+ hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w")
+ if encoder_hidden_states.shape[0] != hidden_states.shape[0]:
+ encoder_hidden_states = repeat(
+ encoder_hidden_states, "b n c -> (b f) n c", f=video_length
+ )
+
+ batch, channel, height, weight = hidden_states.shape
+ residual = hidden_states
+
+ hidden_states = self.norm(hidden_states)
+ if not self.use_linear_projection:
+ hidden_states = self.proj_in(hidden_states)
+ inner_dim = hidden_states.shape[1]
+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(
+ batch, height * weight, inner_dim
+ )
+ else:
+ inner_dim = hidden_states.shape[1]
+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(
+ batch, height * weight, inner_dim
+ )
+ hidden_states = self.proj_in(hidden_states)
+
+ # Blocks
+ for i, block in enumerate(self.transformer_blocks):
+ if reference is None:
+ hidden_states = block(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ timestep=timestep,
+ video_length=video_length,
+ **block_kwargs
+ )
+ else:
+ hidden_states = block(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ timestep=timestep,
+ video_length=video_length,
+ reference=reference,
+ **block_kwargs
+ )
+
+ # Output
+ if not self.use_linear_projection:
+ hidden_states = (
+ hidden_states.reshape(batch, height, weight, inner_dim)
+ .permute(0, 3, 1, 2)
+ .contiguous()
+ )
+ hidden_states = self.proj_out(hidden_states)
+ else:
+ hidden_states = self.proj_out(hidden_states)
+ hidden_states = (
+ hidden_states.reshape(batch, height, weight, inner_dim)
+ .permute(0, 3, 1, 2)
+ .contiguous()
+ )
+
+ output = hidden_states + residual
+
+ output = rearrange(output, "(b f) c h w -> b c f h w", f=video_length)
+ if not return_dict:
+ return (output,)
+
+ return Transformer3DModelOutput(sample=output)
diff --git a/src/models/unet_2d_blocks.py b/src/models/unet_2d_blocks.py
new file mode 100644
index 0000000..99c7ed0
--- /dev/null
+++ b/src/models/unet_2d_blocks.py
@@ -0,0 +1,1073 @@
+from typing import Any, Dict, Optional, Tuple, Union
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from diffusers.models.activations import get_activation
+from diffusers.models.attention_processor import Attention
+from diffusers.models import DualTransformer2DModel
+from diffusers.models.resnet import Downsample2D, ResnetBlock2D, Upsample2D
+from diffusers.utils import is_torch_version, logging
+from diffusers.utils.torch_utils import apply_freeu
+from torch import nn
+
+from .transformer_2d import Transformer2DModel
+
+logger = logging.get_logger(__name__) # pylint: disable=invalid-name
+
+
+def get_down_block(
+ down_block_type: str,
+ num_layers: int,
+ in_channels: int,
+ out_channels: int,
+ temb_channels: int,
+ add_downsample: bool,
+ resnet_eps: float,
+ resnet_act_fn: str,
+ transformer_layers_per_block: int = 1,
+ num_attention_heads: Optional[int] = None,
+ resnet_groups: Optional[int] = None,
+ cross_attention_dim: Optional[int] = None,
+ downsample_padding: Optional[int] = None,
+ dual_cross_attention: bool = False,
+ use_linear_projection: bool = False,
+ only_cross_attention: bool = False,
+ upcast_attention: bool = False,
+ resnet_time_scale_shift: str = "default",
+ attention_type: str = "default",
+ resnet_skip_time_act: bool = False,
+ resnet_out_scale_factor: float = 1.0,
+ cross_attention_norm: Optional[str] = None,
+ attention_head_dim: Optional[int] = None,
+ downsample_type: Optional[str] = None,
+ dropout: float = 0.0,
+):
+ # If attn head dim is not defined, we default it to the number of heads
+ if attention_head_dim is None:
+ logger.warn(
+ f"It is recommended to provide `attention_head_dim` when calling `get_down_block`. Defaulting `attention_head_dim` to {num_attention_heads}."
+ )
+ attention_head_dim = num_attention_heads
+
+ down_block_type = (
+ down_block_type[7:]
+ if down_block_type.startswith("UNetRes")
+ else down_block_type
+ )
+ if down_block_type == "DownBlock2D":
+ return DownBlock2D(
+ num_layers=num_layers,
+ in_channels=in_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ dropout=dropout,
+ add_downsample=add_downsample,
+ resnet_eps=resnet_eps,
+ resnet_act_fn=resnet_act_fn,
+ resnet_groups=resnet_groups,
+ downsample_padding=downsample_padding,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ )
+ elif down_block_type == "CrossAttnDownBlock2D":
+ if cross_attention_dim is None:
+ raise ValueError(
+ "cross_attention_dim must be specified for CrossAttnDownBlock2D"
+ )
+ return CrossAttnDownBlock2D(
+ num_layers=num_layers,
+ transformer_layers_per_block=transformer_layers_per_block,
+ in_channels=in_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ dropout=dropout,
+ add_downsample=add_downsample,
+ resnet_eps=resnet_eps,
+ resnet_act_fn=resnet_act_fn,
+ resnet_groups=resnet_groups,
+ downsample_padding=downsample_padding,
+ cross_attention_dim=cross_attention_dim,
+ num_attention_heads=num_attention_heads,
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention,
+ upcast_attention=upcast_attention,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ attention_type=attention_type,
+ )
+ raise ValueError(f"{down_block_type} does not exist.")
+
+
+def get_up_block(
+ up_block_type: str,
+ num_layers: int,
+ in_channels: int,
+ out_channels: int,
+ prev_output_channel: int,
+ temb_channels: int,
+ add_upsample: bool,
+ resnet_eps: float,
+ resnet_act_fn: str,
+ resolution_idx: Optional[int] = None,
+ transformer_layers_per_block: int = 1,
+ num_attention_heads: Optional[int] = None,
+ resnet_groups: Optional[int] = None,
+ cross_attention_dim: Optional[int] = None,
+ dual_cross_attention: bool = False,
+ use_linear_projection: bool = False,
+ only_cross_attention: bool = False,
+ upcast_attention: bool = False,
+ resnet_time_scale_shift: str = "default",
+ attention_type: str = "default",
+ resnet_skip_time_act: bool = False,
+ resnet_out_scale_factor: float = 1.0,
+ cross_attention_norm: Optional[str] = None,
+ attention_head_dim: Optional[int] = None,
+ upsample_type: Optional[str] = None,
+ dropout: float = 0.0,
+) -> nn.Module:
+ # If attn head dim is not defined, we default it to the number of heads
+ if attention_head_dim is None:
+ logger.warn(
+ f"It is recommended to provide `attention_head_dim` when calling `get_up_block`. Defaulting `attention_head_dim` to {num_attention_heads}."
+ )
+ attention_head_dim = num_attention_heads
+
+ up_block_type = (
+ up_block_type[7:] if up_block_type.startswith("UNetRes") else up_block_type
+ )
+ if up_block_type == "UpBlock2D":
+ return UpBlock2D(
+ num_layers=num_layers,
+ in_channels=in_channels,
+ out_channels=out_channels,
+ prev_output_channel=prev_output_channel,
+ temb_channels=temb_channels,
+ resolution_idx=resolution_idx,
+ dropout=dropout,
+ add_upsample=add_upsample,
+ resnet_eps=resnet_eps,
+ resnet_act_fn=resnet_act_fn,
+ resnet_groups=resnet_groups,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ )
+ elif up_block_type == "CrossAttnUpBlock2D":
+ if cross_attention_dim is None:
+ raise ValueError(
+ "cross_attention_dim must be specified for CrossAttnUpBlock2D"
+ )
+ return CrossAttnUpBlock2D(
+ num_layers=num_layers,
+ transformer_layers_per_block=transformer_layers_per_block,
+ in_channels=in_channels,
+ out_channels=out_channels,
+ prev_output_channel=prev_output_channel,
+ temb_channels=temb_channels,
+ resolution_idx=resolution_idx,
+ dropout=dropout,
+ add_upsample=add_upsample,
+ resnet_eps=resnet_eps,
+ resnet_act_fn=resnet_act_fn,
+ resnet_groups=resnet_groups,
+ cross_attention_dim=cross_attention_dim,
+ num_attention_heads=num_attention_heads,
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention,
+ upcast_attention=upcast_attention,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ attention_type=attention_type,
+ )
+
+ raise ValueError(f"{up_block_type} does not exist.")
+
+
+class AutoencoderTinyBlock(nn.Module):
+ """
+ Tiny Autoencoder block used in [`AutoencoderTiny`]. It is a mini residual module consisting of plain conv + ReLU
+ blocks.
+
+ Args:
+ in_channels (`int`): The number of input channels.
+ out_channels (`int`): The number of output channels.
+ act_fn (`str`):
+ ` The activation function to use. Supported values are `"swish"`, `"mish"`, `"gelu"`, and `"relu"`.
+
+ Returns:
+ `torch.FloatTensor`: A tensor with the same shape as the input tensor, but with the number of channels equal to
+ `out_channels`.
+ """
+
+ def __init__(self, in_channels: int, out_channels: int, act_fn: str):
+ super().__init__()
+ act_fn = get_activation(act_fn)
+ self.conv = nn.Sequential(
+ nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1),
+ act_fn,
+ nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1),
+ act_fn,
+ nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1),
+ )
+ self.skip = (
+ nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
+ if in_channels != out_channels
+ else nn.Identity()
+ )
+ self.fuse = nn.ReLU()
+
+ def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:
+ return self.fuse(self.conv(x) + self.skip(x))
+
+
+class UNetMidBlock2D(nn.Module):
+ """
+ A 2D UNet mid-block [`UNetMidBlock2D`] with multiple residual blocks and optional attention blocks.
+
+ Args:
+ in_channels (`int`): The number of input channels.
+ temb_channels (`int`): The number of temporal embedding channels.
+ dropout (`float`, *optional*, defaults to 0.0): The dropout rate.
+ num_layers (`int`, *optional*, defaults to 1): The number of residual blocks.
+ resnet_eps (`float`, *optional*, 1e-6 ): The epsilon value for the resnet blocks.
+ resnet_time_scale_shift (`str`, *optional*, defaults to `default`):
+ The type of normalization to apply to the time embeddings. This can help to improve the performance of the
+ model on tasks with long-range temporal dependencies.
+ resnet_act_fn (`str`, *optional*, defaults to `swish`): The activation function for the resnet blocks.
+ resnet_groups (`int`, *optional*, defaults to 32):
+ The number of groups to use in the group normalization layers of the resnet blocks.
+ attn_groups (`Optional[int]`, *optional*, defaults to None): The number of groups for the attention blocks.
+ resnet_pre_norm (`bool`, *optional*, defaults to `True`):
+ Whether to use pre-normalization for the resnet blocks.
+ add_attention (`bool`, *optional*, defaults to `True`): Whether to add attention blocks.
+ attention_head_dim (`int`, *optional*, defaults to 1):
+ Dimension of a single attention head. The number of attention heads is determined based on this value and
+ the number of input channels.
+ output_scale_factor (`float`, *optional*, defaults to 1.0): The output scale factor.
+
+ Returns:
+ `torch.FloatTensor`: The output of the last residual block, which is a tensor of shape `(batch_size,
+ in_channels, height, width)`.
+
+ """
+
+ def __init__(
+ self,
+ in_channels: int,
+ temb_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default", # default, spatial
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ attn_groups: Optional[int] = None,
+ resnet_pre_norm: bool = True,
+ add_attention: bool = True,
+ attention_head_dim: int = 1,
+ output_scale_factor: float = 1.0,
+ ):
+ super().__init__()
+ resnet_groups = (
+ resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)
+ )
+ self.add_attention = add_attention
+
+ if attn_groups is None:
+ attn_groups = (
+ resnet_groups if resnet_time_scale_shift == "default" else None
+ )
+
+ # there is always at least one resnet
+ resnets = [
+ ResnetBlock2D(
+ in_channels=in_channels,
+ out_channels=in_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ )
+ ]
+ attentions = []
+
+ if attention_head_dim is None:
+ logger.warn(
+ f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: {in_channels}."
+ )
+ attention_head_dim = in_channels
+
+ for _ in range(num_layers):
+ if self.add_attention:
+ attentions.append(
+ Attention(
+ in_channels,
+ heads=in_channels // attention_head_dim,
+ dim_head=attention_head_dim,
+ rescale_output_factor=output_scale_factor,
+ eps=resnet_eps,
+ norm_num_groups=attn_groups,
+ spatial_norm_dim=temb_channels
+ if resnet_time_scale_shift == "spatial"
+ else None,
+ residual_connection=True,
+ bias=True,
+ upcast_softmax=True,
+ _from_deprecated_attn_block=True,
+ )
+ )
+ else:
+ attentions.append(None)
+
+ resnets.append(
+ ResnetBlock2D(
+ in_channels=in_channels,
+ out_channels=in_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ )
+ )
+
+ self.attentions = nn.ModuleList(attentions)
+ self.resnets = nn.ModuleList(resnets)
+
+ def forward(
+ self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None
+ ) -> torch.FloatTensor:
+ hidden_states = self.resnets[0](hidden_states, temb)
+ for attn, resnet in zip(self.attentions, self.resnets[1:]):
+ if attn is not None:
+ hidden_states = attn(hidden_states, temb=temb)
+ hidden_states = resnet(hidden_states, temb)
+
+ return hidden_states
+
+
+class UNetMidBlock2DCrossAttn(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ temb_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ transformer_layers_per_block: Union[int, Tuple[int]] = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default",
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ resnet_pre_norm: bool = True,
+ num_attention_heads: int = 1,
+ output_scale_factor: float = 1.0,
+ cross_attention_dim: int = 1280,
+ dual_cross_attention: bool = False,
+ use_linear_projection: bool = False,
+ upcast_attention: bool = False,
+ attention_type: str = "default",
+ ):
+ super().__init__()
+
+ self.has_cross_attention = True
+ self.num_attention_heads = num_attention_heads
+ resnet_groups = (
+ resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)
+ )
+
+ # support for variable transformer layers per block
+ if isinstance(transformer_layers_per_block, int):
+ transformer_layers_per_block = [transformer_layers_per_block] * num_layers
+
+ # there is always at least one resnet
+ resnets = [
+ ResnetBlock2D(
+ in_channels=in_channels,
+ out_channels=in_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ )
+ ]
+ attentions = []
+
+ for i in range(num_layers):
+ if not dual_cross_attention:
+ attentions.append(
+ Transformer2DModel(
+ num_attention_heads,
+ in_channels // num_attention_heads,
+ in_channels=in_channels,
+ num_layers=transformer_layers_per_block[i],
+ cross_attention_dim=cross_attention_dim,
+ norm_num_groups=resnet_groups,
+ use_linear_projection=use_linear_projection,
+ upcast_attention=upcast_attention,
+ attention_type=attention_type,
+ )
+ )
+ else:
+ attentions.append(
+ DualTransformer2DModel(
+ num_attention_heads,
+ in_channels // num_attention_heads,
+ in_channels=in_channels,
+ num_layers=1,
+ cross_attention_dim=cross_attention_dim,
+ norm_num_groups=resnet_groups,
+ )
+ )
+ resnets.append(
+ ResnetBlock2D(
+ in_channels=in_channels,
+ out_channels=in_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ )
+ )
+
+ self.attentions = nn.ModuleList(attentions)
+ self.resnets = nn.ModuleList(resnets)
+
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ hidden_states: torch.FloatTensor,
+ temb: Optional[torch.FloatTensor] = None,
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
+ attention_mask: Optional[torch.FloatTensor] = None,
+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
+ ) -> torch.FloatTensor:
+ lora_scale = (
+ cross_attention_kwargs.get("scale", 1.0)
+ if cross_attention_kwargs is not None
+ else 1.0
+ )
+ hidden_states = self.resnets[0](hidden_states, temb, scale=lora_scale)
+ for attn, resnet in zip(self.attentions, self.resnets[1:]):
+ if self.training and self.gradient_checkpointing:
+
+ def create_custom_forward(module, return_dict=None):
+ def custom_forward(*inputs):
+ if return_dict is not None:
+ return module(*inputs, return_dict=return_dict)
+ else:
+ return module(*inputs)
+
+ return custom_forward
+
+ ckpt_kwargs: Dict[str, Any] = (
+ {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {}
+ )
+ hidden_states, ref_feature = attn(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ cross_attention_kwargs=cross_attention_kwargs,
+ attention_mask=attention_mask,
+ encoder_attention_mask=encoder_attention_mask,
+ return_dict=False,
+ )
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet),
+ hidden_states,
+ temb,
+ **ckpt_kwargs,
+ )
+ else:
+ hidden_states, ref_feature = attn(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ cross_attention_kwargs=cross_attention_kwargs,
+ attention_mask=attention_mask,
+ encoder_attention_mask=encoder_attention_mask,
+ return_dict=False,
+ )
+ hidden_states = resnet(hidden_states, temb, scale=lora_scale)
+
+ return hidden_states
+
+
+class CrossAttnDownBlock2D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ temb_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ transformer_layers_per_block: Union[int, Tuple[int]] = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default",
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ resnet_pre_norm: bool = True,
+ num_attention_heads: int = 1,
+ cross_attention_dim: int = 1280,
+ output_scale_factor: float = 1.0,
+ downsample_padding: int = 1,
+ add_downsample: bool = True,
+ dual_cross_attention: bool = False,
+ use_linear_projection: bool = False,
+ only_cross_attention: bool = False,
+ upcast_attention: bool = False,
+ attention_type: str = "default",
+ ):
+ super().__init__()
+ resnets = []
+ attentions = []
+
+ self.has_cross_attention = True
+ self.num_attention_heads = num_attention_heads
+ if isinstance(transformer_layers_per_block, int):
+ transformer_layers_per_block = [transformer_layers_per_block] * num_layers
+
+ for i in range(num_layers):
+ in_channels = in_channels if i == 0 else out_channels
+ resnets.append(
+ ResnetBlock2D(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ )
+ )
+ if not dual_cross_attention:
+ attentions.append(
+ Transformer2DModel(
+ num_attention_heads,
+ out_channels // num_attention_heads,
+ in_channels=out_channels,
+ num_layers=transformer_layers_per_block[i],
+ cross_attention_dim=cross_attention_dim,
+ norm_num_groups=resnet_groups,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention,
+ upcast_attention=upcast_attention,
+ attention_type=attention_type,
+ )
+ )
+ else:
+ attentions.append(
+ DualTransformer2DModel(
+ num_attention_heads,
+ out_channels // num_attention_heads,
+ in_channels=out_channels,
+ num_layers=1,
+ cross_attention_dim=cross_attention_dim,
+ norm_num_groups=resnet_groups,
+ )
+ )
+ self.attentions = nn.ModuleList(attentions)
+ self.resnets = nn.ModuleList(resnets)
+
+ if add_downsample:
+ self.downsamplers = nn.ModuleList(
+ [
+ Downsample2D(
+ out_channels,
+ use_conv=True,
+ out_channels=out_channels,
+ padding=downsample_padding,
+ name="op",
+ )
+ ]
+ )
+ else:
+ self.downsamplers = None
+
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ hidden_states: torch.FloatTensor,
+ temb: Optional[torch.FloatTensor] = None,
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
+ attention_mask: Optional[torch.FloatTensor] = None,
+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
+ additional_residuals: Optional[torch.FloatTensor] = None,
+ ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]:
+ output_states = ()
+
+ lora_scale = (
+ cross_attention_kwargs.get("scale", 1.0)
+ if cross_attention_kwargs is not None
+ else 1.0
+ )
+
+ blocks = list(zip(self.resnets, self.attentions))
+
+ for i, (resnet, attn) in enumerate(blocks):
+ if self.training and self.gradient_checkpointing:
+
+ def create_custom_forward(module, return_dict=None):
+ def custom_forward(*inputs):
+ if return_dict is not None:
+ return module(*inputs, return_dict=return_dict)
+ else:
+ return module(*inputs)
+
+ return custom_forward
+
+ ckpt_kwargs: Dict[str, Any] = (
+ {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {}
+ )
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet),
+ hidden_states,
+ temb,
+ **ckpt_kwargs,
+ )
+ hidden_states, ref_feature = attn(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ cross_attention_kwargs=cross_attention_kwargs,
+ attention_mask=attention_mask,
+ encoder_attention_mask=encoder_attention_mask,
+ return_dict=False,
+ )
+ else:
+ hidden_states = resnet(hidden_states, temb, scale=lora_scale)
+ hidden_states, ref_feature = attn(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ cross_attention_kwargs=cross_attention_kwargs,
+ attention_mask=attention_mask,
+ encoder_attention_mask=encoder_attention_mask,
+ return_dict=False,
+ )
+
+ # apply additional residuals to the output of the last pair of resnet and attention blocks
+ if i == len(blocks) - 1 and additional_residuals is not None:
+ hidden_states = hidden_states + additional_residuals
+
+ output_states = output_states + (hidden_states,)
+
+ if self.downsamplers is not None:
+ for downsampler in self.downsamplers:
+ hidden_states = downsampler(hidden_states, scale=lora_scale)
+
+ output_states = output_states + (hidden_states,)
+
+ return hidden_states, output_states
+
+
+class DownBlock2D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ temb_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default",
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ resnet_pre_norm: bool = True,
+ output_scale_factor: float = 1.0,
+ add_downsample: bool = True,
+ downsample_padding: int = 1,
+ ):
+ super().__init__()
+ resnets = []
+
+ for i in range(num_layers):
+ in_channels = in_channels if i == 0 else out_channels
+ resnets.append(
+ ResnetBlock2D(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ )
+ )
+
+ self.resnets = nn.ModuleList(resnets)
+
+ if add_downsample:
+ self.downsamplers = nn.ModuleList(
+ [
+ Downsample2D(
+ out_channels,
+ use_conv=True,
+ out_channels=out_channels,
+ padding=downsample_padding,
+ name="op",
+ )
+ ]
+ )
+ else:
+ self.downsamplers = None
+
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ hidden_states: torch.FloatTensor,
+ temb: Optional[torch.FloatTensor] = None,
+ scale: float = 1.0,
+ ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]:
+ output_states = ()
+
+ for resnet in self.resnets:
+ if self.training and self.gradient_checkpointing:
+
+ def create_custom_forward(module):
+ def custom_forward(*inputs):
+ return module(*inputs)
+
+ return custom_forward
+
+ if is_torch_version(">=", "1.11.0"):
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet),
+ hidden_states,
+ temb,
+ use_reentrant=False,
+ )
+ else:
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet), hidden_states, temb
+ )
+ else:
+ hidden_states = resnet(hidden_states, temb, scale=scale)
+
+ output_states = output_states + (hidden_states,)
+
+ if self.downsamplers is not None:
+ for downsampler in self.downsamplers:
+ hidden_states = downsampler(hidden_states, scale=scale)
+
+ output_states = output_states + (hidden_states,)
+
+ return hidden_states, output_states
+
+
+class CrossAttnUpBlock2D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ prev_output_channel: int,
+ temb_channels: int,
+ resolution_idx: Optional[int] = None,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ transformer_layers_per_block: Union[int, Tuple[int]] = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default",
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ resnet_pre_norm: bool = True,
+ num_attention_heads: int = 1,
+ cross_attention_dim: int = 1280,
+ output_scale_factor: float = 1.0,
+ add_upsample: bool = True,
+ dual_cross_attention: bool = False,
+ use_linear_projection: bool = False,
+ only_cross_attention: bool = False,
+ upcast_attention: bool = False,
+ attention_type: str = "default",
+ ):
+ super().__init__()
+ resnets = []
+ attentions = []
+
+ self.has_cross_attention = True
+ self.num_attention_heads = num_attention_heads
+
+ if isinstance(transformer_layers_per_block, int):
+ transformer_layers_per_block = [transformer_layers_per_block] * num_layers
+
+ for i in range(num_layers):
+ res_skip_channels = in_channels if (i == num_layers - 1) else out_channels
+ resnet_in_channels = prev_output_channel if i == 0 else out_channels
+
+ resnets.append(
+ ResnetBlock2D(
+ in_channels=resnet_in_channels + res_skip_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ )
+ )
+ if not dual_cross_attention:
+ attentions.append(
+ Transformer2DModel(
+ num_attention_heads,
+ out_channels // num_attention_heads,
+ in_channels=out_channels,
+ num_layers=transformer_layers_per_block[i],
+ cross_attention_dim=cross_attention_dim,
+ norm_num_groups=resnet_groups,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention,
+ upcast_attention=upcast_attention,
+ attention_type=attention_type,
+ )
+ )
+ else:
+ attentions.append(
+ DualTransformer2DModel(
+ num_attention_heads,
+ out_channels // num_attention_heads,
+ in_channels=out_channels,
+ num_layers=1,
+ cross_attention_dim=cross_attention_dim,
+ norm_num_groups=resnet_groups,
+ )
+ )
+ self.attentions = nn.ModuleList(attentions)
+ self.resnets = nn.ModuleList(resnets)
+
+ if add_upsample:
+ self.upsamplers = nn.ModuleList(
+ [Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]
+ )
+ else:
+ self.upsamplers = None
+
+ self.gradient_checkpointing = False
+ self.resolution_idx = resolution_idx
+
+ def forward(
+ self,
+ hidden_states: torch.FloatTensor,
+ res_hidden_states_tuple: Tuple[torch.FloatTensor, ...],
+ temb: Optional[torch.FloatTensor] = None,
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,
+ upsample_size: Optional[int] = None,
+ attention_mask: Optional[torch.FloatTensor] = None,
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
+ ) -> torch.FloatTensor:
+ lora_scale = (
+ cross_attention_kwargs.get("scale", 1.0)
+ if cross_attention_kwargs is not None
+ else 1.0
+ )
+ is_freeu_enabled = (
+ getattr(self, "s1", None)
+ and getattr(self, "s2", None)
+ and getattr(self, "b1", None)
+ and getattr(self, "b2", None)
+ )
+
+ for resnet, attn in zip(self.resnets, self.attentions):
+ # pop res hidden states
+ res_hidden_states = res_hidden_states_tuple[-1]
+ res_hidden_states_tuple = res_hidden_states_tuple[:-1]
+
+ # FreeU: Only operate on the first two stages
+ if is_freeu_enabled:
+ hidden_states, res_hidden_states = apply_freeu(
+ self.resolution_idx,
+ hidden_states,
+ res_hidden_states,
+ s1=self.s1,
+ s2=self.s2,
+ b1=self.b1,
+ b2=self.b2,
+ )
+
+ hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)
+
+ if self.training and self.gradient_checkpointing:
+
+ def create_custom_forward(module, return_dict=None):
+ def custom_forward(*inputs):
+ if return_dict is not None:
+ return module(*inputs, return_dict=return_dict)
+ else:
+ return module(*inputs)
+
+ return custom_forward
+
+ ckpt_kwargs: Dict[str, Any] = (
+ {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {}
+ )
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet),
+ hidden_states,
+ temb,
+ **ckpt_kwargs,
+ )
+ hidden_states, ref_feature = attn(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ cross_attention_kwargs=cross_attention_kwargs,
+ attention_mask=attention_mask,
+ encoder_attention_mask=encoder_attention_mask,
+ return_dict=False,
+ )
+ else:
+ hidden_states = resnet(hidden_states, temb, scale=lora_scale)
+ hidden_states, ref_feature = attn(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ cross_attention_kwargs=cross_attention_kwargs,
+ attention_mask=attention_mask,
+ encoder_attention_mask=encoder_attention_mask,
+ return_dict=False,
+ )
+
+ if self.upsamplers is not None:
+ for upsampler in self.upsamplers:
+ hidden_states = upsampler(
+ hidden_states, upsample_size, scale=lora_scale
+ )
+
+ return hidden_states
+
+
+class UpBlock2D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ prev_output_channel: int,
+ out_channels: int,
+ temb_channels: int,
+ resolution_idx: Optional[int] = None,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default",
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ resnet_pre_norm: bool = True,
+ output_scale_factor: float = 1.0,
+ add_upsample: bool = True,
+ ):
+ super().__init__()
+ resnets = []
+
+ for i in range(num_layers):
+ res_skip_channels = in_channels if (i == num_layers - 1) else out_channels
+ resnet_in_channels = prev_output_channel if i == 0 else out_channels
+
+ resnets.append(
+ ResnetBlock2D(
+ in_channels=resnet_in_channels + res_skip_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ )
+ )
+
+ self.resnets = nn.ModuleList(resnets)
+
+ if add_upsample:
+ self.upsamplers = nn.ModuleList(
+ [Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]
+ )
+ else:
+ self.upsamplers = None
+
+ self.gradient_checkpointing = False
+ self.resolution_idx = resolution_idx
+
+ def forward(
+ self,
+ hidden_states: torch.FloatTensor,
+ res_hidden_states_tuple: Tuple[torch.FloatTensor, ...],
+ temb: Optional[torch.FloatTensor] = None,
+ upsample_size: Optional[int] = None,
+ scale: float = 1.0,
+ ) -> torch.FloatTensor:
+ is_freeu_enabled = (
+ getattr(self, "s1", None)
+ and getattr(self, "s2", None)
+ and getattr(self, "b1", None)
+ and getattr(self, "b2", None)
+ )
+
+ for resnet in self.resnets:
+ # pop res hidden states
+ res_hidden_states = res_hidden_states_tuple[-1]
+ res_hidden_states_tuple = res_hidden_states_tuple[:-1]
+
+ # FreeU: Only operate on the first two stages
+ if is_freeu_enabled:
+ hidden_states, res_hidden_states = apply_freeu(
+ self.resolution_idx,
+ hidden_states,
+ res_hidden_states,
+ s1=self.s1,
+ s2=self.s2,
+ b1=self.b1,
+ b2=self.b2,
+ )
+
+ hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)
+
+ if self.training and self.gradient_checkpointing:
+
+ def create_custom_forward(module):
+ def custom_forward(*inputs):
+ return module(*inputs)
+
+ return custom_forward
+
+ if is_torch_version(">=", "1.11.0"):
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet),
+ hidden_states,
+ temb,
+ use_reentrant=False,
+ )
+ else:
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet), hidden_states, temb
+ )
+ else:
+ hidden_states = resnet(hidden_states, temb, scale=scale)
+
+ if self.upsamplers is not None:
+ for upsampler in self.upsamplers:
+ hidden_states = upsampler(hidden_states, upsample_size, scale=scale)
+
+ return hidden_states
diff --git a/src/models/unet_2d_condition.py b/src/models/unet_2d_condition.py
new file mode 100644
index 0000000..38969ba
--- /dev/null
+++ b/src/models/unet_2d_condition.py
@@ -0,0 +1,1311 @@
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.utils.checkpoint
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.loaders import UNet2DConditionLoadersMixin
+from diffusers.models.activations import get_activation
+from diffusers.models.attention_processor import (
+ ADDED_KV_ATTENTION_PROCESSORS,
+ CROSS_ATTENTION_PROCESSORS,
+ AttentionProcessor,
+ AttnAddedKVProcessor,
+ AttnProcessor,
+)
+from diffusers.models.embeddings import (
+ GaussianFourierProjection,
+ ImageHintTimeEmbedding,
+ ImageProjection,
+ ImageTimeEmbedding,
+ TextImageProjection,
+ TextImageTimeEmbedding,
+ TextTimeEmbedding,
+ TimestepEmbedding,
+ Timesteps,
+)
+from diffusers.models.embeddings import GLIGENTextBoundingboxProjection as PositionNet
+from diffusers.models.modeling_utils import ModelMixin
+from diffusers.utils import (
+ USE_PEFT_BACKEND,
+ BaseOutput,
+ deprecate,
+ logging,
+ scale_lora_layers,
+ unscale_lora_layers,
+)
+
+from .unet_2d_blocks import (
+ UNetMidBlock2D,
+ UNetMidBlock2DCrossAttn,
+ get_down_block,
+ get_up_block,
+)
+
+logger = logging.get_logger(__name__) # pylint: disable=invalid-name
+
+
+@dataclass
+class UNet2DConditionOutput(BaseOutput):
+ """
+ The output of [`UNet2DConditionModel`].
+
+ Args:
+ sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+ The hidden states output conditioned on `encoder_hidden_states` input. Output of last layer of model.
+ """
+
+ sample: torch.FloatTensor = None
+ ref_features: Tuple[torch.FloatTensor] = None
+
+
+class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin):
+ r"""
+ A conditional 2D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample
+ shaped output.
+
+ This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented
+ for all models (such as downloading or saving).
+
+ Parameters:
+ sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`):
+ Height and width of input/output sample.
+ in_channels (`int`, *optional*, defaults to 4): Number of channels in the input sample.
+ out_channels (`int`, *optional*, defaults to 4): Number of channels in the output.
+ center_input_sample (`bool`, *optional*, defaults to `False`): Whether to center the input sample.
+ flip_sin_to_cos (`bool`, *optional*, defaults to `False`):
+ Whether to flip the sin to cos in the time embedding.
+ freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding.
+ down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`):
+ The tuple of downsample blocks to use.
+ mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2DCrossAttn"`):
+ Block type for middle of UNet, it can be one of `UNetMidBlock2DCrossAttn`, `UNetMidBlock2D`, or
+ `UNetMidBlock2DSimpleCrossAttn`. If `None`, the mid block layer is skipped.
+ up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`):
+ The tuple of upsample blocks to use.
+ only_cross_attention(`bool` or `Tuple[bool]`, *optional*, default to `False`):
+ Whether to include self-attention in the basic transformer blocks, see
+ [`~models.attention.BasicTransformerBlock`].
+ block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`):
+ The tuple of output channels for each block.
+ layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block.
+ downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution.
+ mid_block_scale_factor (`float`, *optional*, defaults to 1.0): The scale factor to use for the mid block.
+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
+ act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use.
+ norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization.
+ If `None`, normalization and activation layers is skipped in post-processing.
+ norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization.
+ cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280):
+ The dimension of the cross attention features.
+ transformer_layers_per_block (`int`, `Tuple[int]`, or `Tuple[Tuple]` , *optional*, defaults to 1):
+ The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for
+ [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`],
+ [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`].
+ reverse_transformer_layers_per_block : (`Tuple[Tuple]`, *optional*, defaults to None):
+ The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`], in the upsampling
+ blocks of the U-Net. Only relevant if `transformer_layers_per_block` is of type `Tuple[Tuple]` and for
+ [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`],
+ [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`].
+ encoder_hid_dim (`int`, *optional*, defaults to None):
+ If `encoder_hid_dim_type` is defined, `encoder_hidden_states` will be projected from `encoder_hid_dim`
+ dimension to `cross_attention_dim`.
+ encoder_hid_dim_type (`str`, *optional*, defaults to `None`):
+ If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text
+ embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`.
+ attention_head_dim (`int`, *optional*, defaults to 8): The dimension of the attention heads.
+ num_attention_heads (`int`, *optional*):
+ The number of attention heads. If not defined, defaults to `attention_head_dim`
+ resnet_time_scale_shift (`str`, *optional*, defaults to `"default"`): Time scale shift config
+ for ResNet blocks (see [`~models.resnet.ResnetBlock2D`]). Choose from `default` or `scale_shift`.
+ class_embed_type (`str`, *optional*, defaults to `None`):
+ The type of class embedding to use which is ultimately summed with the time embeddings. Choose from `None`,
+ `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`.
+ addition_embed_type (`str`, *optional*, defaults to `None`):
+ Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or
+ "text". "text" will use the `TextTimeEmbedding` layer.
+ addition_time_embed_dim: (`int`, *optional*, defaults to `None`):
+ Dimension for the timestep embeddings.
+ num_class_embeds (`int`, *optional*, defaults to `None`):
+ Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing
+ class conditioning with `class_embed_type` equal to `None`.
+ time_embedding_type (`str`, *optional*, defaults to `positional`):
+ The type of position embedding to use for timesteps. Choose from `positional` or `fourier`.
+ time_embedding_dim (`int`, *optional*, defaults to `None`):
+ An optional override for the dimension of the projected time embedding.
+ time_embedding_act_fn (`str`, *optional*, defaults to `None`):
+ Optional activation function to use only once on the time embeddings before they are passed to the rest of
+ the UNet. Choose from `silu`, `mish`, `gelu`, and `swish`.
+ timestep_post_act (`str`, *optional*, defaults to `None`):
+ The second activation function to use in timestep embedding. Choose from `silu`, `mish` and `gelu`.
+ time_cond_proj_dim (`int`, *optional*, defaults to `None`):
+ The dimension of `cond_proj` layer in the timestep embedding.
+ conv_in_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_in` layer. conv_out_kernel (`int`,
+ *optional*, default to `3`): The kernel size of `conv_out` layer. projection_class_embeddings_input_dim (`int`,
+ *optional*): The dimension of the `class_labels` input when
+ `class_embed_type="projection"`. Required when `class_embed_type="projection"`.
+ class_embeddings_concat (`bool`, *optional*, defaults to `False`): Whether to concatenate the time
+ embeddings with the class embeddings.
+ mid_block_only_cross_attention (`bool`, *optional*, defaults to `None`):
+ Whether to use cross attention with the mid block when using the `UNetMidBlock2DSimpleCrossAttn`. If
+ `only_cross_attention` is given as a single boolean and `mid_block_only_cross_attention` is `None`, the
+ `only_cross_attention` value is used as the value for `mid_block_only_cross_attention`. Default to `False`
+ otherwise.
+ """
+
+ _supports_gradient_checkpointing = True
+
+ @register_to_config
+ def __init__(
+ self,
+ sample_size: Optional[int] = None,
+ in_channels: int = 4,
+ out_channels: int = 4,
+ center_input_sample: bool = False,
+ flip_sin_to_cos: bool = True,
+ freq_shift: int = 0,
+ down_block_types: Tuple[str] = (
+ "CrossAttnDownBlock2D",
+ "CrossAttnDownBlock2D",
+ "CrossAttnDownBlock2D",
+ "DownBlock2D",
+ ),
+ mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn",
+ up_block_types: Tuple[str] = (
+ "UpBlock2D",
+ "CrossAttnUpBlock2D",
+ "CrossAttnUpBlock2D",
+ "CrossAttnUpBlock2D",
+ ),
+ only_cross_attention: Union[bool, Tuple[bool]] = False,
+ block_out_channels: Tuple[int] = (320, 640, 1280, 1280),
+ layers_per_block: Union[int, Tuple[int]] = 2,
+ downsample_padding: int = 1,
+ mid_block_scale_factor: float = 1,
+ dropout: float = 0.0,
+ act_fn: str = "silu",
+ norm_num_groups: Optional[int] = 32,
+ norm_eps: float = 1e-5,
+ cross_attention_dim: Union[int, Tuple[int]] = 1280,
+ transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple]] = 1,
+ reverse_transformer_layers_per_block: Optional[Tuple[Tuple[int]]] = None,
+ encoder_hid_dim: Optional[int] = None,
+ encoder_hid_dim_type: Optional[str] = None,
+ attention_head_dim: Union[int, Tuple[int]] = 8,
+ num_attention_heads: Optional[Union[int, Tuple[int]]] = None,
+ dual_cross_attention: bool = False,
+ use_linear_projection: bool = False,
+ class_embed_type: Optional[str] = None,
+ addition_embed_type: Optional[str] = None,
+ addition_time_embed_dim: Optional[int] = None,
+ num_class_embeds: Optional[int] = None,
+ upcast_attention: bool = False,
+ resnet_time_scale_shift: str = "default",
+ resnet_skip_time_act: bool = False,
+ resnet_out_scale_factor: int = 1.0,
+ time_embedding_type: str = "positional",
+ time_embedding_dim: Optional[int] = None,
+ time_embedding_act_fn: Optional[str] = None,
+ timestep_post_act: Optional[str] = None,
+ time_cond_proj_dim: Optional[int] = None,
+ conv_in_kernel: int = 3,
+ conv_out_kernel: int = 3,
+ projection_class_embeddings_input_dim: Optional[int] = None,
+ attention_type: str = "default",
+ class_embeddings_concat: bool = False,
+ mid_block_only_cross_attention: Optional[bool] = None,
+ cross_attention_norm: Optional[str] = None,
+ addition_embed_type_num_heads=64,
+ ):
+ super().__init__()
+
+ self.sample_size = sample_size
+
+ if num_attention_heads is not None:
+ raise ValueError(
+ "At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19."
+ )
+
+ # If `num_attention_heads` is not defined (which is the case for most models)
+ # it will default to `attention_head_dim`. This looks weird upon first reading it and it is.
+ # The reason for this behavior is to correct for incorrectly named variables that were introduced
+ # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131
+ # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking
+ # which is why we correct for the naming here.
+ num_attention_heads = num_attention_heads or attention_head_dim
+
+ # Check inputs
+ if len(down_block_types) != len(up_block_types):
+ raise ValueError(
+ f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}."
+ )
+
+ if len(block_out_channels) != len(down_block_types):
+ raise ValueError(
+ f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}."
+ )
+
+ if not isinstance(only_cross_attention, bool) and len(
+ only_cross_attention
+ ) != len(down_block_types):
+ raise ValueError(
+ f"Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}."
+ )
+
+ if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(
+ down_block_types
+ ):
+ raise ValueError(
+ f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}."
+ )
+
+ if not isinstance(attention_head_dim, int) and len(attention_head_dim) != len(
+ down_block_types
+ ):
+ raise ValueError(
+ f"Must provide the same number of `attention_head_dim` as `down_block_types`. `attention_head_dim`: {attention_head_dim}. `down_block_types`: {down_block_types}."
+ )
+
+ if isinstance(cross_attention_dim, list) and len(cross_attention_dim) != len(
+ down_block_types
+ ):
+ raise ValueError(
+ f"Must provide the same number of `cross_attention_dim` as `down_block_types`. `cross_attention_dim`: {cross_attention_dim}. `down_block_types`: {down_block_types}."
+ )
+
+ if not isinstance(layers_per_block, int) and len(layers_per_block) != len(
+ down_block_types
+ ):
+ raise ValueError(
+ f"Must provide the same number of `layers_per_block` as `down_block_types`. `layers_per_block`: {layers_per_block}. `down_block_types`: {down_block_types}."
+ )
+ if (
+ isinstance(transformer_layers_per_block, list)
+ and reverse_transformer_layers_per_block is None
+ ):
+ for layer_number_per_block in transformer_layers_per_block:
+ if isinstance(layer_number_per_block, list):
+ raise ValueError(
+ "Must provide 'reverse_transformer_layers_per_block` if using asymmetrical UNet."
+ )
+
+ # input
+ conv_in_padding = (conv_in_kernel - 1) // 2
+ self.conv_in = nn.Conv2d(
+ in_channels,
+ block_out_channels[0],
+ kernel_size=conv_in_kernel,
+ padding=conv_in_padding,
+ )
+
+ # time
+ if time_embedding_type == "fourier":
+ time_embed_dim = time_embedding_dim or block_out_channels[0] * 2
+ if time_embed_dim % 2 != 0:
+ raise ValueError(
+ f"`time_embed_dim` should be divisible by 2, but is {time_embed_dim}."
+ )
+ self.time_proj = GaussianFourierProjection(
+ time_embed_dim // 2,
+ set_W_to_weight=False,
+ log=False,
+ flip_sin_to_cos=flip_sin_to_cos,
+ )
+ timestep_input_dim = time_embed_dim
+ elif time_embedding_type == "positional":
+ time_embed_dim = time_embedding_dim or block_out_channels[0] * 4
+
+ self.time_proj = Timesteps(
+ block_out_channels[0], flip_sin_to_cos, freq_shift
+ )
+ timestep_input_dim = block_out_channels[0]
+ else:
+ raise ValueError(
+ f"{time_embedding_type} does not exist. Please make sure to use one of `fourier` or `positional`."
+ )
+
+ self.time_embedding = TimestepEmbedding(
+ timestep_input_dim,
+ time_embed_dim,
+ act_fn=act_fn,
+ post_act_fn=timestep_post_act,
+ cond_proj_dim=time_cond_proj_dim,
+ )
+
+ if encoder_hid_dim_type is None and encoder_hid_dim is not None:
+ encoder_hid_dim_type = "text_proj"
+ self.register_to_config(encoder_hid_dim_type=encoder_hid_dim_type)
+ logger.info(
+ "encoder_hid_dim_type defaults to 'text_proj' as `encoder_hid_dim` is defined."
+ )
+
+ if encoder_hid_dim is None and encoder_hid_dim_type is not None:
+ raise ValueError(
+ f"`encoder_hid_dim` has to be defined when `encoder_hid_dim_type` is set to {encoder_hid_dim_type}."
+ )
+
+ if encoder_hid_dim_type == "text_proj":
+ self.encoder_hid_proj = nn.Linear(encoder_hid_dim, cross_attention_dim)
+ elif encoder_hid_dim_type == "text_image_proj":
+ # image_embed_dim DOESN'T have to be `cross_attention_dim`. To not clutter the __init__ too much
+ # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use
+ # case when `addition_embed_type == "text_image_proj"` (Kadinsky 2.1)`
+ self.encoder_hid_proj = TextImageProjection(
+ text_embed_dim=encoder_hid_dim,
+ image_embed_dim=cross_attention_dim,
+ cross_attention_dim=cross_attention_dim,
+ )
+ elif encoder_hid_dim_type == "image_proj":
+ # Kandinsky 2.2
+ self.encoder_hid_proj = ImageProjection(
+ image_embed_dim=encoder_hid_dim,
+ cross_attention_dim=cross_attention_dim,
+ )
+ elif encoder_hid_dim_type is not None:
+ raise ValueError(
+ f"encoder_hid_dim_type: {encoder_hid_dim_type} must be None, 'text_proj' or 'text_image_proj'."
+ )
+ else:
+ self.encoder_hid_proj = None
+
+ # class embedding
+ if class_embed_type is None and num_class_embeds is not None:
+ self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim)
+ elif class_embed_type == "timestep":
+ self.class_embedding = TimestepEmbedding(
+ timestep_input_dim, time_embed_dim, act_fn=act_fn
+ )
+ elif class_embed_type == "identity":
+ self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim)
+ elif class_embed_type == "projection":
+ if projection_class_embeddings_input_dim is None:
+ raise ValueError(
+ "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set"
+ )
+ # The projection `class_embed_type` is the same as the timestep `class_embed_type` except
+ # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings
+ # 2. it projects from an arbitrary input dimension.
+ #
+ # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations.
+ # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings.
+ # As a result, `TimestepEmbedding` can be passed arbitrary vectors.
+ self.class_embedding = TimestepEmbedding(
+ projection_class_embeddings_input_dim, time_embed_dim
+ )
+ elif class_embed_type == "simple_projection":
+ if projection_class_embeddings_input_dim is None:
+ raise ValueError(
+ "`class_embed_type`: 'simple_projection' requires `projection_class_embeddings_input_dim` be set"
+ )
+ self.class_embedding = nn.Linear(
+ projection_class_embeddings_input_dim, time_embed_dim
+ )
+ else:
+ self.class_embedding = None
+
+ if addition_embed_type == "text":
+ if encoder_hid_dim is not None:
+ text_time_embedding_from_dim = encoder_hid_dim
+ else:
+ text_time_embedding_from_dim = cross_attention_dim
+
+ self.add_embedding = TextTimeEmbedding(
+ text_time_embedding_from_dim,
+ time_embed_dim,
+ num_heads=addition_embed_type_num_heads,
+ )
+ elif addition_embed_type == "text_image":
+ # text_embed_dim and image_embed_dim DON'T have to be `cross_attention_dim`. To not clutter the __init__ too much
+ # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use
+ # case when `addition_embed_type == "text_image"` (Kadinsky 2.1)`
+ self.add_embedding = TextImageTimeEmbedding(
+ text_embed_dim=cross_attention_dim,
+ image_embed_dim=cross_attention_dim,
+ time_embed_dim=time_embed_dim,
+ )
+ elif addition_embed_type == "text_time":
+ self.add_time_proj = Timesteps(
+ addition_time_embed_dim, flip_sin_to_cos, freq_shift
+ )
+ self.add_embedding = TimestepEmbedding(
+ projection_class_embeddings_input_dim, time_embed_dim
+ )
+ elif addition_embed_type == "image":
+ # Kandinsky 2.2
+ self.add_embedding = ImageTimeEmbedding(
+ image_embed_dim=encoder_hid_dim, time_embed_dim=time_embed_dim
+ )
+ elif addition_embed_type == "image_hint":
+ # Kandinsky 2.2 ControlNet
+ self.add_embedding = ImageHintTimeEmbedding(
+ image_embed_dim=encoder_hid_dim, time_embed_dim=time_embed_dim
+ )
+ elif addition_embed_type is not None:
+ raise ValueError(
+ f"addition_embed_type: {addition_embed_type} must be None, 'text' or 'text_image'."
+ )
+
+ if time_embedding_act_fn is None:
+ self.time_embed_act = None
+ else:
+ self.time_embed_act = get_activation(time_embedding_act_fn)
+
+ self.down_blocks = nn.ModuleList([])
+ self.up_blocks = nn.ModuleList([])
+
+ if isinstance(only_cross_attention, bool):
+ if mid_block_only_cross_attention is None:
+ mid_block_only_cross_attention = only_cross_attention
+
+ only_cross_attention = [only_cross_attention] * len(down_block_types)
+
+ if mid_block_only_cross_attention is None:
+ mid_block_only_cross_attention = False
+
+ if isinstance(num_attention_heads, int):
+ num_attention_heads = (num_attention_heads,) * len(down_block_types)
+
+ if isinstance(attention_head_dim, int):
+ attention_head_dim = (attention_head_dim,) * len(down_block_types)
+
+ if isinstance(cross_attention_dim, int):
+ cross_attention_dim = (cross_attention_dim,) * len(down_block_types)
+
+ if isinstance(layers_per_block, int):
+ layers_per_block = [layers_per_block] * len(down_block_types)
+
+ if isinstance(transformer_layers_per_block, int):
+ transformer_layers_per_block = [transformer_layers_per_block] * len(
+ down_block_types
+ )
+
+ if class_embeddings_concat:
+ # The time embeddings are concatenated with the class embeddings. The dimension of the
+ # time embeddings passed to the down, middle, and up blocks is twice the dimension of the
+ # regular time embeddings
+ blocks_time_embed_dim = time_embed_dim * 2
+ else:
+ blocks_time_embed_dim = time_embed_dim
+
+ # down
+ output_channel = block_out_channels[0]
+ for i, down_block_type in enumerate(down_block_types):
+ input_channel = output_channel
+ output_channel = block_out_channels[i]
+ is_final_block = i == len(block_out_channels) - 1
+
+ down_block = get_down_block(
+ down_block_type,
+ num_layers=layers_per_block[i],
+ transformer_layers_per_block=transformer_layers_per_block[i],
+ in_channels=input_channel,
+ out_channels=output_channel,
+ temb_channels=blocks_time_embed_dim,
+ add_downsample=not is_final_block,
+ resnet_eps=norm_eps,
+ resnet_act_fn=act_fn,
+ resnet_groups=norm_num_groups,
+ cross_attention_dim=cross_attention_dim[i],
+ num_attention_heads=num_attention_heads[i],
+ downsample_padding=downsample_padding,
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention[i],
+ upcast_attention=upcast_attention,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ attention_type=attention_type,
+ resnet_skip_time_act=resnet_skip_time_act,
+ resnet_out_scale_factor=resnet_out_scale_factor,
+ cross_attention_norm=cross_attention_norm,
+ attention_head_dim=attention_head_dim[i]
+ if attention_head_dim[i] is not None
+ else output_channel,
+ dropout=dropout,
+ )
+ self.down_blocks.append(down_block)
+
+ # mid
+ if mid_block_type == "UNetMidBlock2DCrossAttn":
+ self.mid_block = UNetMidBlock2DCrossAttn(
+ transformer_layers_per_block=transformer_layers_per_block[-1],
+ in_channels=block_out_channels[-1],
+ temb_channels=blocks_time_embed_dim,
+ dropout=dropout,
+ resnet_eps=norm_eps,
+ resnet_act_fn=act_fn,
+ output_scale_factor=mid_block_scale_factor,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ cross_attention_dim=cross_attention_dim[-1],
+ num_attention_heads=num_attention_heads[-1],
+ resnet_groups=norm_num_groups,
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ upcast_attention=upcast_attention,
+ attention_type=attention_type,
+ )
+ elif mid_block_type == "UNetMidBlock2DSimpleCrossAttn":
+ raise NotImplementedError(f"Unsupport mid_block_type: {mid_block_type}")
+ elif mid_block_type == "UNetMidBlock2D":
+ self.mid_block = UNetMidBlock2D(
+ in_channels=block_out_channels[-1],
+ temb_channels=blocks_time_embed_dim,
+ dropout=dropout,
+ num_layers=0,
+ resnet_eps=norm_eps,
+ resnet_act_fn=act_fn,
+ output_scale_factor=mid_block_scale_factor,
+ resnet_groups=norm_num_groups,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ add_attention=False,
+ )
+ elif mid_block_type is None:
+ self.mid_block = None
+ else:
+ raise ValueError(f"unknown mid_block_type : {mid_block_type}")
+
+ # count how many layers upsample the images
+ self.num_upsamplers = 0
+
+ # up
+ reversed_block_out_channels = list(reversed(block_out_channels))
+ reversed_num_attention_heads = list(reversed(num_attention_heads))
+ reversed_layers_per_block = list(reversed(layers_per_block))
+ reversed_cross_attention_dim = list(reversed(cross_attention_dim))
+ reversed_transformer_layers_per_block = (
+ list(reversed(transformer_layers_per_block))
+ if reverse_transformer_layers_per_block is None
+ else reverse_transformer_layers_per_block
+ )
+ only_cross_attention = list(reversed(only_cross_attention))
+
+ output_channel = reversed_block_out_channels[0]
+ for i, up_block_type in enumerate(up_block_types):
+ is_final_block = i == len(block_out_channels) - 1
+
+ prev_output_channel = output_channel
+ output_channel = reversed_block_out_channels[i]
+ input_channel = reversed_block_out_channels[
+ min(i + 1, len(block_out_channels) - 1)
+ ]
+
+ # add upsample block for all BUT final layer
+ if not is_final_block:
+ add_upsample = True
+ self.num_upsamplers += 1
+ else:
+ add_upsample = False
+
+ up_block = get_up_block(
+ up_block_type,
+ num_layers=reversed_layers_per_block[i] + 1,
+ transformer_layers_per_block=reversed_transformer_layers_per_block[i],
+ in_channels=input_channel,
+ out_channels=output_channel,
+ prev_output_channel=prev_output_channel,
+ temb_channels=blocks_time_embed_dim,
+ add_upsample=add_upsample,
+ resnet_eps=norm_eps,
+ resnet_act_fn=act_fn,
+ resolution_idx=i,
+ resnet_groups=norm_num_groups,
+ cross_attention_dim=reversed_cross_attention_dim[i],
+ num_attention_heads=reversed_num_attention_heads[i],
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention[i],
+ upcast_attention=upcast_attention,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ attention_type=attention_type,
+ resnet_skip_time_act=resnet_skip_time_act,
+ resnet_out_scale_factor=resnet_out_scale_factor,
+ cross_attention_norm=cross_attention_norm,
+ attention_head_dim=attention_head_dim[i]
+ if attention_head_dim[i] is not None
+ else output_channel,
+ dropout=dropout,
+ )
+ self.up_blocks.append(up_block)
+ prev_output_channel = output_channel
+
+ # out
+ if norm_num_groups is not None:
+ self.conv_norm_out = nn.GroupNorm(
+ num_channels=block_out_channels[0],
+ num_groups=norm_num_groups,
+ eps=norm_eps,
+ )
+
+ self.conv_act = get_activation(act_fn)
+
+ else:
+ self.conv_norm_out = None
+ self.conv_act = None
+ self.conv_norm_out = None
+
+ conv_out_padding = (conv_out_kernel - 1) // 2
+ # self.conv_out = nn.Conv2d(
+ # block_out_channels[0],
+ # out_channels,
+ # kernel_size=conv_out_kernel,
+ # padding=conv_out_padding,
+ # )
+
+ if attention_type in ["gated", "gated-text-image"]:
+ positive_len = 768
+ if isinstance(cross_attention_dim, int):
+ positive_len = cross_attention_dim
+ elif isinstance(cross_attention_dim, tuple) or isinstance(
+ cross_attention_dim, list
+ ):
+ positive_len = cross_attention_dim[0]
+
+ feature_type = "text-only" if attention_type == "gated" else "text-image"
+ self.position_net = PositionNet(
+ positive_len=positive_len,
+ out_dim=cross_attention_dim,
+ feature_type=feature_type,
+ )
+
+ @property
+ def attn_processors(self) -> Dict[str, AttentionProcessor]:
+ r"""
+ Returns:
+ `dict` of attention processors: A dictionary containing all attention processors used in the model with
+ indexed by its weight name.
+ """
+ # set recursively
+ processors = {}
+
+ def fn_recursive_add_processors(
+ name: str,
+ module: torch.nn.Module,
+ processors: Dict[str, AttentionProcessor],
+ ):
+ if hasattr(module, "get_processor"):
+ processors[f"{name}.processor"] = module.get_processor(
+ return_deprecated_lora=True
+ )
+
+ for sub_name, child in module.named_children():
+ fn_recursive_add_processors(f"{name}.{sub_name}", child, processors)
+
+ return processors
+
+ for name, module in self.named_children():
+ fn_recursive_add_processors(name, module, processors)
+
+ return processors
+
+ def set_attn_processor(
+ self,
+ processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]],
+ _remove_lora=False,
+ ):
+ r"""
+ Sets the attention processor to use to compute attention.
+
+ Parameters:
+ processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):
+ The instantiated processor class or a dictionary of processor classes that will be set as the processor
+ for **all** `Attention` layers.
+
+ If `processor` is a dict, the key needs to define the path to the corresponding cross attention
+ processor. This is strongly recommended when setting trainable attention processors.
+
+ """
+ count = len(self.attn_processors.keys())
+
+ if isinstance(processor, dict) and len(processor) != count:
+ raise ValueError(
+ f"A dict of processors was passed, but the number of processors {len(processor)} does not match the"
+ f" number of attention layers: {count}. Please make sure to pass {count} processor classes."
+ )
+
+ def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor):
+ if hasattr(module, "set_processor"):
+ if not isinstance(processor, dict):
+ module.set_processor(processor, _remove_lora=_remove_lora)
+ else:
+ module.set_processor(
+ processor.pop(f"{name}.processor"), _remove_lora=_remove_lora
+ )
+
+ for sub_name, child in module.named_children():
+ fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor)
+
+ for name, module in self.named_children():
+ fn_recursive_attn_processor(name, module, processor)
+
+ def set_default_attn_processor(self):
+ """
+ Disables custom attention processors and sets the default attention implementation.
+ """
+ if all(
+ proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS
+ for proc in self.attn_processors.values()
+ ):
+ processor = AttnAddedKVProcessor()
+ elif all(
+ proc.__class__ in CROSS_ATTENTION_PROCESSORS
+ for proc in self.attn_processors.values()
+ ):
+ processor = AttnProcessor()
+ else:
+ raise ValueError(
+ f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}"
+ )
+
+ self.set_attn_processor(processor, _remove_lora=True)
+
+ def set_attention_slice(self, slice_size):
+ r"""
+ Enable sliced attention computation.
+
+ When this option is enabled, the attention module splits the input tensor in slices to compute attention in
+ several steps. This is useful for saving some memory in exchange for a small decrease in speed.
+
+ Args:
+ slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`):
+ When `"auto"`, input to the attention heads is halved, so attention is computed in two steps. If
+ `"max"`, maximum amount of memory is saved by running only one slice at a time. If a number is
+ provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim`
+ must be a multiple of `slice_size`.
+ """
+ sliceable_head_dims = []
+
+ def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module):
+ if hasattr(module, "set_attention_slice"):
+ sliceable_head_dims.append(module.sliceable_head_dim)
+
+ for child in module.children():
+ fn_recursive_retrieve_sliceable_dims(child)
+
+ # retrieve number of attention layers
+ for module in self.children():
+ fn_recursive_retrieve_sliceable_dims(module)
+
+ num_sliceable_layers = len(sliceable_head_dims)
+
+ if slice_size == "auto":
+ # half the attention head size is usually a good trade-off between
+ # speed and memory
+ slice_size = [dim // 2 for dim in sliceable_head_dims]
+ elif slice_size == "max":
+ # make smallest slice possible
+ slice_size = num_sliceable_layers * [1]
+
+ slice_size = (
+ num_sliceable_layers * [slice_size]
+ if not isinstance(slice_size, list)
+ else slice_size
+ )
+
+ if len(slice_size) != len(sliceable_head_dims):
+ raise ValueError(
+ f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different"
+ f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}."
+ )
+
+ for i in range(len(slice_size)):
+ size = slice_size[i]
+ dim = sliceable_head_dims[i]
+ if size is not None and size > dim:
+ raise ValueError(f"size {size} has to be smaller or equal to {dim}.")
+
+ # Recursively walk through all the children.
+ # Any children which exposes the set_attention_slice method
+ # gets the message
+ def fn_recursive_set_attention_slice(
+ module: torch.nn.Module, slice_size: List[int]
+ ):
+ if hasattr(module, "set_attention_slice"):
+ module.set_attention_slice(slice_size.pop())
+
+ for child in module.children():
+ fn_recursive_set_attention_slice(child, slice_size)
+
+ reversed_slice_size = list(reversed(slice_size))
+ for module in self.children():
+ fn_recursive_set_attention_slice(module, reversed_slice_size)
+
+ def _set_gradient_checkpointing(self, module, value=False):
+ if hasattr(module, "gradient_checkpointing"):
+ module.gradient_checkpointing = value
+
+ def enable_freeu(self, s1, s2, b1, b2):
+ r"""Enables the FreeU mechanism from https://arxiv.org/abs/2309.11497.
+
+ The suffixes after the scaling factors represent the stage blocks where they are being applied.
+
+ Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of values that
+ are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL.
+
+ Args:
+ s1 (`float`):
+ Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to
+ mitigate the "oversmoothing effect" in the enhanced denoising process.
+ s2 (`float`):
+ Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to
+ mitigate the "oversmoothing effect" in the enhanced denoising process.
+ b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features.
+ b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features.
+ """
+ for i, upsample_block in enumerate(self.up_blocks):
+ setattr(upsample_block, "s1", s1)
+ setattr(upsample_block, "s2", s2)
+ setattr(upsample_block, "b1", b1)
+ setattr(upsample_block, "b2", b2)
+
+ def disable_freeu(self):
+ """Disables the FreeU mechanism."""
+ freeu_keys = {"s1", "s2", "b1", "b2"}
+ for i, upsample_block in enumerate(self.up_blocks):
+ for k in freeu_keys:
+ if (
+ hasattr(upsample_block, k)
+ or getattr(upsample_block, k, None) is not None
+ ):
+ setattr(upsample_block, k, None)
+
+ def forward(
+ self,
+ sample: torch.FloatTensor,
+ timestep: Union[torch.Tensor, float, int],
+ encoder_hidden_states: torch.Tensor,
+ class_labels: Optional[torch.Tensor] = None,
+ timestep_cond: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,
+ added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,
+ down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
+ mid_block_additional_residual: Optional[torch.Tensor] = None,
+ down_intrablock_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
+ encoder_attention_mask: Optional[torch.Tensor] = None,
+ return_dict: bool = True,
+ only_return_middle_fea: bool = False,
+ ) -> Union[UNet2DConditionOutput, Tuple]:
+ r"""
+ The [`UNet2DConditionModel`] forward method.
+
+ Args:
+ sample (`torch.FloatTensor`):
+ The noisy input tensor with the following shape `(batch, channel, height, width)`.
+ timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input.
+ encoder_hidden_states (`torch.FloatTensor`):
+ The encoder hidden states with shape `(batch, sequence_length, feature_dim)`.
+ class_labels (`torch.Tensor`, *optional*, defaults to `None`):
+ Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings.
+ timestep_cond: (`torch.Tensor`, *optional*, defaults to `None`):
+ Conditional embeddings for timestep. If provided, the embeddings will be summed with the samples passed
+ through the `self.time_embedding` layer to obtain the timestep embeddings.
+ attention_mask (`torch.Tensor`, *optional*, defaults to `None`):
+ An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask
+ is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large
+ negative values to the attention scores corresponding to "discard" tokens.
+ cross_attention_kwargs (`dict`, *optional*):
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
+ `self.processor` in
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
+ added_cond_kwargs: (`dict`, *optional*):
+ A kwargs dictionary containing additional embeddings that if specified are added to the embeddings that
+ are passed along to the UNet blocks.
+ down_block_additional_residuals: (`tuple` of `torch.Tensor`, *optional*):
+ A tuple of tensors that if specified are added to the residuals of down unet blocks.
+ mid_block_additional_residual: (`torch.Tensor`, *optional*):
+ A tensor that if specified is added to the residual of the middle unet block.
+ encoder_attention_mask (`torch.Tensor`):
+ A cross-attention mask of shape `(batch, sequence_length)` is applied to `encoder_hidden_states`. If
+ `True` the mask is kept, otherwise if `False` it is discarded. Mask will be converted into a bias,
+ which adds large negative values to the attention scores corresponding to "discard" tokens.
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether or not to return a [`~models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain
+ tuple.
+ cross_attention_kwargs (`dict`, *optional*):
+ A kwargs dictionary that if specified is passed along to the [`AttnProcessor`].
+ added_cond_kwargs: (`dict`, *optional*):
+ A kwargs dictionary containin additional embeddings that if specified are added to the embeddings that
+ are passed along to the UNet blocks.
+ down_block_additional_residuals (`tuple` of `torch.Tensor`, *optional*):
+ additional residuals to be added to UNet long skip connections from down blocks to up blocks for
+ example from ControlNet side model(s)
+ mid_block_additional_residual (`torch.Tensor`, *optional*):
+ additional residual to be added to UNet mid block output, for example from ControlNet side model
+ down_intrablock_additional_residuals (`tuple` of `torch.Tensor`, *optional*):
+ additional residuals to be added within UNet down blocks, for example from T2I-Adapter side model(s)
+
+ Returns:
+ [`~models.unet_2d_condition.UNet2DConditionOutput`] or `tuple`:
+ If `return_dict` is True, an [`~models.unet_2d_condition.UNet2DConditionOutput`] is returned, otherwise
+ a `tuple` is returned where the first element is the sample tensor.
+ """
+ # By default samples have to be AT least a multiple of the overall upsampling factor.
+ # The overall upsampling factor is equal to 2 ** (# num of upsampling layers).
+ # However, the upsampling interpolation output size can be forced to fit any upsampling size
+ # on the fly if necessary.
+ default_overall_up_factor = 2**self.num_upsamplers
+
+ # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`
+ forward_upsample_size = False
+ upsample_size = None
+
+ for dim in sample.shape[-2:]:
+ if dim % default_overall_up_factor != 0:
+ # Forward upsample size to force interpolation output size.
+ forward_upsample_size = True
+ break
+
+ # ensure attention_mask is a bias, and give it a singleton query_tokens dimension
+ # expects mask of shape:
+ # [batch, key_tokens]
+ # adds singleton query_tokens dimension:
+ # [batch, 1, key_tokens]
+ # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
+ # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn)
+ # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
+ if attention_mask is not None:
+ # assume that mask is expressed as:
+ # (1 = keep, 0 = discard)
+ # convert mask into a bias that can be added to attention scores:
+ # (keep = +0, discard = -10000.0)
+ attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0
+ attention_mask = attention_mask.unsqueeze(1)
+
+ # convert encoder_attention_mask to a bias the same way we do for attention_mask
+ if encoder_attention_mask is not None:
+ encoder_attention_mask = (
+ 1 - encoder_attention_mask.to(sample.dtype)
+ ) * -10000.0
+ encoder_attention_mask = encoder_attention_mask.unsqueeze(1)
+
+ # 0. center input if necessary
+ if self.config.center_input_sample:
+ sample = 2 * sample - 1.0
+
+ # 1. time
+ timesteps = timestep
+ if not torch.is_tensor(timesteps):
+ # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can
+ # This would be a good case for the `match` statement (Python 3.10+)
+ is_mps = sample.device.type == "mps"
+ if isinstance(timestep, float):
+ dtype = torch.float32 if is_mps else torch.float64
+ else:
+ dtype = torch.int32 if is_mps else torch.int64
+ timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)
+ elif len(timesteps.shape) == 0:
+ timesteps = timesteps[None].to(sample.device)
+
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
+ timesteps = timesteps.expand(sample.shape[0])
+
+ t_emb = self.time_proj(timesteps)
+
+ # `Timesteps` does not contain any weights and will always return f32 tensors
+ # but time_embedding might actually be running in fp16. so we need to cast here.
+ # there might be better ways to encapsulate this.
+ t_emb = t_emb.to(dtype=sample.dtype)
+
+ emb = self.time_embedding(t_emb, timestep_cond)
+ aug_emb = None
+
+ if self.class_embedding is not None:
+ if class_labels is None:
+ raise ValueError(
+ "class_labels should be provided when num_class_embeds > 0"
+ )
+
+ if self.config.class_embed_type == "timestep":
+ class_labels = self.time_proj(class_labels)
+
+ # `Timesteps` does not contain any weights and will always return f32 tensors
+ # there might be better ways to encapsulate this.
+ class_labels = class_labels.to(dtype=sample.dtype)
+
+ class_emb = self.class_embedding(class_labels).to(dtype=sample.dtype)
+
+ if self.config.class_embeddings_concat:
+ emb = torch.cat([emb, class_emb], dim=-1)
+ else:
+ emb = emb + class_emb
+
+ if self.config.addition_embed_type == "text":
+ aug_emb = self.add_embedding(encoder_hidden_states)
+ elif self.config.addition_embed_type == "text_image":
+ # Kandinsky 2.1 - style
+ if "image_embeds" not in added_cond_kwargs:
+ raise ValueError(
+ f"{self.__class__} has the config param `addition_embed_type` set to 'text_image' which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`"
+ )
+
+ image_embs = added_cond_kwargs.get("image_embeds")
+ text_embs = added_cond_kwargs.get("text_embeds", encoder_hidden_states)
+ aug_emb = self.add_embedding(text_embs, image_embs)
+ elif self.config.addition_embed_type == "text_time":
+ # SDXL - style
+ if "text_embeds" not in added_cond_kwargs:
+ raise ValueError(
+ f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`"
+ )
+ text_embeds = added_cond_kwargs.get("text_embeds")
+ if "time_ids" not in added_cond_kwargs:
+ raise ValueError(
+ f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`"
+ )
+ time_ids = added_cond_kwargs.get("time_ids")
+ time_embeds = self.add_time_proj(time_ids.flatten())
+ time_embeds = time_embeds.reshape((text_embeds.shape[0], -1))
+ add_embeds = torch.concat([text_embeds, time_embeds], dim=-1)
+ add_embeds = add_embeds.to(emb.dtype)
+ aug_emb = self.add_embedding(add_embeds)
+ elif self.config.addition_embed_type == "image":
+ # Kandinsky 2.2 - style
+ if "image_embeds" not in added_cond_kwargs:
+ raise ValueError(
+ f"{self.__class__} has the config param `addition_embed_type` set to 'image' which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`"
+ )
+ image_embs = added_cond_kwargs.get("image_embeds")
+ aug_emb = self.add_embedding(image_embs)
+ elif self.config.addition_embed_type == "image_hint":
+ # Kandinsky 2.2 - style
+ if (
+ "image_embeds" not in added_cond_kwargs
+ or "hint" not in added_cond_kwargs
+ ):
+ raise ValueError(
+ f"{self.__class__} has the config param `addition_embed_type` set to 'image_hint' which requires the keyword arguments `image_embeds` and `hint` to be passed in `added_cond_kwargs`"
+ )
+ image_embs = added_cond_kwargs.get("image_embeds")
+ hint = added_cond_kwargs.get("hint")
+ aug_emb, hint = self.add_embedding(image_embs, hint)
+ sample = torch.cat([sample, hint], dim=1)
+
+ emb = emb + aug_emb if aug_emb is not None else emb
+
+ if self.time_embed_act is not None:
+ emb = self.time_embed_act(emb)
+
+ if (
+ self.encoder_hid_proj is not None
+ and self.config.encoder_hid_dim_type == "text_proj"
+ ):
+ encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states)
+ elif (
+ self.encoder_hid_proj is not None
+ and self.config.encoder_hid_dim_type == "text_image_proj"
+ ):
+ # Kadinsky 2.1 - style
+ if "image_embeds" not in added_cond_kwargs:
+ raise ValueError(
+ f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'text_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`"
+ )
+
+ image_embeds = added_cond_kwargs.get("image_embeds")
+ encoder_hidden_states = self.encoder_hid_proj(
+ encoder_hidden_states, image_embeds
+ )
+ elif (
+ self.encoder_hid_proj is not None
+ and self.config.encoder_hid_dim_type == "image_proj"
+ ):
+ # Kandinsky 2.2 - style
+ if "image_embeds" not in added_cond_kwargs:
+ raise ValueError(
+ f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`"
+ )
+ image_embeds = added_cond_kwargs.get("image_embeds")
+ encoder_hidden_states = self.encoder_hid_proj(image_embeds)
+ elif (
+ self.encoder_hid_proj is not None
+ and self.config.encoder_hid_dim_type == "ip_image_proj"
+ ):
+ if "image_embeds" not in added_cond_kwargs:
+ raise ValueError(
+ f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'ip_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`"
+ )
+ image_embeds = added_cond_kwargs.get("image_embeds")
+ image_embeds = self.encoder_hid_proj(image_embeds).to(
+ encoder_hidden_states.dtype
+ )
+ encoder_hidden_states = torch.cat(
+ [encoder_hidden_states, image_embeds], dim=1
+ )
+
+ # 2. pre-process
+ sample = self.conv_in(sample)
+
+ # 2.5 GLIGEN position net
+ if (
+ cross_attention_kwargs is not None
+ and cross_attention_kwargs.get("gligen", None) is not None
+ ):
+ cross_attention_kwargs = cross_attention_kwargs.copy()
+ gligen_args = cross_attention_kwargs.pop("gligen")
+ cross_attention_kwargs["gligen"] = {
+ "objs": self.position_net(**gligen_args)
+ }
+
+ # 3. down
+ lora_scale = (
+ cross_attention_kwargs.get("scale", 1.0)
+ if cross_attention_kwargs is not None
+ else 1.0
+ )
+ if USE_PEFT_BACKEND:
+ # weight the lora layers by setting `lora_scale` for each PEFT layer
+ scale_lora_layers(self, lora_scale)
+
+ is_controlnet = (
+ mid_block_additional_residual is not None
+ and down_block_additional_residuals is not None
+ )
+ # using new arg down_intrablock_additional_residuals for T2I-Adapters, to distinguish from controlnets
+ is_adapter = down_intrablock_additional_residuals is not None
+ # maintain backward compatibility for legacy usage, where
+ # T2I-Adapter and ControlNet both use down_block_additional_residuals arg
+ # but can only use one or the other
+ if (
+ not is_adapter
+ and mid_block_additional_residual is None
+ and down_block_additional_residuals is not None
+ ):
+ deprecate(
+ "T2I should not use down_block_additional_residuals",
+ "1.3.0",
+ "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated \
+ and will be removed in diffusers 1.3.0. `down_block_additional_residuals` should only be used \
+ for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ",
+ standard_warn=False,
+ )
+ down_intrablock_additional_residuals = down_block_additional_residuals
+ is_adapter = True
+
+ down_block_res_samples = (sample,)
+ tot_referece_features = ()
+ for downsample_block in self.down_blocks:
+ if (
+ hasattr(downsample_block, "has_cross_attention")
+ and downsample_block.has_cross_attention
+ ):
+ # For t2i-adapter CrossAttnDownBlock2D
+ additional_residuals = {}
+ if is_adapter and len(down_intrablock_additional_residuals) > 0:
+ additional_residuals[
+ "additional_residuals"
+ ] = down_intrablock_additional_residuals.pop(0)
+
+ sample, res_samples = downsample_block(
+ hidden_states=sample,
+ temb=emb,
+ encoder_hidden_states=encoder_hidden_states,
+ attention_mask=attention_mask,
+ cross_attention_kwargs=cross_attention_kwargs,
+ encoder_attention_mask=encoder_attention_mask,
+ **additional_residuals,
+ )
+ else:
+ sample, res_samples = downsample_block(
+ hidden_states=sample, temb=emb, scale=lora_scale
+ )
+ if is_adapter and len(down_intrablock_additional_residuals) > 0:
+ sample += down_intrablock_additional_residuals.pop(0)
+
+ down_block_res_samples += res_samples
+
+ if only_return_middle_fea:
+ return sample
+
+ if is_controlnet:
+ new_down_block_res_samples = ()
+
+ for down_block_res_sample, down_block_additional_residual in zip(
+ down_block_res_samples, down_block_additional_residuals
+ ):
+ down_block_res_sample = (
+ down_block_res_sample + down_block_additional_residual
+ )
+ new_down_block_res_samples = new_down_block_res_samples + (
+ down_block_res_sample,
+ )
+
+ down_block_res_samples = new_down_block_res_samples
+
+ # 4. mid
+ if self.mid_block is not None:
+ if (
+ hasattr(self.mid_block, "has_cross_attention")
+ and self.mid_block.has_cross_attention
+ ):
+ sample = self.mid_block(
+ sample,
+ emb,
+ encoder_hidden_states=encoder_hidden_states,
+ attention_mask=attention_mask,
+ cross_attention_kwargs=cross_attention_kwargs,
+ encoder_attention_mask=encoder_attention_mask,
+ )
+ else:
+ sample = self.mid_block(sample, emb)
+
+ # To support T2I-Adapter-XL
+ if (
+ is_adapter
+ and len(down_intrablock_additional_residuals) > 0
+ and sample.shape == down_intrablock_additional_residuals[0].shape
+ ):
+ sample += down_intrablock_additional_residuals.pop(0)
+
+ if is_controlnet:
+ sample = sample + mid_block_additional_residual
+
+ # 5. up
+ for i, upsample_block in enumerate(self.up_blocks):
+ is_final_block = i == len(self.up_blocks) - 1
+
+ res_samples = down_block_res_samples[-len(upsample_block.resnets) :]
+ down_block_res_samples = down_block_res_samples[
+ : -len(upsample_block.resnets)
+ ]
+
+ # if we have not reached the final block and need to forward the
+ # upsample size, we do it here
+ if not is_final_block and forward_upsample_size:
+ upsample_size = down_block_res_samples[-1].shape[2:]
+
+ if (
+ hasattr(upsample_block, "has_cross_attention")
+ and upsample_block.has_cross_attention
+ ):
+ sample = upsample_block(
+ hidden_states=sample,
+ temb=emb,
+ res_hidden_states_tuple=res_samples,
+ encoder_hidden_states=encoder_hidden_states,
+ cross_attention_kwargs=cross_attention_kwargs,
+ upsample_size=upsample_size,
+ attention_mask=attention_mask,
+ encoder_attention_mask=encoder_attention_mask,
+ )
+ else:
+ sample = upsample_block(
+ hidden_states=sample,
+ temb=emb,
+ res_hidden_states_tuple=res_samples,
+ upsample_size=upsample_size,
+ scale=lora_scale,
+ )
+
+ # 6. post-process
+ # if self.conv_norm_out:
+ # sample = self.conv_norm_out(sample)
+ # sample = self.conv_act(sample)
+ # sample = self.conv_out(sample)
+
+ if USE_PEFT_BACKEND:
+ # remove `lora_scale` from each PEFT layer
+ unscale_lora_layers(self, lora_scale)
+
+ if not return_dict:
+ return (sample,)
+
+ return UNet2DConditionOutput(sample=sample)
diff --git a/src/models/unet_2d_decoder.py b/src/models/unet_2d_decoder.py
new file mode 100644
index 0000000..e0a1933
--- /dev/null
+++ b/src/models/unet_2d_decoder.py
@@ -0,0 +1,1011 @@
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.loaders import FromOriginalControlNetMixin
+from diffusers.utils import BaseOutput, logging
+from diffusers.models.embeddings import (
+ TextImageProjection,
+ TextImageTimeEmbedding,
+ TextTimeEmbedding,
+ TimestepEmbedding,
+ Timesteps,
+)
+from diffusers.models.modeling_utils import ModelMixin
+from diffusers.models.unet_2d_blocks import (
+ CrossAttnDownBlock2D,
+ DownBlock2D,
+ UNetMidBlock2DCrossAttn,
+ get_down_block,
+)
+from diffusers.models.unet_2d_condition import UNet2DConditionModel
+from diffusers.models.controlnet import (
+ ControlNetConditioningEmbedding,
+ ControlNetOutput,
+ ControlNetModel,
+)
+
+logger = logging.get_logger(__name__) # pylint: disable=invalid-name
+
+
+class UNetDec_ControlNetModel(ModelMixin, ConfigMixin, FromOriginalControlNetMixin):
+ """
+ A ControlNet model.
+
+ Args:
+ in_channels (`int`, defaults to 4):
+ The number of channels in the input sample.
+ flip_sin_to_cos (`bool`, defaults to `True`):
+ Whether to flip the sin to cos in the time embedding.
+ freq_shift (`int`, defaults to 0):
+ The frequency shift to apply to the time embedding.
+ down_block_types (`tuple[str]`, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`):
+ The tuple of downsample blocks to use.
+ only_cross_attention (`Union[bool, Tuple[bool]]`, defaults to `False`):
+ block_out_channels (`tuple[int]`, defaults to `(320, 640, 1280, 1280)`):
+ The tuple of output channels for each block.
+ layers_per_block (`int`, defaults to 2):
+ The number of layers per block.
+ downsample_padding (`int`, defaults to 1):
+ The padding to use for the downsampling convolution.
+ mid_block_scale_factor (`float`, defaults to 1):
+ The scale factor to use for the mid block.
+ act_fn (`str`, defaults to "silu"):
+ The activation function to use.
+ norm_num_groups (`int`, *optional*, defaults to 32):
+ The number of groups to use for the normalization. If None, normalization and activation layers is skipped
+ in post-processing.
+ norm_eps (`float`, defaults to 1e-5):
+ The epsilon to use for the normalization.
+ cross_attention_dim (`int`, defaults to 1280):
+ The dimension of the cross attention features.
+ transformer_layers_per_block (`int` or `Tuple[int]`, *optional*, defaults to 1):
+ The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for
+ [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`],
+ [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`].
+ encoder_hid_dim (`int`, *optional*, defaults to None):
+ If `encoder_hid_dim_type` is defined, `encoder_hidden_states` will be projected from `encoder_hid_dim`
+ dimension to `cross_attention_dim`.
+ encoder_hid_dim_type (`str`, *optional*, defaults to `None`):
+ If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text
+ embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`.
+ attention_head_dim (`Union[int, Tuple[int]]`, defaults to 8):
+ The dimension of the attention heads.
+ use_linear_projection (`bool`, defaults to `False`):
+ class_embed_type (`str`, *optional*, defaults to `None`):
+ The type of class embedding to use which is ultimately summed with the time embeddings. Choose from None,
+ `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`.
+ addition_embed_type (`str`, *optional*, defaults to `None`):
+ Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or
+ "text". "text" will use the `TextTimeEmbedding` layer.
+ num_class_embeds (`int`, *optional*, defaults to 0):
+ Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing
+ class conditioning with `class_embed_type` equal to `None`.
+ upcast_attention (`bool`, defaults to `False`):
+ resnet_time_scale_shift (`str`, defaults to `"default"`):
+ Time scale shift config for ResNet blocks (see `ResnetBlock2D`). Choose from `default` or `scale_shift`.
+ projection_class_embeddings_input_dim (`int`, *optional*, defaults to `None`):
+ The dimension of the `class_labels` input when `class_embed_type="projection"`. Required when
+ `class_embed_type="projection"`.
+ controlnet_conditioning_channel_order (`str`, defaults to `"rgb"`):
+ The channel order of conditional image. Will convert to `rgb` if it's `bgr`.
+ conditioning_embedding_out_channels (`tuple[int]`, *optional*, defaults to `(16, 32, 96, 256)`):
+ The tuple of output channel for each block in the `conditioning_embedding` layer.
+ global_pool_conditions (`bool`, defaults to `False`):
+ """
+
+ _supports_gradient_checkpointing = True
+
+ @register_to_config
+ def __init__(
+ self,
+ in_channels: int = 4,
+ conditioning_channels: int = 3,
+ flip_sin_to_cos: bool = True,
+ freq_shift: int = 0,
+ down_block_types: Tuple[str] = (
+ "CrossAttnDownBlock2D",
+ "CrossAttnDownBlock2D",
+ "CrossAttnDownBlock2D",
+ "DownBlock2D",
+ ),
+ up_block_types: Tuple[str] = (
+ "UpBlock2D",
+ "CrossAttnUpBlock2D",
+ "CrossAttnUpBlock2D",
+ "CrossAttnUpBlock2D",
+ ),
+ only_cross_attention: Union[bool, Tuple[bool]] = False,
+ block_out_channels: Tuple[int] = (320, 640, 1280, 1280),
+ layers_per_block: int = 2,
+ downsample_padding: int = 1,
+ mid_block_scale_factor: float = 1,
+ act_fn: str = "silu",
+ norm_num_groups: Optional[int] = 32,
+ norm_eps: float = 1e-5,
+ cross_attention_dim: int = 1280,
+ transformer_layers_per_block: Union[int, Tuple[int]] = 1,
+ encoder_hid_dim: Optional[int] = None,
+ encoder_hid_dim_type: Optional[str] = None,
+ attention_head_dim: Union[int, Tuple[int]] = 8,
+ num_attention_heads: Optional[Union[int, Tuple[int]]] = None,
+ use_linear_projection: bool = False,
+ class_embed_type: Optional[str] = None,
+ addition_embed_type: Optional[str] = None,
+ addition_time_embed_dim: Optional[int] = None,
+ num_class_embeds: Optional[int] = None,
+ upcast_attention: bool = False,
+ resnet_time_scale_shift: str = "default",
+ projection_class_embeddings_input_dim: Optional[int] = None,
+ controlnet_conditioning_channel_order: str = "rgb",
+ conditioning_embedding_out_channels: Optional[Tuple[int]] = (16, 32, 96, 256),
+ global_pool_conditions: bool = False,
+ addition_embed_type_num_heads=64,
+ ):
+ super().__init__()
+
+ # If `num_attention_heads` is not defined (which is the case for most models)
+ # it will default to `attention_head_dim`. This looks weird upon first reading it and it is.
+ # The reason for this behavior is to correct for incorrectly named variables that were introduced
+ # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131
+ # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking
+ # which is why we correct for the naming here.
+ num_attention_heads = num_attention_heads or attention_head_dim
+
+ # Check inputs
+ if len(block_out_channels) != len(down_block_types):
+ raise ValueError(
+ f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}."
+ )
+
+ if not isinstance(only_cross_attention, bool) and len(
+ only_cross_attention
+ ) != len(down_block_types):
+ raise ValueError(
+ f"Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}."
+ )
+
+ if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(down_block_types):
+ raise ValueError(
+ f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}."
+ )
+
+ if isinstance(transformer_layers_per_block, int):
+ transformer_layers_per_block = [transformer_layers_per_block] * len(down_block_types)
+
+ # # input
+ # conv_in_kernel = 3
+ # conv_in_padding = (conv_in_kernel - 1) // 2
+ # self.conv_in = nn.Conv2d(
+ # in_channels,
+ # block_out_channels[0],
+ # kernel_size=conv_in_kernel,
+ # padding=conv_in_padding,
+ # )
+ self.conv_in = None
+
+ # time
+ time_embed_dim = block_out_channels[0] * 4
+ self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift)
+ timestep_input_dim = block_out_channels[0]
+ self.time_embedding = TimestepEmbedding(
+ timestep_input_dim,
+ time_embed_dim,
+ act_fn=act_fn,
+ )
+
+ if encoder_hid_dim_type is None and encoder_hid_dim is not None:
+ encoder_hid_dim_type = "text_proj"
+ self.register_to_config(encoder_hid_dim_type=encoder_hid_dim_type)
+ logger.info(
+ "encoder_hid_dim_type defaults to 'text_proj' as `encoder_hid_dim` is defined."
+ )
+
+ if encoder_hid_dim is None and encoder_hid_dim_type is not None:
+ raise ValueError(
+ f"`encoder_hid_dim` has to be defined when `encoder_hid_dim_type` is set to {encoder_hid_dim_type}."
+ )
+
+ if encoder_hid_dim_type == "text_proj":
+ self.encoder_hid_proj = nn.Linear(encoder_hid_dim, cross_attention_dim)
+ elif encoder_hid_dim_type == "text_image_proj":
+ # image_embed_dim DOESN'T have to be `cross_attention_dim`. To not clutter the __init__ too much
+ # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use
+ # case when `addition_embed_type == "text_image_proj"` (Kadinsky 2.1)`
+ self.encoder_hid_proj = TextImageProjection(
+ text_embed_dim=encoder_hid_dim,
+ image_embed_dim=cross_attention_dim,
+ cross_attention_dim=cross_attention_dim,
+ )
+
+ elif encoder_hid_dim_type is not None:
+ raise ValueError(
+ f"encoder_hid_dim_type: {encoder_hid_dim_type} must be None, 'text_proj' or 'text_image_proj'."
+ )
+ else:
+ self.encoder_hid_proj = None
+
+ # class embedding
+ if class_embed_type is None and num_class_embeds is not None:
+ self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim)
+ elif class_embed_type == "timestep":
+ self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim)
+ elif class_embed_type == "identity":
+ self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim)
+ elif class_embed_type == "projection":
+ if projection_class_embeddings_input_dim is None:
+ raise ValueError(
+ "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set"
+ )
+ # The projection `class_embed_type` is the same as the timestep `class_embed_type` except
+ # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings
+ # 2. it projects from an arbitrary input dimension.
+ #
+ # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations.
+ # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings.
+ # As a result, `TimestepEmbedding` can be passed arbitrary vectors.
+ self.class_embedding = TimestepEmbedding(
+ projection_class_embeddings_input_dim, time_embed_dim
+ )
+ else:
+ self.class_embedding = None
+
+ if addition_embed_type == "text":
+ if encoder_hid_dim is not None:
+ text_time_embedding_from_dim = encoder_hid_dim
+ else:
+ text_time_embedding_from_dim = cross_attention_dim
+
+ self.add_embedding = TextTimeEmbedding(
+ text_time_embedding_from_dim,
+ time_embed_dim,
+ num_heads=addition_embed_type_num_heads,
+ )
+ elif addition_embed_type == "text_image":
+ # text_embed_dim and image_embed_dim DON'T have to be `cross_attention_dim`. To not clutter the __init__ too much
+ # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use
+ # case when `addition_embed_type == "text_image"` (Kadinsky 2.1)`
+ self.add_embedding = TextImageTimeEmbedding(
+ text_embed_dim=cross_attention_dim,
+ image_embed_dim=cross_attention_dim,
+ time_embed_dim=time_embed_dim,
+ )
+ elif addition_embed_type == "text_time":
+ self.add_time_proj = Timesteps(addition_time_embed_dim, flip_sin_to_cos, freq_shift)
+ self.add_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim)
+ elif addition_embed_type is not None:
+ raise ValueError(
+ f"addition_embed_type: {addition_embed_type} must be None, 'text' or 'text_image'."
+ )
+
+ # control net conditioning embedding
+ self.controlnet_cond_embedding = ControlNetConditioningEmbedding(
+ conditioning_embedding_channels=block_out_channels[0],
+ block_out_channels=conditioning_embedding_out_channels,
+ conditioning_channels=conditioning_channels,
+ )
+
+ self.down_blocks = nn.ModuleList([])
+
+ if isinstance(only_cross_attention, bool):
+ only_cross_attention = [only_cross_attention] * len(down_block_types)
+
+ if isinstance(attention_head_dim, int):
+ attention_head_dim = (attention_head_dim,) * len(down_block_types)
+
+ if isinstance(num_attention_heads, int):
+ num_attention_heads = (num_attention_heads,) * len(down_block_types)
+
+ # # down
+ # output_channel = block_out_channels[0]
+
+ # controlnet_block = nn.Conv2d(output_channel, output_channel, kernel_size=1)
+ # controlnet_block = zero_module(controlnet_block)
+ # self.controlnet_down_blocks.append(controlnet_block)
+
+ # for i, down_block_type in enumerate(down_block_types):
+ # input_channel = output_channel
+ # output_channel = block_out_channels[i]
+ # is_final_block = i == len(block_out_channels) - 1
+
+ # for _ in range(layers_per_block):
+ # controlnet_block = nn.Conv2d(output_channel, output_channel, kernel_size=1)
+ # controlnet_block = zero_module(controlnet_block)
+ # self.controlnet_down_blocks.append(controlnet_block)
+
+ # if not is_final_block:
+ # controlnet_block = nn.Conv2d(output_channel, output_channel, kernel_size=1)
+ # controlnet_block = zero_module(controlnet_block)
+ # self.controlnet_down_blocks.append(controlnet_block)
+
+ # mid
+ mid_block_channel = block_out_channels[-1]
+
+ controlnet_block = nn.Conv2d(mid_block_channel, mid_block_channel, kernel_size=1)
+ controlnet_block = zero_module(controlnet_block)
+ self.controlnet_mid_block = controlnet_block
+
+ self.mid_block = UNetMidBlock2DCrossAttn(
+ transformer_layers_per_block=transformer_layers_per_block[-1],
+ in_channels=mid_block_channel,
+ temb_channels=time_embed_dim,
+ resnet_eps=norm_eps,
+ resnet_act_fn=act_fn,
+ output_scale_factor=mid_block_scale_factor,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ cross_attention_dim=cross_attention_dim,
+ num_attention_heads=num_attention_heads[-1],
+ resnet_groups=norm_num_groups,
+ use_linear_projection=use_linear_projection,
+ upcast_attention=upcast_attention,
+ )
+
+ # up
+ self.controlnet_up_blocks = nn.ModuleList([])
+ self.num_upsamplers = 0
+ self.up_blocks = nn.ModuleList([])
+ reversed_block_out_channels = list(reversed(block_out_channels))
+ reversed_num_attention_heads = list(reversed(num_attention_heads))
+ reversed_transformer_layers_per_block = list(reversed(transformer_layers_per_block))
+ only_cross_attention = list(reversed(only_cross_attention))
+
+ output_channel = reversed_block_out_channels[0]
+ for i, up_block_type in enumerate(up_block_types):
+ is_final_block = i == len(block_out_channels) - 1
+
+ prev_output_channel = output_channel
+ output_channel = reversed_block_out_channels[i]
+ input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)]
+
+ # add upsample block for all BUT final layer
+ if not is_final_block:
+ add_upsample = True
+ self.num_upsamplers += 1
+ else:
+ add_upsample = False
+
+ up_block = get_up_block(
+ up_block_type,
+ num_layers=layers_per_block + 1,
+ transformer_layers_per_block=reversed_transformer_layers_per_block[i],
+ in_channels=input_channel,
+ out_channels=output_channel,
+ prev_output_channel=prev_output_channel,
+ temb_channels=time_embed_dim,
+ add_upsample=add_upsample,
+ resnet_eps=norm_eps,
+ resnet_act_fn=act_fn,
+ resnet_groups=norm_num_groups,
+ cross_attention_dim=cross_attention_dim,
+ num_attention_heads=reversed_num_attention_heads[i],
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention[i],
+ upcast_attention=upcast_attention,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ attention_head_dim=(
+ attention_head_dim[i]
+ if attention_head_dim[i] is not None
+ else output_channel
+ ),
+ )
+ self.up_blocks.append(up_block)
+ prev_output_channel = output_channel
+
+ # if i>0: # 因为我们只输出transformer相关的,而第一级的upblock是纯conv
+ for _ in range(layers_per_block + 1):
+ controlnet_block = nn.Conv2d(output_channel, output_channel, kernel_size=1)
+ controlnet_block = zero_module(controlnet_block)
+ self.controlnet_up_blocks.append(controlnet_block)
+
+ @classmethod
+ def from_unet(
+ cls,
+ unet: UNet2DConditionModel,
+ controlnet_conditioning_channel_order: str = "rgb",
+ conditioning_embedding_out_channels: Optional[Tuple[int]] = (16, 32, 96, 256),
+ load_weights_from_unet: bool = True,
+ ):
+ r"""
+ Instantiate a [`ControlNetModel`] from [`UNet2DConditionModel`].
+
+ Parameters:
+ unet (`UNet2DConditionModel`):
+ The UNet model weights to copy to the [`ControlNetModel`]. All configuration options are also copied
+ where applicable.
+ """
+ transformer_layers_per_block = (
+ unet.config.transformer_layers_per_block
+ if "transformer_layers_per_block" in unet.config
+ else 1
+ )
+ encoder_hid_dim = (
+ unet.config.encoder_hid_dim if "encoder_hid_dim" in unet.config else None
+ )
+ encoder_hid_dim_type = (
+ unet.config.encoder_hid_dim_type
+ if "encoder_hid_dim_type" in unet.config
+ else None
+ )
+ addition_embed_type = (
+ unet.config.addition_embed_type
+ if "addition_embed_type" in unet.config
+ else None
+ )
+ addition_time_embed_dim = (
+ unet.config.addition_time_embed_dim
+ if "addition_time_embed_dim" in unet.config
+ else None
+ )
+
+ controlnet = cls(
+ encoder_hid_dim=encoder_hid_dim,
+ encoder_hid_dim_type=encoder_hid_dim_type,
+ addition_embed_type=addition_embed_type,
+ addition_time_embed_dim=addition_time_embed_dim,
+ transformer_layers_per_block=transformer_layers_per_block,
+ in_channels=unet.config.in_channels,
+ flip_sin_to_cos=unet.config.flip_sin_to_cos,
+ freq_shift=unet.config.freq_shift,
+ down_block_types=unet.config.down_block_types,
+ only_cross_attention=unet.config.only_cross_attention,
+ block_out_channels=unet.config.block_out_channels,
+ layers_per_block=unet.config.layers_per_block,
+ downsample_padding=unet.config.downsample_padding,
+ mid_block_scale_factor=unet.config.mid_block_scale_factor,
+ act_fn=unet.config.act_fn,
+ norm_num_groups=unet.config.norm_num_groups,
+ norm_eps=unet.config.norm_eps,
+ cross_attention_dim=unet.config.cross_attention_dim,
+ attention_head_dim=unet.config.attention_head_dim,
+ num_attention_heads=unet.config.num_attention_heads,
+ use_linear_projection=unet.config.use_linear_projection,
+ class_embed_type=unet.config.class_embed_type,
+ num_class_embeds=unet.config.num_class_embeds,
+ upcast_attention=unet.config.upcast_attention,
+ resnet_time_scale_shift=unet.config.resnet_time_scale_shift,
+ projection_class_embeddings_input_dim=unet.config.projection_class_embeddings_input_dim,
+ controlnet_conditioning_channel_order=controlnet_conditioning_channel_order,
+ conditioning_embedding_out_channels=conditioning_embedding_out_channels,
+ ).requires_grad_(False)
+
+ if load_weights_from_unet:
+ # controlnet.conv_in.load_state_dict(unet.conv_in.state_dict())
+ controlnet.time_proj.load_state_dict(unet.time_proj.state_dict())
+ controlnet.time_embedding.load_state_dict(unet.time_embedding.state_dict())
+
+ if controlnet.class_embedding:
+ controlnet.class_embedding.load_state_dict(unet.class_embedding.state_dict())
+
+ # controlnet.down_blocks.load_state_dict(unet.down_blocks.state_dict())
+ controlnet.mid_block.load_state_dict(unet.mid_block.state_dict())
+ # controlnet.up_blocks.load_state_dict(unet.up_blocks.state_dict(), strict=False) # TODO:将各个upblock第一层融合层使用weight进行部分初始化
+
+ src_tensors = dict(named_params_and_buffers(unet.up_blocks))
+ for name, tensor in named_params_and_buffers(controlnet.up_blocks):
+ assert (name in src_tensors), name
+ try:
+ # print('Successfully initializing ControlNet!', name, tensor.shape, src_tensors[name].shape)
+ tensor.copy_(src_tensors[name].detach())
+ except:
+ # print('Mismatch occured in initializing ControlNet!', name, tensor.shape, src_tensors[name].shape)
+ # TODO: 确保所有upblock参数有初始化
+ if tensor.dim() == 1:
+ tensor.copy_(src_tensors[name].detach()[:tensor.shape[0]])
+ else:
+ tensor.copy_(src_tensors[name].detach()[:, :tensor.shape[1]])
+
+ return controlnet
+
+ def _set_gradient_checkpointing(self, module, value=False):
+ if hasattr(module, "gradient_checkpointing"):
+ module.gradient_checkpointing = value
+
+ def forward(
+ self,
+ sample: torch.FloatTensor,
+ timestep: Union[torch.Tensor, float, int],
+ encoder_hidden_states: torch.Tensor,
+ # controlnet_cond: torch.FloatTensor,
+ conditioning_scale: float = 1.0,
+ class_labels: Optional[torch.Tensor] = None,
+ timestep_cond: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,
+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,
+ guess_mode: bool = False,
+ return_dict: bool = True,
+ only_return_transformer_layers_out: bool = False,
+ ) -> Union[ControlNetOutput, Tuple]:
+ """
+ The [`ControlNetModel`] forward method.
+
+ Args:
+ sample (`torch.FloatTensor`):
+ The noisy input tensor.
+ timestep (`Union[torch.Tensor, float, int]`):
+ The number of timesteps to denoise an input.
+ encoder_hidden_states (`torch.Tensor`):
+ The encoder hidden states.
+ controlnet_cond (`torch.FloatTensor`):
+ The conditional input tensor of shape `(batch_size, sequence_length, hidden_size)`.
+ conditioning_scale (`float`, defaults to `1.0`):
+ The scale factor for ControlNet outputs.
+ class_labels (`torch.Tensor`, *optional*, defaults to `None`):
+ Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings.
+ timestep_cond (`torch.Tensor`, *optional*, defaults to `None`):
+ attention_mask (`torch.Tensor`, *optional*, defaults to `None`):
+ added_cond_kwargs (`dict`):
+ Additional conditions for the Stable Diffusion XL UNet.
+ cross_attention_kwargs (`dict[str]`, *optional*, defaults to `None`):
+ A kwargs dictionary that if specified is passed along to the `AttnProcessor`.
+ guess_mode (`bool`, defaults to `False`):
+ In this mode, the ControlNet encoder tries its best to recognize the input content of the input even if
+ you remove all prompts. A `guidance_scale` between 3.0 and 5.0 is recommended.
+ return_dict (`bool`, defaults to `True`):
+ Whether or not to return a [`~models.controlnet.ControlNetOutput`] instead of a plain tuple.
+
+ Returns:
+ [`~models.controlnet.ControlNetOutput`] **or** `tuple`:
+ If `return_dict` is `True`, a [`~models.controlnet.ControlNetOutput`] is returned, otherwise a tuple is
+ returned where the first element is the sample tensor.
+ """
+
+ # prepare attention_mask
+ if attention_mask is not None:
+ attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0
+ attention_mask = attention_mask.unsqueeze(1)
+
+ # 1. time
+ timesteps = timestep
+ if not torch.is_tensor(timesteps):
+ # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can
+ # This would be a good case for the `match` statement (Python 3.10+)
+ is_mps = sample.device.type == "mps"
+ if isinstance(timestep, float):
+ dtype = torch.float32 if is_mps else torch.float64
+ else:
+ dtype = torch.int32 if is_mps else torch.int64
+ timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)
+ elif len(timesteps.shape) == 0:
+ timesteps = timesteps[None].to(sample.device)
+
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
+ timesteps = timesteps.expand(sample.shape[0])
+
+ t_emb = self.time_proj(timesteps)
+
+ # timesteps does not contain any weights and will always return f32 tensors
+ # but time_embedding might actually be running in fp16. so we need to cast here.
+ # there might be better ways to encapsulate this.
+ t_emb = t_emb.to(dtype=sample.dtype)
+
+ emb = self.time_embedding(t_emb, timestep_cond)
+ aug_emb = None
+
+ if self.class_embedding is not None:
+ if class_labels is None:
+ raise ValueError(
+ "class_labels should be provided when num_class_embeds > 0"
+ )
+
+ if self.config.class_embed_type == "timestep":
+ class_labels = self.time_proj(class_labels)
+
+ class_emb = self.class_embedding(class_labels).to(dtype=self.dtype)
+ emb = emb + class_emb
+
+ if self.config.addition_embed_type is not None:
+ if self.config.addition_embed_type == "text":
+ aug_emb = self.add_embedding(encoder_hidden_states)
+
+ elif self.config.addition_embed_type == "text_time":
+ if "text_embeds" not in added_cond_kwargs:
+ raise ValueError(
+ f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`"
+ )
+ text_embeds = added_cond_kwargs.get("text_embeds")
+ if "time_ids" not in added_cond_kwargs:
+ raise ValueError(
+ f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`"
+ )
+ time_ids = added_cond_kwargs.get("time_ids")
+ time_embeds = self.add_time_proj(time_ids.flatten())
+ time_embeds = time_embeds.reshape((text_embeds.shape[0], -1))
+
+ add_embeds = torch.concat([text_embeds, time_embeds], dim=-1)
+ add_embeds = add_embeds.to(emb.dtype)
+ aug_emb = self.add_embedding(add_embeds)
+
+ emb = emb + aug_emb if aug_emb is not None else emb
+
+ if self.conv_in is not None:
+ sample = self.conv_in(sample)
+
+ # 4. mid
+ if self.mid_block is not None:
+ sample = self.mid_block(
+ sample,
+ emb,
+ encoder_hidden_states=encoder_hidden_states,
+ attention_mask=attention_mask,
+ cross_attention_kwargs=cross_attention_kwargs,
+ )
+ mid_block_res_sample = self.controlnet_mid_block(sample)
+
+ # 5. up
+
+ out_block_res_samples = []
+ for i, upsample_block in enumerate(self.up_blocks):
+ if (hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention):
+ sample, res_samples = upsample_block(
+ hidden_states=sample,
+ temb=emb,
+ # res_hidden_states_tuple=res_samples,
+ encoder_hidden_states=encoder_hidden_states,
+ cross_attention_kwargs=cross_attention_kwargs,
+ attention_mask=attention_mask,
+ )
+ else:
+ sample, res_samples = upsample_block(
+ hidden_states=sample,
+ temb=emb,
+ # res_hidden_states_tuple=res_samples,
+ )
+ # print('Out', i, [res_sample.shape for res_sample in res_samples])
+ # if i in range(1, len(self.up_blocks)-1):
+ # res_samples[-1] = res_samples[-1][:, :sample.shape[1]//2] # 为了适配controlNet输出的shape
+
+ out_block_res_samples += res_samples
+
+ # 5. Control net blocks
+
+ controlnet_up_block_res_samples = ()
+ assert len(out_block_res_samples) == len(self.controlnet_up_blocks), (len(out_block_res_samples), len(self.controlnet_up_blocks))
+ for i, controlnet_block in enumerate(self.controlnet_up_blocks): # zero proj out
+ # if only_return_transformer_layers_out and i < 3: # 第一个upblock是纯conv的
+ # continue
+ up_block_res_sample = controlnet_block(out_block_res_samples[i])
+ controlnet_up_block_res_samples = controlnet_up_block_res_samples + (up_block_res_sample,)
+
+ up_block_res_samples = controlnet_up_block_res_samples
+
+ # 6. scaling
+ if guess_mode and not self.config.global_pool_conditions:
+ raise NotImplementedError
+ scales = torch.logspace(-1, 0, len(down_block_res_samples) + 1, device=sample.device) # 0.1 to 1.0
+
+ scales = scales * conditioning_scale
+ down_block_res_samples = [sample * scale for sample, scale in zip(down_block_res_samples, scales)]
+ mid_block_res_sample = mid_block_res_sample * scales[-1] # last one
+ else:
+ up_block_res_samples = [sample * conditioning_scale for sample in up_block_res_samples]
+ mid_block_res_sample = mid_block_res_sample * conditioning_scale
+
+ if self.config.global_pool_conditions:
+ up_block_res_samples = [torch.mean(sample, dim=(2, 3), keepdim=True) for sample in up_block_res_samples]
+ mid_block_res_sample = torch.mean(mid_block_res_sample, dim=(2, 3), keepdim=True)
+
+ if only_return_transformer_layers_out:
+ down_block_res_samples = [sample for i, sample in enumerate(up_block_res_samples) if not i % 3== 2][::-1]
+ else:
+ down_block_res_samples = list(reversed(up_block_res_samples))
+
+ if not return_dict:
+ return (down_block_res_samples, mid_block_res_sample, up_block_res_samples)
+
+ return ControlNetOutput(
+ down_block_res_samples=down_block_res_samples,
+ mid_block_res_sample=mid_block_res_sample,
+ up_block_res_samples=up_block_res_samples,
+ )
+
+
+def zero_module(module):
+ for p in module.parameters():
+ nn.init.zeros_(p)
+ return module
+
+
+def named_params_and_buffers(module):
+ assert isinstance(module, torch.nn.Module)
+ return list(module.named_parameters()) + list(module.named_buffers())
+
+
+from src.models.unet_2d_blocks import UpBlock2D, CrossAttnUpBlock2D
+from diffusers.models.resnet import ResnetBlock2D
+
+from diffusers.utils import is_torch_version
+class UpBlock2D_woskip(UpBlock2D):
+ def __init__(
+ self,
+ prev_output_channel: int,
+ out_channels: int,
+ temb_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default",
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ resnet_pre_norm: bool = True,
+ output_scale_factor=1.0,
+ add_upsample=True,
+ ):
+ super(UpBlock2D_woskip, self).__init__(
+ in_channels=0,
+ prev_output_channel=prev_output_channel,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ dropout=dropout,
+ num_layers=num_layers,
+ resnet_eps=resnet_eps,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ resnet_act_fn=resnet_act_fn,
+ resnet_groups=resnet_groups,
+ resnet_pre_norm=resnet_pre_norm,
+ output_scale_factor=output_scale_factor,
+ add_upsample=add_upsample,
+ )
+ resnets = []
+
+ for i in range(num_layers):
+ resnet_in_channels = prev_output_channel if i == 0 else out_channels
+
+ resnets.append(
+ ResnetBlock2D(
+ in_channels=resnet_in_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ )
+ )
+
+ self.resnets = nn.ModuleList(resnets)
+
+ def forward(self, hidden_states, temb=None, upsample_size=None, scale: float = 1.0):
+ output_states = []
+ for resnet in self.resnets:
+ if self.training and self.gradient_checkpointing:
+
+ def create_custom_forward(module):
+ def custom_forward(*inputs):
+ return module(*inputs)
+
+ return custom_forward
+
+ if is_torch_version(">=", "1.11.0"):
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet), hidden_states, temb, use_reentrant=False
+ )
+ else:
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet), hidden_states, temb
+ )
+ else:
+ hidden_states = resnet(hidden_states, temb, scale=scale)
+ output_states = output_states + [hidden_states,]
+
+ if self.upsamplers is not None:
+ for upsampler in self.upsamplers:
+ hidden_states = upsampler(hidden_states, upsample_size, scale=scale)
+
+ return hidden_states, output_states
+
+
+class CrossAttnUpBlock2D_woskip(CrossAttnUpBlock2D):
+ def __init__(
+ self,
+ out_channels: int,
+ prev_output_channel: int,
+ temb_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ transformer_layers_per_block: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default",
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ resnet_pre_norm: bool = True,
+ num_attention_heads=1,
+ cross_attention_dim=1280,
+ output_scale_factor=1.0,
+ add_upsample=True,
+ dual_cross_attention=False,
+ use_linear_projection=False,
+ only_cross_attention=False,
+ upcast_attention=False,
+ attention_type="default",
+ ):
+ super(CrossAttnUpBlock2D_woskip, self).__init__(
+ in_channels=0,
+ out_channels=out_channels,
+ prev_output_channel=prev_output_channel,
+ temb_channels=temb_channels,
+ dropout=dropout,
+ num_layers=num_layers,
+ transformer_layers_per_block=transformer_layers_per_block,
+ resnet_eps=resnet_eps,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ resnet_act_fn=resnet_act_fn,
+ resnet_groups=resnet_groups,
+ resnet_pre_norm=resnet_pre_norm,
+ num_attention_heads=num_attention_heads,
+ cross_attention_dim=cross_attention_dim,
+ output_scale_factor=output_scale_factor,
+ add_upsample=add_upsample,
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention,
+ upcast_attention=upcast_attention,
+ attention_type=attention_type,
+ )
+ resnets = []
+
+ for i in range(num_layers):
+ resnet_in_channels = prev_output_channel if i == 0 else out_channels
+ resnets.append(
+ ResnetBlock2D(
+ in_channels=resnet_in_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ )
+ )
+ self.resnets = nn.ModuleList(resnets)
+
+ def forward(
+ self,
+ hidden_states: torch.FloatTensor,
+ temb: Optional[torch.FloatTensor] = None,
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,
+ upsample_size: Optional[int] = None,
+ attention_mask: Optional[torch.FloatTensor] = None,
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
+ ):
+ lora_scale = (
+ cross_attention_kwargs.get("scale", 1.0)
+ if cross_attention_kwargs is not None
+ else 1.0
+ )
+ output_states = []
+ for resnet, attn in zip(self.resnets, self.attentions):
+
+ if self.training and self.gradient_checkpointing:
+
+ def create_custom_forward(module, return_dict=None):
+ def custom_forward(*inputs):
+ if return_dict is not None:
+ return module(*inputs, return_dict=return_dict)
+ else:
+ return module(*inputs)
+
+ return custom_forward
+
+ ckpt_kwargs: Dict[str, Any] = (
+ {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {}
+ )
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet),
+ hidden_states,
+ temb,
+ **ckpt_kwargs,
+ )
+ hidden_states = attn(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ cross_attention_kwargs=cross_attention_kwargs,
+ attention_mask=attention_mask,
+ encoder_attention_mask=encoder_attention_mask,
+ return_dict=False,
+ )[0]
+ else:
+ hidden_states = resnet(hidden_states, temb, scale=lora_scale)
+ hidden_states = attn(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ cross_attention_kwargs=cross_attention_kwargs,
+ attention_mask=attention_mask,
+ encoder_attention_mask=encoder_attention_mask,
+ return_dict=False,
+ )[0]
+ output_states = output_states + [hidden_states,]
+ if self.upsamplers is not None:
+ for upsampler in self.upsamplers:
+ hidden_states = upsampler(hidden_states, upsample_size, scale=lora_scale)
+
+ return hidden_states, output_states
+
+
+def get_up_block(
+ up_block_type,
+ num_layers,
+ in_channels,
+ out_channels,
+ prev_output_channel,
+ temb_channels,
+ add_upsample,
+ resnet_eps,
+ resnet_act_fn,
+ transformer_layers_per_block=1,
+ num_attention_heads=None,
+ resnet_groups=None,
+ cross_attention_dim=None,
+ dual_cross_attention=False,
+ use_linear_projection=False,
+ only_cross_attention=False,
+ upcast_attention=False,
+ resnet_time_scale_shift="default",
+ attention_type="default",
+ resnet_skip_time_act=False,
+ resnet_out_scale_factor=1.0,
+ cross_attention_norm=None,
+ attention_head_dim=None,
+ upsample_type=None,
+ dropout=0.0,
+):
+ # If attn head dim is not defined, we default it to the number of heads
+ if attention_head_dim is None:
+ logger.warn(
+ f"It is recommended to provide `attention_head_dim` when calling `get_up_block`. Defaulting `attention_head_dim` to {num_attention_heads}."
+ )
+ attention_head_dim = num_attention_heads
+
+ up_block_type = up_block_type[7:] if up_block_type.startswith("UNetRes") else up_block_type
+
+ if up_block_type == "UpBlock2D":
+ return UpBlock2D_woskip(
+ num_layers=num_layers,
+ # in_channels=in_channels,
+ out_channels=out_channels,
+ prev_output_channel=prev_output_channel,
+ temb_channels=temb_channels,
+ dropout=dropout,
+ add_upsample=add_upsample,
+ resnet_eps=resnet_eps,
+ resnet_act_fn=resnet_act_fn,
+ resnet_groups=resnet_groups,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ )
+ elif up_block_type == "CrossAttnUpBlock2D":
+ if cross_attention_dim is None:
+ raise ValueError("cross_attention_dim must be specified for CrossAttnUpBlock2D")
+ return CrossAttnUpBlock2D_woskip(
+ num_layers=num_layers,
+ transformer_layers_per_block=transformer_layers_per_block,
+ # in_channels=in_channels,
+ out_channels=out_channels,
+ prev_output_channel=prev_output_channel,
+ temb_channels=temb_channels,
+ dropout=dropout,
+ add_upsample=add_upsample,
+ resnet_eps=resnet_eps,
+ resnet_act_fn=resnet_act_fn,
+ resnet_groups=resnet_groups,
+ cross_attention_dim=cross_attention_dim,
+ num_attention_heads=num_attention_heads,
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention,
+ upcast_attention=upcast_attention,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ attention_type=attention_type,
+ )
+
+ raise ValueError(f"{up_block_type} does not exist.")
diff --git a/src/models/unet_3d.py b/src/models/unet_3d.py
new file mode 100644
index 0000000..cec8390
--- /dev/null
+++ b/src/models/unet_3d.py
@@ -0,0 +1,716 @@
+from collections import OrderedDict
+from dataclasses import dataclass
+import pdb
+from os import PathLike
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.utils.checkpoint
+import torch.nn.functional as F
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.models.attention_processor import AttentionProcessor
+from diffusers.models.embeddings import TimestepEmbedding, Timesteps
+from diffusers.models.modeling_utils import ModelMixin
+from diffusers.utils import SAFETENSORS_WEIGHTS_NAME, WEIGHTS_NAME, BaseOutput, logging
+from safetensors.torch import load_file
+
+from .resnet import InflatedConv3d, InflatedGroupNorm
+from .unet_3d_blocks import UNetMidBlock3DCrossAttn, get_down_block, get_up_block
+
+logger = logging.get_logger(__name__) # pylint: disable=invalid-name
+
+
+@dataclass
+class UNet3DConditionOutput(BaseOutput):
+ sample: torch.FloatTensor
+
+
+class UNet3DConditionModel(ModelMixin, ConfigMixin):
+ _supports_gradient_checkpointing = True
+
+ @register_to_config
+ def __init__(
+ self,
+ sample_size: Optional[int] = None,
+ in_channels: int = 4,
+ out_channels: int = 4,
+ center_input_sample: bool = False,
+ flip_sin_to_cos: bool = True,
+ freq_shift: int = 0,
+ down_block_types: Tuple[str] = (
+ "CrossAttnDownBlock3D",
+ "CrossAttnDownBlock3D",
+ "CrossAttnDownBlock3D",
+ "DownBlock3D",
+ ),
+ mid_block_type: str = "UNetMidBlock3DCrossAttn",
+ up_block_types: Tuple[str] = (
+ "UpBlock3D",
+ "CrossAttnUpBlock3D",
+ "CrossAttnUpBlock3D",
+ "CrossAttnUpBlock3D",
+ ),
+ only_cross_attention: Union[bool, Tuple[bool]] = False,
+ block_out_channels: Tuple[int] = (320, 640, 1280, 1280),
+ layers_per_block: int = 2,
+ downsample_padding: int = 1,
+ mid_block_scale_factor: float = 1,
+ act_fn: str = "silu",
+ norm_num_groups: int = 32,
+ norm_eps: float = 1e-5,
+ cross_attention_dim: int = 1280,
+ attention_head_dim: Union[int, Tuple[int]] = 8,
+ dual_cross_attention: bool = False,
+ use_linear_projection: bool = False,
+ class_embed_type: Optional[str] = None,
+ num_class_embeds: Optional[int] = None,
+ upcast_attention: bool = False,
+ resnet_time_scale_shift: str = "default",
+ use_inflated_groupnorm=False,
+ # Additional
+ use_motion_module=False,
+ use_temporal_module=False,
+ motion_module_resolutions=(1, 2, 4, 8),
+ motion_module_mid_block=False,
+ motion_module_decoder_only=False,
+ motion_module_type=None,
+ temporal_module_type=None,
+ motion_module_kwargs={},
+ temporal_module_kwargs={},
+ unet_use_cross_frame_attention=None,
+ unet_use_temporal_attention=None,
+ ):
+ super().__init__()
+
+ self.sample_size = sample_size
+ time_embed_dim = block_out_channels[0] * 4
+
+ # input
+ self.conv_in = InflatedConv3d(
+ in_channels, block_out_channels[0], kernel_size=3, padding=(1, 1)
+ )
+
+ # time
+ self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift)
+ timestep_input_dim = block_out_channels[0]
+
+ self.time_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim)
+
+ # class embedding
+ if class_embed_type is None and num_class_embeds is not None:
+ self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim)
+ elif class_embed_type == "timestep":
+ self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim)
+ elif class_embed_type == "identity":
+ self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim)
+ else:
+ self.class_embedding = None
+
+ self.down_blocks = nn.ModuleList([])
+ self.mid_block = None
+ self.up_blocks = nn.ModuleList([])
+
+ if isinstance(only_cross_attention, bool):
+ only_cross_attention = [only_cross_attention] * len(down_block_types)
+
+ if isinstance(attention_head_dim, int):
+ attention_head_dim = (attention_head_dim,) * len(down_block_types)
+
+ # down
+ output_channel = block_out_channels[0]
+ for i, down_block_type in enumerate(down_block_types):
+ res = 2**i
+ input_channel = output_channel
+ output_channel = block_out_channels[i]
+ is_final_block = i == len(block_out_channels) - 1
+
+ down_block = get_down_block(
+ down_block_type,
+ num_layers=layers_per_block,
+ in_channels=input_channel,
+ out_channels=output_channel,
+ temb_channels=time_embed_dim,
+ add_downsample=not is_final_block,
+ resnet_eps=norm_eps,
+ resnet_act_fn=act_fn,
+ resnet_groups=norm_num_groups,
+ cross_attention_dim=cross_attention_dim,
+ attn_num_head_channels=attention_head_dim[i],
+ downsample_padding=downsample_padding,
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention[i],
+ upcast_attention=upcast_attention,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
+ unet_use_temporal_attention=unet_use_temporal_attention,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ use_motion_module=use_motion_module
+ and (res in motion_module_resolutions)
+ and (not motion_module_decoder_only),
+ use_temporal_module=use_temporal_module
+ and (res in motion_module_resolutions)
+ and (not motion_module_decoder_only),
+ motion_module_type=motion_module_type,
+ temporal_module_type=temporal_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ temporal_module_kwargs=temporal_module_kwargs
+ )
+ self.down_blocks.append(down_block)
+
+ # mid
+ if mid_block_type == "UNetMidBlock3DCrossAttn":
+ self.mid_block = UNetMidBlock3DCrossAttn(
+ in_channels=block_out_channels[-1],
+ temb_channels=time_embed_dim,
+ resnet_eps=norm_eps,
+ resnet_act_fn=act_fn,
+ output_scale_factor=mid_block_scale_factor,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ cross_attention_dim=cross_attention_dim,
+ attn_num_head_channels=attention_head_dim[-1],
+ resnet_groups=norm_num_groups,
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ upcast_attention=upcast_attention,
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
+ unet_use_temporal_attention=unet_use_temporal_attention,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ use_motion_module=use_motion_module and motion_module_mid_block,
+ use_temporal_module=use_temporal_module and motion_module_mid_block,
+ motion_module_type=motion_module_type,
+ temporal_module_type=temporal_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ temporal_module_kwargs=temporal_module_kwargs,
+ )
+ else:
+ raise ValueError(f"unknown mid_block_type : {mid_block_type}")
+
+ # count how many layers upsample the videos
+ self.num_upsamplers = 0
+
+ # up
+ reversed_block_out_channels = list(reversed(block_out_channels))
+ reversed_attention_head_dim = list(reversed(attention_head_dim))
+ only_cross_attention = list(reversed(only_cross_attention))
+ output_channel = reversed_block_out_channels[0]
+ for i, up_block_type in enumerate(up_block_types):
+ res = 2 ** (3 - i)
+ is_final_block = i == len(block_out_channels) - 1
+
+ prev_output_channel = output_channel
+ output_channel = reversed_block_out_channels[i]
+ input_channel = reversed_block_out_channels[
+ min(i + 1, len(block_out_channels) - 1)
+ ]
+
+ # add upsample block for all BUT final layer
+ if not is_final_block:
+ add_upsample = True
+ self.num_upsamplers += 1
+ else:
+ add_upsample = False
+
+ up_block = get_up_block(
+ up_block_type,
+ num_layers=layers_per_block + 1,
+ in_channels=input_channel,
+ out_channels=output_channel,
+ prev_output_channel=prev_output_channel,
+ temb_channels=time_embed_dim,
+ add_upsample=add_upsample,
+ resnet_eps=norm_eps,
+ resnet_act_fn=act_fn,
+ resnet_groups=norm_num_groups,
+ cross_attention_dim=cross_attention_dim,
+ attn_num_head_channels=reversed_attention_head_dim[i],
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention[i],
+ upcast_attention=upcast_attention,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
+ unet_use_temporal_attention=unet_use_temporal_attention,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ use_motion_module=use_motion_module
+ and (res in motion_module_resolutions),
+ use_temporal_module=use_temporal_module
+ and (res in motion_module_resolutions),
+ motion_module_type=motion_module_type,
+ temporal_module_type=temporal_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ temporal_module_kwargs=temporal_module_kwargs,
+ )
+ self.up_blocks.append(up_block)
+ prev_output_channel = output_channel
+
+ # out
+ if use_inflated_groupnorm:
+ self.conv_norm_out = InflatedGroupNorm(
+ num_channels=block_out_channels[0],
+ num_groups=norm_num_groups,
+ eps=norm_eps,
+ )
+ else:
+ self.conv_norm_out = nn.GroupNorm(
+ num_channels=block_out_channels[0],
+ num_groups=norm_num_groups,
+ eps=norm_eps,
+ )
+ self.conv_act = nn.SiLU()
+ self.conv_out = InflatedConv3d(
+ block_out_channels[0], out_channels, kernel_size=3, padding=1
+ )
+
+ @property
+ # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors
+ def attn_processors(self) -> Dict[str, AttentionProcessor]:
+ r"""
+ Returns:
+ `dict` of attention processors: A dictionary containing all attention processors used in the model with
+ indexed by its weight name.
+ """
+ # set recursively
+ processors = {}
+
+ def fn_recursive_add_processors(
+ name: str,
+ module: torch.nn.Module,
+ processors: Dict[str, AttentionProcessor],
+ ):
+ if hasattr(module, "set_processor"):
+ processors[f"{name}.processor"] = module.processor
+
+ for sub_name, child in module.named_children():
+ if "temporal_transformer" not in sub_name:
+ fn_recursive_add_processors(f"{name}.{sub_name}", child, processors)
+
+ return processors
+
+ for name, module in self.named_children():
+ if "temporal_transformer" not in name:
+ fn_recursive_add_processors(name, module, processors)
+
+ return processors
+
+ def set_attention_slice(self, slice_size):
+ r"""
+ Enable sliced attention computation.
+
+ When this option is enabled, the attention module will split the input tensor in slices, to compute attention
+ in several steps. This is useful to save some memory in exchange for a small speed decrease.
+
+ Args:
+ slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`):
+ When `"auto"`, halves the input to the attention heads, so attention will be computed in two steps. If
+ `"max"`, maxium amount of memory will be saved by running only one slice at a time. If a number is
+ provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim`
+ must be a multiple of `slice_size`.
+ """
+ sliceable_head_dims = []
+
+ def fn_recursive_retrieve_slicable_dims(module: torch.nn.Module):
+ if hasattr(module, "set_attention_slice"):
+ sliceable_head_dims.append(module.sliceable_head_dim)
+
+ for child in module.children():
+ fn_recursive_retrieve_slicable_dims(child)
+
+ # retrieve number of attention layers
+ for module in self.children():
+ fn_recursive_retrieve_slicable_dims(module)
+
+ num_slicable_layers = len(sliceable_head_dims)
+
+ if slice_size == "auto":
+ # half the attention head size is usually a good trade-off between
+ # speed and memory
+ slice_size = [dim // 2 for dim in sliceable_head_dims]
+ elif slice_size == "max":
+ # make smallest slice possible
+ slice_size = num_slicable_layers * [1]
+
+ slice_size = (
+ num_slicable_layers * [slice_size]
+ if not isinstance(slice_size, list)
+ else slice_size
+ )
+
+ if len(slice_size) != len(sliceable_head_dims):
+ raise ValueError(
+ f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different"
+ f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}."
+ )
+
+ for i in range(len(slice_size)):
+ size = slice_size[i]
+ dim = sliceable_head_dims[i]
+ if size is not None and size > dim:
+ raise ValueError(f"size {size} has to be smaller or equal to {dim}.")
+
+ # Recursively walk through all the children.
+ # Any children which exposes the set_attention_slice method
+ # gets the message
+ def fn_recursive_set_attention_slice(
+ module: torch.nn.Module, slice_size: List[int]
+ ):
+ if hasattr(module, "set_attention_slice"):
+ module.set_attention_slice(slice_size.pop())
+
+ for child in module.children():
+ fn_recursive_set_attention_slice(child, slice_size)
+
+ reversed_slice_size = list(reversed(slice_size))
+ for module in self.children():
+ fn_recursive_set_attention_slice(module, reversed_slice_size)
+
+ def set_use_cross_frame_attention(self, value):
+
+ def fn_recursive_set_use_cf_att(module: torch.nn.Module, value):
+ if hasattr(module, "set_use_cross_frame_attention"):
+ module.set_use_cross_frame_attention(value)
+
+ for child in module.children():
+ fn_recursive_set_use_cf_att(child, value)
+
+ for module in self.children():
+ fn_recursive_set_use_cf_att(module, value)
+
+ def _set_gradient_checkpointing(self, module, value=False):
+ if hasattr(module, "gradient_checkpointing"):
+ module.gradient_checkpointing = value
+
+ # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.set_attn_processor
+ def set_attn_processor(
+ self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]
+ ):
+ r"""
+ Sets the attention processor to use to compute attention.
+
+ Parameters:
+ processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):
+ The instantiated processor class or a dictionary of processor classes that will be set as the processor
+ for **all** `Attention` layers.
+
+ If `processor` is a dict, the key needs to define the path to the corresponding cross attention
+ processor. This is strongly recommended when setting trainable attention processors.
+
+ """
+ count = len(self.attn_processors.keys())
+
+ if isinstance(processor, dict) and len(processor) != count:
+ raise ValueError(
+ f"A dict of processors was passed, but the number of processors {len(processor)} does not match the"
+ f" number of attention layers: {count}. Please make sure to pass {count} processor classes."
+ )
+
+ def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor):
+ if hasattr(module, "set_processor"):
+ if not isinstance(processor, dict):
+ module.set_processor(processor)
+ else:
+ module.set_processor(processor.pop(f"{name}.processor"))
+
+ for sub_name, child in module.named_children():
+ fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor)
+
+ for name, module in self.named_children():
+ fn_recursive_attn_processor(name, module, processor)
+
+ def forward(
+ self,
+ sample: torch.FloatTensor,
+ timestep: Union[torch.Tensor, float, int],
+ encoder_hidden_states: torch.Tensor,
+ class_labels: Optional[torch.Tensor] = None,
+ pose_cond_fea = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
+ mid_block_additional_residual: Optional[torch.Tensor] = None,
+ return_dict: bool = True,
+ skip_mm: bool = False,
+ ) -> Union[UNet3DConditionOutput, Tuple]:
+ r"""
+ Args:
+ sample (`torch.FloatTensor`): (batch, channel, height, width) noisy inputs tensor
+ timestep (`torch.FloatTensor` or `float` or `int`): (batch) timesteps
+ encoder_hidden_states (`torch.FloatTensor`): (batch, sequence_length, feature_dim) encoder hidden states
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether or not to return a [`models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain tuple.
+
+ Returns:
+ [`~models.unet_2d_condition.UNet2DConditionOutput`] or `tuple`:
+ [`~models.unet_2d_condition.UNet2DConditionOutput`] if `return_dict` is True, otherwise a `tuple`. When
+ returning a tuple, the first element is the sample tensor.
+ """
+ # By default samples have to be AT least a multiple of the overall upsampling factor.
+ # The overall upsampling factor is equal to 2 ** (# num of upsampling layears).
+ # However, the upsampling interpolation output size can be forced to fit any upsampling size
+ # on the fly if necessary.
+ default_overall_up_factor = 2**self.num_upsamplers
+
+ # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`
+ forward_upsample_size = False
+ upsample_size = None
+
+ if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]):
+ logger.info("Forward upsample size to force interpolation output size.")
+ forward_upsample_size = True
+
+ # prepare attention_mask
+ if attention_mask is not None:
+ attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0
+ attention_mask = attention_mask.unsqueeze(1)
+
+ # center input if necessary
+ if self.config.center_input_sample:
+ sample = 2 * sample - 1.0
+
+ # time
+ timesteps = timestep
+ if not torch.is_tensor(timesteps):
+ # This would be a good case for the `match` statement (Python 3.10+)
+ is_mps = sample.device.type == "mps"
+ if isinstance(timestep, float):
+ dtype = torch.float32 if is_mps else torch.float64
+ else:
+ dtype = torch.int32 if is_mps else torch.int64
+ timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)
+ elif len(timesteps.shape) == 0:
+ timesteps = timesteps[None].to(sample.device)
+
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
+ # try:
+ # timesteps = timesteps.expand(sample.shape[0])
+ # except Exception:
+ # timesteps = timesteps.expand(sample.shape[0] * sample.shape[2])
+
+ t_emb = self.time_proj(timesteps)
+
+ # timesteps does not contain any weights and will always return f32 tensors
+ # but time_embedding might actually be running in fp16. so we need to cast here.
+ # there might be better ways to encapsulate this.
+ t_emb = t_emb.to(dtype=self.dtype)
+ emb = self.time_embedding(t_emb)
+ if self.class_embedding is not None:
+ if class_labels is None:
+ raise ValueError(
+ "class_labels should be provided when num_class_embeds > 0"
+ )
+
+ if self.config.class_embed_type == "timestep":
+ class_labels = self.time_proj(class_labels)
+
+ class_emb = self.class_embedding(class_labels).to(dtype=self.dtype)
+ emb = emb + class_emb
+
+ # pre-process
+ sample = self.conv_in(sample)
+ if pose_cond_fea is not None:
+ sample = sample + pose_cond_fea
+
+ # down
+ down_block_res_samples = (sample,)
+ block_count = 1
+ for downsample_block in self.down_blocks:
+ if (
+ hasattr(downsample_block, "has_cross_attention")
+ and downsample_block.has_cross_attention
+ ):
+ sample, res_samples = downsample_block(
+ hidden_states=sample,
+ temb=emb,
+ encoder_hidden_states=encoder_hidden_states,
+ attention_mask=attention_mask,
+ skip_mm=skip_mm,
+ )
+ else:
+ sample, res_samples = downsample_block(
+ hidden_states=sample,
+ temb=emb,
+ encoder_hidden_states=encoder_hidden_states,
+ skip_mm=skip_mm,
+ )
+ # if pose_cond_fea is not None:
+ # sample = sample + pose_cond_fea[block_count]
+ # block_count += 1
+ down_block_res_samples += res_samples
+
+ if down_block_additional_residuals is not None:
+ new_down_block_res_samples = ()
+
+ for down_block_res_sample, down_block_additional_residual in zip(
+ down_block_res_samples, down_block_additional_residuals
+ ):
+ down_block_res_sample = (
+ down_block_res_sample + down_block_additional_residual
+ )
+ new_down_block_res_samples += (down_block_res_sample,)
+
+ down_block_res_samples = new_down_block_res_samples
+
+ # mid
+ sample = self.mid_block(
+ sample,
+ emb,
+ encoder_hidden_states=encoder_hidden_states,
+ attention_mask=attention_mask,
+ skip_mm=skip_mm,
+ )
+
+ if mid_block_additional_residual is not None:
+ sample = sample + mid_block_additional_residual
+
+ # up
+ for i, upsample_block in enumerate(self.up_blocks):
+ is_final_block = i == len(self.up_blocks) - 1
+
+ res_samples = down_block_res_samples[-len(upsample_block.resnets) :]
+ down_block_res_samples = down_block_res_samples[
+ : -len(upsample_block.resnets)
+ ]
+
+ # if we have not reached the final block and need to forward the
+ # upsample size, we do it here
+ if not is_final_block and forward_upsample_size:
+ upsample_size = down_block_res_samples[-1].shape[2:]
+
+ if (
+ hasattr(upsample_block, "has_cross_attention")
+ and upsample_block.has_cross_attention
+ ):
+ sample = upsample_block(
+ hidden_states=sample,
+ temb=emb,
+ res_hidden_states_tuple=res_samples,
+ encoder_hidden_states=encoder_hidden_states,
+ upsample_size=upsample_size,
+ attention_mask=attention_mask,
+ skip_mm=skip_mm,
+ )
+ else:
+ sample = upsample_block(
+ hidden_states=sample,
+ temb=emb,
+ res_hidden_states_tuple=res_samples,
+ upsample_size=upsample_size,
+ encoder_hidden_states=encoder_hidden_states,
+ skip_mm=skip_mm,
+ )
+
+ # post-process
+ sample = self.conv_norm_out(sample)
+ sample = self.conv_act(sample)
+ sample = self.conv_out(sample)
+
+ if not return_dict:
+ return (sample,)
+
+ return UNet3DConditionOutput(sample=sample)
+
+ @classmethod
+ def from_pretrained_2d(
+ cls,
+ pretrained_model_path: PathLike,
+ motion_module_path: PathLike,
+ subfolder=None,
+ unet_additional_kwargs=None,
+ mm_zero_proj_out=False,
+ ):
+ pretrained_model_path = Path(pretrained_model_path)
+ motion_module_path = Path(motion_module_path)
+ if subfolder is not None:
+ pretrained_model_path = pretrained_model_path.joinpath(subfolder)
+ logger.info(
+ f"loaded temporal unet's pretrained weights from {pretrained_model_path} ..."
+ )
+
+ config_file = pretrained_model_path / "config.json"
+ if not (config_file.exists() and config_file.is_file()):
+ raise RuntimeError(f"{config_file} does not exist or is not a file")
+
+ unet_config = cls.load_config(config_file)
+ unet_config["_class_name"] = cls.__name__
+ unet_config["down_block_types"] = [
+ "CrossAttnDownBlock3D",
+ "CrossAttnDownBlock3D",
+ "CrossAttnDownBlock3D",
+ "DownBlock3D",
+ ]
+ unet_config["up_block_types"] = [
+ "UpBlock3D",
+ "CrossAttnUpBlock3D",
+ "CrossAttnUpBlock3D",
+ "CrossAttnUpBlock3D",
+ ]
+ unet_config["mid_block_type"] = "UNetMidBlock3DCrossAttn"
+
+ model = cls.from_config(unet_config, **unet_additional_kwargs)
+ # load the vanilla weights
+ if pretrained_model_path.joinpath(SAFETENSORS_WEIGHTS_NAME).exists():
+ logger.debug(
+ f"loading safeTensors weights from {pretrained_model_path} ..."
+ )
+ state_dict = load_file(
+ pretrained_model_path.joinpath(SAFETENSORS_WEIGHTS_NAME), device="cpu"
+ )
+
+ elif pretrained_model_path.joinpath(WEIGHTS_NAME).exists():
+ logger.debug(f"loading weights from {pretrained_model_path} ...")
+ state_dict = torch.load(
+ pretrained_model_path.joinpath(WEIGHTS_NAME),
+ map_location="cpu",
+ weights_only=True,
+ )
+ else:
+ raise FileNotFoundError(f"no weights file found in {pretrained_model_path}")
+
+ # load the motion module weights
+ if motion_module_path.exists() and motion_module_path.is_file():
+ if motion_module_path.suffix.lower() in [".pth", ".pt", ".ckpt"]:
+ logger.info(f"Load motion module params from {motion_module_path}")
+ motion_state_dict = torch.load(
+ motion_module_path, map_location="cpu", weights_only=True
+ )
+ elif motion_module_path.suffix.lower() == ".safetensors":
+ motion_state_dict = load_file(motion_module_path, device="cpu")
+ else:
+ raise RuntimeError(
+ f"unknown file format for motion module weights: {motion_module_path.suffix}"
+ )
+
+ motion_state_dict = {
+ k.replace('motion_modules.', 'temporal_modules.'): v for k, v in motion_state_dict.items() if not "pos_encoder" in k
+ }
+
+ if mm_zero_proj_out:
+ logger.info(f"Zero initialize proj_out layers in motion module...")
+ new_motion_state_dict = OrderedDict()
+ for k in motion_state_dict:
+ if "proj_out" in k:
+ continue
+ new_motion_state_dict[k] = motion_state_dict[k]
+ motion_state_dict = new_motion_state_dict
+
+ # merge the state dicts
+ state_dict.update(motion_state_dict)
+
+ # load the weights into the model
+ m, u = model.load_state_dict(state_dict, strict=False)
+ logger.debug(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};")
+
+ params = [
+ p.numel() if "temporal_modules" in n else 0
+ for n, p in model.named_parameters()
+ ]
+ mm_params = [
+ p.numel() if "motion_modules" in n else 0
+ for n, p in model.named_parameters()
+ ]
+ logger.info(
+ f"Loaded {sum(mm_params) / 1e6}M-parameter motion module, Loaded {sum(params) / 1e6}M-parameter temporal module"
+ )
+
+ return model
diff --git a/src/models/unet_3d_blocks.py b/src/models/unet_3d_blocks.py
new file mode 100644
index 0000000..bdec28f
--- /dev/null
+++ b/src/models/unet_3d_blocks.py
@@ -0,0 +1,1121 @@
+import pdb
+from typing import Dict, Optional
+import torch
+from torch import nn
+
+from src.models.motion_module import get_motion_module
+
+# from .motion_module import get_motion_module
+from src.models.resnet import Downsample3D, ResnetBlock3D, Upsample3D
+from .transformer_3d import Transformer3DModel
+
+
+def get_down_block(
+ down_block_type,
+ num_layers,
+ in_channels,
+ out_channels,
+ temb_channels,
+ add_downsample,
+ resnet_eps,
+ resnet_act_fn,
+ attn_num_head_channels,
+ resnet_groups=None,
+ cross_attention_dim=None,
+ downsample_padding=None,
+ dual_cross_attention=False,
+ use_linear_projection=False,
+ only_cross_attention=False,
+ upcast_attention=False,
+ resnet_time_scale_shift="default",
+ unet_use_cross_frame_attention=None,
+ unet_use_temporal_attention=None,
+ use_inflated_groupnorm=None,
+ use_motion_module=None,
+ motion_module_type=None,
+ motion_module_kwargs=None,
+ use_temporal_module=None,
+ temporal_module_type=None,
+ temporal_module_kwargs=None,
+):
+ down_block_type = (
+ down_block_type[7:]
+ if down_block_type.startswith("UNetRes")
+ else down_block_type
+ )
+ if down_block_type == "DownBlock3D":
+ return DownBlock3D(
+ num_layers=num_layers,
+ in_channels=in_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ add_downsample=add_downsample,
+ resnet_eps=resnet_eps,
+ resnet_act_fn=resnet_act_fn,
+ resnet_groups=resnet_groups,
+ downsample_padding=downsample_padding,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ use_motion_module=use_motion_module,
+ motion_module_type=motion_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ use_temporal_module=use_temporal_module,
+ temporal_module_type=temporal_module_type,
+ temporal_module_kwargs=temporal_module_kwargs,
+ )
+ elif down_block_type == "CrossAttnDownBlock3D":
+ if cross_attention_dim is None:
+ raise ValueError(
+ "cross_attention_dim must be specified for CrossAttnDownBlock3D"
+ )
+ return CrossAttnDownBlock3D(
+ num_layers=num_layers,
+ in_channels=in_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ add_downsample=add_downsample,
+ resnet_eps=resnet_eps,
+ resnet_act_fn=resnet_act_fn,
+ resnet_groups=resnet_groups,
+ downsample_padding=downsample_padding,
+ cross_attention_dim=cross_attention_dim,
+ attn_num_head_channels=attn_num_head_channels,
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention,
+ upcast_attention=upcast_attention,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
+ unet_use_temporal_attention=unet_use_temporal_attention,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ use_motion_module=use_motion_module,
+ motion_module_type=motion_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ use_temporal_module=use_temporal_module,
+ temporal_module_type=temporal_module_type,
+ temporal_module_kwargs=temporal_module_kwargs,
+ )
+ raise ValueError(f"{down_block_type} does not exist.")
+
+
+def get_up_block(
+ up_block_type,
+ num_layers,
+ in_channels,
+ out_channels,
+ prev_output_channel,
+ temb_channels,
+ add_upsample,
+ resnet_eps,
+ resnet_act_fn,
+ attn_num_head_channels,
+ resnet_groups=None,
+ cross_attention_dim=None,
+ dual_cross_attention=False,
+ use_linear_projection=False,
+ only_cross_attention=False,
+ upcast_attention=False,
+ resnet_time_scale_shift="default",
+ unet_use_cross_frame_attention=None,
+ unet_use_temporal_attention=None,
+ use_inflated_groupnorm=None,
+ use_motion_module=None,
+ motion_module_type=None,
+ motion_module_kwargs=None,
+ use_temporal_module=None,
+ temporal_module_type=None,
+ temporal_module_kwargs=None,
+):
+ up_block_type = (
+ up_block_type[7:] if up_block_type.startswith("UNetRes") else up_block_type
+ )
+ if up_block_type == "UpBlock3D":
+ return UpBlock3D(
+ num_layers=num_layers,
+ in_channels=in_channels,
+ out_channels=out_channels,
+ prev_output_channel=prev_output_channel,
+ temb_channels=temb_channels,
+ add_upsample=add_upsample,
+ resnet_eps=resnet_eps,
+ resnet_act_fn=resnet_act_fn,
+ resnet_groups=resnet_groups,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ use_motion_module=use_motion_module,
+ motion_module_type=motion_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ use_temporal_module=use_temporal_module,
+ temporal_module_type=temporal_module_type,
+ temporal_module_kwargs=temporal_module_kwargs,
+ )
+ elif up_block_type == "CrossAttnUpBlock3D":
+ if cross_attention_dim is None:
+ raise ValueError(
+ "cross_attention_dim must be specified for CrossAttnUpBlock3D"
+ )
+ return CrossAttnUpBlock3D(
+ num_layers=num_layers,
+ in_channels=in_channels,
+ out_channels=out_channels,
+ prev_output_channel=prev_output_channel,
+ temb_channels=temb_channels,
+ add_upsample=add_upsample,
+ resnet_eps=resnet_eps,
+ resnet_act_fn=resnet_act_fn,
+ resnet_groups=resnet_groups,
+ cross_attention_dim=cross_attention_dim,
+ attn_num_head_channels=attn_num_head_channels,
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention,
+ upcast_attention=upcast_attention,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
+ unet_use_temporal_attention=unet_use_temporal_attention,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ use_motion_module=use_motion_module,
+ motion_module_type=motion_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ use_temporal_module=use_temporal_module,
+ temporal_module_type=temporal_module_type,
+ temporal_module_kwargs=temporal_module_kwargs,
+ )
+ raise ValueError(f"{up_block_type} does not exist.")
+
+
+class UNetMidBlock3DCrossAttn(nn.Module):
+
+ def __init__(
+ self,
+ in_channels: int,
+ temb_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default",
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ resnet_pre_norm: bool = True,
+ attn_num_head_channels=1,
+ output_scale_factor=1.0,
+ cross_attention_dim=1280,
+ dual_cross_attention=False,
+ use_linear_projection=False,
+ upcast_attention=False,
+ unet_use_cross_frame_attention=None,
+ unet_use_temporal_attention=None,
+ use_inflated_groupnorm=None,
+ use_motion_module=None,
+ motion_module_type=None,
+ motion_module_kwargs=None,
+ use_temporal_module=None,
+ temporal_module_type=None,
+ temporal_module_kwargs=None,
+ **transformer_kwargs,
+ ):
+ super().__init__()
+
+ self.has_cross_attention = True
+ self.attn_num_head_channels = attn_num_head_channels
+ resnet_groups = (
+ resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)
+ )
+
+ # there is always at least one resnet
+ resnets = [
+ ResnetBlock3D(
+ in_channels=in_channels,
+ out_channels=in_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ )
+ ]
+ attentions = []
+ motion_modules = []
+
+ for _ in range(num_layers):
+ if dual_cross_attention:
+ raise NotImplementedError
+ attentions.append(
+ Transformer3DModel(
+ attn_num_head_channels,
+ in_channels // attn_num_head_channels,
+ in_channels=in_channels,
+ num_layers=1,
+ cross_attention_dim=cross_attention_dim,
+ norm_num_groups=resnet_groups,
+ use_linear_projection=use_linear_projection,
+ upcast_attention=upcast_attention,
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
+ unet_use_temporal_attention=unet_use_temporal_attention,
+ **transformer_kwargs
+ )
+ )
+ motion_modules.append(
+ get_motion_module(
+ in_channels=in_channels,
+ motion_module_type=motion_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ )
+ if use_motion_module
+ else None
+ )
+ resnets.append(
+ ResnetBlock3D(
+ in_channels=in_channels,
+ out_channels=in_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ )
+ )
+
+ self.attentions = nn.ModuleList(attentions)
+ self.resnets = nn.ModuleList(resnets)
+ self.motion_modules = nn.ModuleList(motion_modules)
+ self.temporal_modules = nn.ModuleList(
+ [
+ (
+ get_motion_module(
+ in_channels=in_channels,
+ motion_module_type=temporal_module_type,
+ motion_module_kwargs=temporal_module_kwargs,
+ )
+ if use_temporal_module
+ else None
+ )
+ for _ in range(num_layers)
+ ]
+ )
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ hidden_states,
+ temb=None,
+ encoder_hidden_states=None,
+ attention_mask=None,
+ skip_mm=False,
+ mid_reference=None,
+ ):
+ if isinstance(encoder_hidden_states, list):
+ encoder_hidden_states, motion_hidden_states = encoder_hidden_states
+ else:
+ motion_hidden_states = encoder_hidden_states
+
+ hidden_states = self.resnets[0](hidden_states, temb)
+ for attn, resnet, motion_module, temporal_module in zip(
+ self.attentions, self.resnets[1:], self.motion_modules, self.temporal_modules
+ ):
+ if self.training and self.gradient_checkpointing:
+ def create_custom_forward(module, return_dict=None):
+ def custom_forward(*inputs):
+ if return_dict is not None:
+ return module(*inputs, return_dict=return_dict)
+ else:
+ return module(*inputs)
+
+ return custom_forward
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(attn, return_dict=False),
+ hidden_states,
+ encoder_hidden_states,
+ )[0]
+ if (motion_module is not None) and not skip_mm:
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(motion_module),
+ hidden_states,
+ temb,
+ motion_hidden_states,
+ )
+ if (temporal_module is not None) and not skip_mm:
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(temporal_module),
+ hidden_states.requires_grad_(),
+ temb,
+ None,
+ )
+ # hidden_states = (
+ # temporal_module(hidden_states, temb, encoder_hidden_states=None)
+ # if (temporal_module is not None) and not skip_mm
+ # else hidden_states
+ # )
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet), hidden_states, temb
+ )
+ else:
+ hidden_states = attn(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ reference=mid_reference,
+ ).sample
+ hidden_states = (
+ motion_module(
+ hidden_states, temb, encoder_hidden_states=motion_hidden_states
+ )
+ if (motion_module is not None) and not skip_mm
+ else hidden_states
+ )
+ hidden_states = (
+ temporal_module(hidden_states, temb, encoder_hidden_states=None, debug=True)
+ if (temporal_module is not None) and not skip_mm
+ else hidden_states
+ )
+
+ hidden_states = resnet(hidden_states, temb)
+
+ return hidden_states
+
+
+class CrossAttnDownBlock3D(nn.Module):
+
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ temb_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default",
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ resnet_pre_norm: bool = True,
+ attn_num_head_channels=1,
+ cross_attention_dim=1280,
+ output_scale_factor=1.0,
+ downsample_padding=1,
+ add_downsample=True,
+ dual_cross_attention=False,
+ use_linear_projection=False,
+ only_cross_attention=False,
+ upcast_attention=False,
+ unet_use_cross_frame_attention=None,
+ unet_use_temporal_attention=None,
+ use_inflated_groupnorm=None,
+ use_motion_module=None,
+ motion_module_type=None,
+ motion_module_kwargs=None,
+ use_temporal_module=None,
+ temporal_module_type=None,
+ temporal_module_kwargs=None,
+ **transformer_kwargs,
+ ):
+ super().__init__()
+ resnets = []
+ attentions = []
+ motion_modules = []
+
+ self.has_cross_attention = True
+ self.attn_num_head_channels = attn_num_head_channels
+
+ for i in range(num_layers):
+ in_channels = in_channels if i == 0 else out_channels
+ resnets.append(
+ ResnetBlock3D(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ )
+ )
+ if dual_cross_attention:
+ raise NotImplementedError
+ attentions.append(
+ Transformer3DModel(
+ attn_num_head_channels,
+ out_channels // attn_num_head_channels,
+ in_channels=out_channels,
+ num_layers=1,
+ cross_attention_dim=cross_attention_dim,
+ norm_num_groups=resnet_groups,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention,
+ upcast_attention=upcast_attention,
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
+ unet_use_temporal_attention=unet_use_temporal_attention,
+ **transformer_kwargs,
+ )
+ )
+ motion_modules.append(
+ get_motion_module(
+ in_channels=out_channels,
+ motion_module_type=motion_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ )
+ if use_motion_module
+ else None
+ )
+
+ self.attentions = nn.ModuleList(attentions)
+ self.resnets = nn.ModuleList(resnets)
+ self.motion_modules = nn.ModuleList(motion_modules)
+ self.temporal_modules = nn.ModuleList(
+ [
+ (
+ get_motion_module(
+ in_channels=out_channels,
+ motion_module_type=temporal_module_type,
+ motion_module_kwargs=temporal_module_kwargs,
+ )
+ if use_temporal_module
+ else None
+ )
+ for _ in range(num_layers)
+ ]
+ )
+
+ if add_downsample:
+ self.downsamplers = nn.ModuleList(
+ [
+ Downsample3D(
+ out_channels,
+ use_conv=True,
+ out_channels=out_channels,
+ padding=downsample_padding,
+ name="op",
+ )
+ ]
+ )
+ else:
+ self.downsamplers = None
+
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ hidden_states,
+ temb=None,
+ encoder_hidden_states=None,
+ attention_mask=None,
+ skip_mm=False,
+ down_reference=None,
+ ):
+ if isinstance(encoder_hidden_states, list):
+ encoder_hidden_states, motion_hidden_states = encoder_hidden_states
+ else:
+ motion_hidden_states = encoder_hidden_states
+
+ output_states = ()
+
+ for i, (resnet, attn, motion_module, temporal_module) in enumerate(
+ zip(self.resnets, self.attentions, self.motion_modules, self.temporal_modules)
+ ):
+ reference = None
+ if not(down_reference is None):
+ reference = down_reference[i]
+ # self.gradient_checkpointing = False
+ if self.training and self.gradient_checkpointing:
+
+ def create_custom_forward(module, return_dict=None):
+ def custom_forward(*inputs):
+ if return_dict is not None:
+ return module(*inputs, return_dict=return_dict)
+ else:
+ return module(*inputs)
+
+ return custom_forward
+
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet), hidden_states, temb
+ )
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(attn, return_dict=False),
+ hidden_states,
+ encoder_hidden_states,
+ )[0]
+
+ # add motion module
+ if (motion_module is not None) and not skip_mm:
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(motion_module),
+ hidden_states,
+ temb,
+ motion_hidden_states,
+ )
+ if (temporal_module is not None) and not skip_mm:
+ # hidden_states = torch.utils.checkpoint.checkpoint(
+ # create_custom_forward(temporal_module),
+ # hidden_states.requires_grad_(),
+ # temb,
+ # None,
+ # )
+ hidden_states = (
+ temporal_module(hidden_states, temb, encoder_hidden_states=None)
+ if (temporal_module is not None) and not skip_mm
+ else hidden_states
+ )
+
+ else:
+ hidden_states = resnet(hidden_states, temb)
+ hidden_states = attn(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ reference=reference,
+ ).sample
+
+ # add motion module
+ hidden_states = (
+ motion_module(
+ hidden_states, temb, encoder_hidden_states=motion_hidden_states
+ )
+ if (motion_module is not None) and not skip_mm
+ else hidden_states
+ )
+ hidden_states = (
+ temporal_module(hidden_states, temb, encoder_hidden_states=None, debug=True)
+ if (temporal_module is not None) and not skip_mm
+ else hidden_states
+ )
+
+ output_states += (hidden_states,)
+
+ if self.downsamplers is not None:
+ for downsampler in self.downsamplers:
+ hidden_states = downsampler(hidden_states)
+
+ output_states += (hidden_states,)
+
+ return hidden_states, output_states
+
+
+class DownBlock3D(nn.Module):
+
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ temb_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default",
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ resnet_pre_norm: bool = True,
+ output_scale_factor=1.0,
+ add_downsample=True,
+ downsample_padding=1,
+ use_inflated_groupnorm=None,
+ use_motion_module=None,
+ motion_module_type=None,
+ motion_module_kwargs=None,
+ use_temporal_module=None,
+ temporal_module_type=None,
+ temporal_module_kwargs=None,
+ ):
+ super().__init__()
+ resnets = []
+ motion_modules = []
+
+ # use_motion_module = False
+ for i in range(num_layers):
+ in_channels = in_channels if i == 0 else out_channels
+ resnets.append(
+ ResnetBlock3D(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ )
+ )
+ motion_modules.append(
+ get_motion_module(
+ in_channels=out_channels,
+ motion_module_type=motion_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ )
+ if use_motion_module
+ else None
+ )
+
+ self.resnets = nn.ModuleList(resnets)
+ self.motion_modules = nn.ModuleList(motion_modules)
+ self.temporal_modules = nn.ModuleList(
+ [
+ (
+ get_motion_module(
+ in_channels=out_channels,
+ motion_module_type=temporal_module_type,
+ motion_module_kwargs=temporal_module_kwargs,
+ )
+ if use_temporal_module
+ else None
+ )
+ for _ in range(num_layers)
+ ]
+ )
+
+ if add_downsample:
+ self.downsamplers = nn.ModuleList(
+ [
+ Downsample3D(
+ out_channels,
+ use_conv=True,
+ out_channels=out_channels,
+ padding=downsample_padding,
+ name="op",
+ )
+ ]
+ )
+ else:
+ self.downsamplers = None
+
+ self.gradient_checkpointing = False
+
+ def forward(self, hidden_states, temb=None, encoder_hidden_states=None, skip_mm=False):
+ output_states = ()
+ if isinstance(encoder_hidden_states, list):
+ encoder_hidden_states, motion_hidden_states = encoder_hidden_states
+ else:
+ motion_hidden_states = encoder_hidden_states
+ for resnet, motion_module, temporal_module in zip(
+ self.resnets, self.motion_modules, self.temporal_modules
+ ):
+ # print(f"DownBlock3D {self.gradient_checkpointing = }")
+ if self.training and self.gradient_checkpointing:
+
+ def create_custom_forward(module):
+ def custom_forward(*inputs):
+ return module(*inputs)
+
+ return custom_forward
+
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet), hidden_states, temb
+ )
+ if (motion_module is not None) and not skip_mm:
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(motion_module),
+ hidden_states,
+ temb,
+ motion_hidden_states,
+ )
+
+ if (temporal_module is not None) and not skip_mm:
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(temporal_module),
+ hidden_states.requires_grad_(),
+ temb,
+ None,
+ )
+ else:
+ hidden_states = resnet(hidden_states, temb)
+
+ # add motion module
+ hidden_states = (
+ motion_module(
+ hidden_states, temb, encoder_hidden_states=motion_hidden_states
+ )
+ if (motion_module is not None) and not skip_mm
+ else hidden_states
+ )
+ hidden_states = (
+ temporal_module(
+ hidden_states, temb, encoder_hidden_states=None, debug=True
+ )
+ if (temporal_module is not None) and not skip_mm
+ else hidden_states
+ )
+
+ output_states += (hidden_states,)
+
+ if self.downsamplers is not None:
+ for downsampler in self.downsamplers:
+ hidden_states = downsampler(hidden_states)
+
+ output_states += (hidden_states,)
+
+ return hidden_states, output_states
+
+
+class CrossAttnUpBlock3D(nn.Module):
+
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ prev_output_channel: int,
+ temb_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default",
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ resnet_pre_norm: bool = True,
+ attn_num_head_channels=1,
+ cross_attention_dim=1280,
+ output_scale_factor=1.0,
+ add_upsample=True,
+ dual_cross_attention=False,
+ use_linear_projection=False,
+ only_cross_attention=False,
+ upcast_attention=False,
+ unet_use_cross_frame_attention=None,
+ unet_use_temporal_attention=None,
+ use_motion_module=None,
+ use_inflated_groupnorm=None,
+ motion_module_type=None,
+ motion_module_kwargs=None,
+ use_temporal_module=None,
+ temporal_module_type=None,
+ temporal_module_kwargs=None,
+ **transformer_kwargs,
+ ):
+ super().__init__()
+ resnets = []
+ attentions = []
+ motion_modules = []
+
+ self.has_cross_attention = True
+ self.attn_num_head_channels = attn_num_head_channels
+
+ for i in range(num_layers):
+ res_skip_channels = in_channels if (i == num_layers - 1) else out_channels
+ resnet_in_channels = prev_output_channel if i == 0 else out_channels
+
+ resnets.append(
+ ResnetBlock3D(
+ in_channels=resnet_in_channels + res_skip_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ )
+ )
+ if dual_cross_attention:
+ raise NotImplementedError
+ attentions.append(
+ Transformer3DModel(
+ attn_num_head_channels,
+ out_channels // attn_num_head_channels,
+ in_channels=out_channels,
+ num_layers=1,
+ cross_attention_dim=cross_attention_dim,
+ norm_num_groups=resnet_groups,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention,
+ upcast_attention=upcast_attention,
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
+ unet_use_temporal_attention=unet_use_temporal_attention,
+ **transformer_kwargs,
+ )
+ )
+ motion_modules.append(
+ get_motion_module(
+ in_channels=out_channels,
+ motion_module_type=motion_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ )
+ if use_motion_module
+ else None
+ )
+
+ self.attentions = nn.ModuleList(attentions)
+ self.resnets = nn.ModuleList(resnets)
+ self.motion_modules = nn.ModuleList(motion_modules)
+ self.temporal_modules = nn.ModuleList(
+ [
+ (
+ get_motion_module(
+ in_channels=out_channels,
+ motion_module_type=temporal_module_type,
+ motion_module_kwargs=temporal_module_kwargs,
+ )
+ if use_temporal_module
+ else None
+ )
+ for _ in range(num_layers)
+ ]
+ )
+
+ if add_upsample:
+ self.upsamplers = nn.ModuleList(
+ [Upsample3D(out_channels, use_conv=True, out_channels=out_channels)]
+ )
+ else:
+ self.upsamplers = None
+
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ hidden_states,
+ res_hidden_states_tuple,
+ temb=None,
+ encoder_hidden_states=None,
+ upsample_size=None,
+ attention_mask=None,
+ skip_mm=False,
+ up_reference=None,
+ ):
+ if isinstance(encoder_hidden_states, list):
+ encoder_hidden_states, motion_hidden_states = encoder_hidden_states
+ else:
+ motion_hidden_states = encoder_hidden_states
+ for i, (resnet, attn, motion_module, temporal_module) in enumerate(
+ zip(self.resnets, self.attentions, self.motion_modules, self.temporal_modules)
+ ):
+ reference = None
+ if not(up_reference is None):
+ reference = up_reference[i]
+ # pop res hidden states
+ res_hidden_states = res_hidden_states_tuple[-1]
+ res_hidden_states_tuple = res_hidden_states_tuple[:-1]
+ hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)
+
+ if self.training and self.gradient_checkpointing:
+
+ def create_custom_forward(module, return_dict=None):
+ def custom_forward(*inputs):
+ if return_dict is not None:
+ return module(*inputs, return_dict=return_dict)
+ else:
+ return module(*inputs)
+
+ return custom_forward
+
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet), hidden_states, temb
+ )
+ # hidden_states = attn(
+ # hidden_states,
+ # encoder_hidden_states=encoder_hidden_states,
+ # ).sample
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(attn, return_dict=False),
+ hidden_states,
+ encoder_hidden_states,
+ )[0]
+ if (motion_module is not None) and not skip_mm:
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(motion_module),
+ hidden_states,
+ temb,
+ motion_hidden_states,
+ )
+ if (temporal_module is not None) and not skip_mm:
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(temporal_module),
+ hidden_states.requires_grad_(),
+ temb,
+ None,
+ )
+
+ else:
+ hidden_states = resnet(hidden_states, temb)
+ hidden_states = attn(
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ reference=reference
+ ).sample
+
+ # add motion module
+ hidden_states = (
+ motion_module(
+ hidden_states, temb, encoder_hidden_states=motion_hidden_states
+ )
+ if (motion_module is not None) and not skip_mm
+ else hidden_states
+ )
+
+ # add temporal_module
+ hidden_states = (
+ temporal_module(hidden_states, temb, encoder_hidden_states=None, debug=True)
+ if (temporal_module is not None) and not skip_mm
+ else hidden_states
+ )
+
+ if self.upsamplers is not None:
+ for upsampler in self.upsamplers:
+ hidden_states = upsampler(hidden_states, upsample_size)
+
+ return hidden_states
+
+
+class UpBlock3D(nn.Module):
+
+ def __init__(
+ self,
+ in_channels: int,
+ prev_output_channel: int,
+ out_channels: int,
+ temb_channels: int,
+ dropout: float = 0.0,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default",
+ resnet_act_fn: str = "swish",
+ resnet_groups: int = 32,
+ resnet_pre_norm: bool = True,
+ output_scale_factor=1.0,
+ add_upsample=True,
+ use_inflated_groupnorm=None,
+ use_motion_module=None,
+ motion_module_type=None,
+ motion_module_kwargs=None,
+ use_temporal_module=None,
+ temporal_module_type=None,
+ temporal_module_kwargs=None,
+ ):
+ super().__init__()
+ resnets = []
+ motion_modules = []
+
+ # use_motion_module = False
+ for i in range(num_layers):
+ res_skip_channels = in_channels if (i == num_layers - 1) else out_channels
+ resnet_in_channels = prev_output_channel if i == 0 else out_channels
+
+ resnets.append(
+ ResnetBlock3D(
+ in_channels=resnet_in_channels + res_skip_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ dropout=dropout,
+ time_embedding_norm=resnet_time_scale_shift,
+ non_linearity=resnet_act_fn,
+ output_scale_factor=output_scale_factor,
+ pre_norm=resnet_pre_norm,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ )
+ )
+ motion_modules.append(
+ get_motion_module(
+ in_channels=out_channels,
+ motion_module_type=motion_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ )
+ if use_motion_module
+ else None
+ )
+
+ self.resnets = nn.ModuleList(resnets)
+ self.motion_modules = nn.ModuleList(motion_modules)
+ self.temporal_modules = nn.ModuleList(
+ [
+ (
+ get_motion_module(
+ in_channels=out_channels,
+ motion_module_type=temporal_module_type,
+ motion_module_kwargs=temporal_module_kwargs,
+ )
+ if use_temporal_module
+ else None
+ )
+ for _ in range(num_layers)
+ ]
+ )
+
+ if add_upsample:
+ self.upsamplers = nn.ModuleList(
+ [Upsample3D(out_channels, use_conv=True, out_channels=out_channels)]
+ )
+ else:
+ self.upsamplers = None
+
+ self.gradient_checkpointing = False
+
+ def forward(
+ self,
+ hidden_states,
+ res_hidden_states_tuple,
+ temb=None,
+ upsample_size=None,
+ encoder_hidden_states=None,
+ skip_mm=False,
+ ):
+ if isinstance(encoder_hidden_states, list):
+ encoder_hidden_states, motion_hidden_states = encoder_hidden_states
+ else:
+ motion_hidden_states = encoder_hidden_states
+ for resnet, motion_module, temporal_module in zip(self.resnets, self.motion_modules, self.temporal_modules):
+ # pop res hidden states
+ res_hidden_states = res_hidden_states_tuple[-1]
+ res_hidden_states_tuple = res_hidden_states_tuple[:-1]
+ hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)
+
+ # print(f"UpBlock3D {self.gradient_checkpointing = }")
+ if self.training and self.gradient_checkpointing:
+
+ def create_custom_forward(module):
+ def custom_forward(*inputs):
+ return module(*inputs)
+
+ return custom_forward
+
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(resnet), hidden_states, temb
+ )
+ if (motion_module is not None) and not skip_mm:
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(motion_module),
+ hidden_states,
+ temb,
+ motion_hidden_states,
+ )
+ if (temporal_module is not None) and not skip_mm:
+ hidden_states = torch.utils.checkpoint.checkpoint(
+ create_custom_forward(temporal_module),
+ hidden_states.requires_grad_(),
+ temb,
+ None,
+ )
+
+ else:
+ hidden_states = resnet(hidden_states, temb)
+ hidden_states = (
+ motion_module(
+ hidden_states, temb, encoder_hidden_states=motion_hidden_states
+ )
+ if (motion_module is not None) and not skip_mm
+ else hidden_states
+ )
+ hidden_states = (
+ temporal_module(hidden_states, temb, encoder_hidden_states=None, debug=True)
+ if (temporal_module is not None) and not skip_mm
+ else hidden_states
+ )
+
+ if self.upsamplers is not None:
+ for upsampler in self.upsamplers:
+ hidden_states = upsampler(hidden_states, upsample_size)
+
+ return hidden_states
diff --git a/src/models/unet_3d_explicit_reference.py b/src/models/unet_3d_explicit_reference.py
new file mode 100644
index 0000000..b51c208
--- /dev/null
+++ b/src/models/unet_3d_explicit_reference.py
@@ -0,0 +1,744 @@
+from collections import OrderedDict
+from dataclasses import dataclass
+import pdb
+from os import PathLike
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.utils.checkpoint
+import torch.nn.functional as F
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.models.attention_processor import AttentionProcessor
+from diffusers.models.embeddings import TimestepEmbedding, Timesteps
+from diffusers.models.modeling_utils import ModelMixin
+from diffusers.utils import SAFETENSORS_WEIGHTS_NAME, WEIGHTS_NAME, BaseOutput, logging
+from safetensors.torch import load_file
+
+from .resnet import InflatedConv3d, InflatedGroupNorm
+from .unet_3d_blocks import UNetMidBlock3DCrossAttn, get_down_block, get_up_block
+from einops import rearrange
+
+logger = logging.get_logger(__name__) # pylint: disable=invalid-name
+
+
+@dataclass
+class UNet3DConditionOutput(BaseOutput):
+ sample: torch.FloatTensor
+
+
+class UNet3DConditionModel(ModelMixin, ConfigMixin):
+ _supports_gradient_checkpointing = True
+
+ @register_to_config
+ def __init__(
+ self,
+ sample_size: Optional[int] = None,
+ in_channels: int = 4,
+ out_channels: int = 4,
+ center_input_sample: bool = False,
+ flip_sin_to_cos: bool = True,
+ freq_shift: int = 0,
+ down_block_types: Tuple[str] = (
+ "CrossAttnDownBlock3D",
+ "CrossAttnDownBlock3D",
+ "CrossAttnDownBlock3D",
+ "DownBlock3D",
+ ),
+ mid_block_type: str = "UNetMidBlock3DCrossAttn",
+ up_block_types: Tuple[str] = (
+ "UpBlock3D",
+ "CrossAttnUpBlock3D",
+ "CrossAttnUpBlock3D",
+ "CrossAttnUpBlock3D",
+ ),
+ only_cross_attention: Union[bool, Tuple[bool]] = False,
+ block_out_channels: Tuple[int] = (320, 640, 1280, 1280),
+ layers_per_block: int = 2,
+ downsample_padding: int = 1,
+ mid_block_scale_factor: float = 1,
+ act_fn: str = "silu",
+ norm_num_groups: int = 32,
+ norm_eps: float = 1e-5,
+ cross_attention_dim: int = 1280,
+ attention_head_dim: Union[int, Tuple[int]] = 8,
+ dual_cross_attention: bool = False,
+ use_linear_projection: bool = False,
+ class_embed_type: Optional[str] = None,
+ num_class_embeds: Optional[int] = None,
+ upcast_attention: bool = False,
+ resnet_time_scale_shift: str = "default",
+ use_inflated_groupnorm=False,
+ # Additional
+ use_motion_module=False,
+ use_temporal_module=False,
+ motion_module_resolutions=(1, 2, 4, 8),
+ motion_module_mid_block=False,
+ motion_module_decoder_only=False,
+ motion_module_type=None,
+ temporal_module_type=None,
+ motion_module_kwargs={},
+ temporal_module_kwargs={},
+ unet_use_cross_frame_attention=None,
+ unet_use_temporal_attention=None,
+ ):
+ super().__init__()
+
+ self.sample_size = sample_size
+ time_embed_dim = block_out_channels[0] * 4
+
+ # input
+ self.conv_in = InflatedConv3d(
+ in_channels, block_out_channels[0], kernel_size=3, padding=(1, 1)
+ )
+
+ # time
+ self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift)
+ timestep_input_dim = block_out_channels[0]
+
+ self.time_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim)
+
+ # class embedding
+ if class_embed_type is None and num_class_embeds is not None:
+ self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim)
+ elif class_embed_type == "timestep":
+ self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim)
+ elif class_embed_type == "identity":
+ self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim)
+ else:
+ self.class_embedding = None
+
+ self.down_blocks = nn.ModuleList([])
+ self.mid_block = None
+ self.up_blocks = nn.ModuleList([])
+
+ if isinstance(only_cross_attention, bool):
+ only_cross_attention = [only_cross_attention] * len(down_block_types)
+
+ if isinstance(attention_head_dim, int):
+ attention_head_dim = (attention_head_dim,) * len(down_block_types)
+
+ # down
+ output_channel = block_out_channels[0]
+ for i, down_block_type in enumerate(down_block_types):
+ res = 2**i
+ input_channel = output_channel
+ output_channel = block_out_channels[i]
+ is_final_block = i == len(block_out_channels) - 1
+
+ down_block = get_down_block(
+ down_block_type,
+ num_layers=layers_per_block,
+ in_channels=input_channel,
+ out_channels=output_channel,
+ temb_channels=time_embed_dim,
+ add_downsample=not is_final_block,
+ resnet_eps=norm_eps,
+ resnet_act_fn=act_fn,
+ resnet_groups=norm_num_groups,
+ cross_attention_dim=cross_attention_dim,
+ attn_num_head_channels=attention_head_dim[i],
+ downsample_padding=downsample_padding,
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention[i],
+ upcast_attention=upcast_attention,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
+ unet_use_temporal_attention=unet_use_temporal_attention,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ use_motion_module=use_motion_module
+ and (res in motion_module_resolutions)
+ and (not motion_module_decoder_only),
+ use_temporal_module=use_temporal_module
+ and (res in motion_module_resolutions)
+ and (not motion_module_decoder_only),
+ motion_module_type=motion_module_type,
+ temporal_module_type=temporal_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ temporal_module_kwargs=temporal_module_kwargs
+ )
+ self.down_blocks.append(down_block)
+
+ # mid
+ if mid_block_type == "UNetMidBlock3DCrossAttn":
+ self.mid_block = UNetMidBlock3DCrossAttn(
+ in_channels=block_out_channels[-1],
+ temb_channels=time_embed_dim,
+ resnet_eps=norm_eps,
+ resnet_act_fn=act_fn,
+ output_scale_factor=mid_block_scale_factor,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ cross_attention_dim=cross_attention_dim,
+ attn_num_head_channels=attention_head_dim[-1],
+ resnet_groups=norm_num_groups,
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ upcast_attention=upcast_attention,
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
+ unet_use_temporal_attention=unet_use_temporal_attention,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ use_motion_module=use_motion_module and motion_module_mid_block,
+ use_temporal_module=use_temporal_module and motion_module_mid_block,
+ motion_module_type=motion_module_type,
+ temporal_module_type=temporal_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ temporal_module_kwargs=temporal_module_kwargs,
+ )
+ else:
+ raise ValueError(f"unknown mid_block_type : {mid_block_type}")
+
+ # count how many layers upsample the videos
+ self.num_upsamplers = 0
+
+ # up
+ reversed_block_out_channels = list(reversed(block_out_channels))
+ reversed_attention_head_dim = list(reversed(attention_head_dim))
+ only_cross_attention = list(reversed(only_cross_attention))
+ output_channel = reversed_block_out_channels[0]
+ for i, up_block_type in enumerate(up_block_types):
+ res = 2 ** (3 - i)
+ is_final_block = i == len(block_out_channels) - 1
+
+ prev_output_channel = output_channel
+ output_channel = reversed_block_out_channels[i]
+ input_channel = reversed_block_out_channels[
+ min(i + 1, len(block_out_channels) - 1)
+ ]
+
+ # add upsample block for all BUT final layer
+ if not is_final_block:
+ add_upsample = True
+ self.num_upsamplers += 1
+ else:
+ add_upsample = False
+
+ up_block = get_up_block(
+ up_block_type,
+ num_layers=layers_per_block + 1,
+ in_channels=input_channel,
+ out_channels=output_channel,
+ prev_output_channel=prev_output_channel,
+ temb_channels=time_embed_dim,
+ add_upsample=add_upsample,
+ resnet_eps=norm_eps,
+ resnet_act_fn=act_fn,
+ resnet_groups=norm_num_groups,
+ cross_attention_dim=cross_attention_dim,
+ attn_num_head_channels=reversed_attention_head_dim[i],
+ dual_cross_attention=dual_cross_attention,
+ use_linear_projection=use_linear_projection,
+ only_cross_attention=only_cross_attention[i],
+ upcast_attention=upcast_attention,
+ resnet_time_scale_shift=resnet_time_scale_shift,
+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,
+ unet_use_temporal_attention=unet_use_temporal_attention,
+ use_inflated_groupnorm=use_inflated_groupnorm,
+ use_motion_module=use_motion_module
+ and (res in motion_module_resolutions),
+ use_temporal_module=use_temporal_module
+ and (res in motion_module_resolutions),
+ motion_module_type=motion_module_type,
+ temporal_module_type=temporal_module_type,
+ motion_module_kwargs=motion_module_kwargs,
+ temporal_module_kwargs=temporal_module_kwargs,
+ )
+ self.up_blocks.append(up_block)
+ prev_output_channel = output_channel
+
+ # out
+ if use_inflated_groupnorm:
+ self.conv_norm_out = InflatedGroupNorm(
+ num_channels=block_out_channels[0],
+ num_groups=norm_num_groups,
+ eps=norm_eps,
+ )
+ else:
+ self.conv_norm_out = nn.GroupNorm(
+ num_channels=block_out_channels[0],
+ num_groups=norm_num_groups,
+ eps=norm_eps,
+ )
+ self.conv_act = nn.SiLU()
+ self.conv_out = InflatedConv3d(
+ block_out_channels[0], out_channels, kernel_size=3, padding=1
+ )
+
+ @property
+ # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors
+ def attn_processors(self) -> Dict[str, AttentionProcessor]:
+ r"""
+ Returns:
+ `dict` of attention processors: A dictionary containing all attention processors used in the model with
+ indexed by its weight name.
+ """
+ # set recursively
+ processors = {}
+
+ def fn_recursive_add_processors(
+ name: str,
+ module: torch.nn.Module,
+ processors: Dict[str, AttentionProcessor],
+ ):
+ if hasattr(module, "set_processor"):
+ processors[f"{name}.processor"] = module.processor
+
+ for sub_name, child in module.named_children():
+ if "temporal_transformer" not in sub_name:
+ fn_recursive_add_processors(f"{name}.{sub_name}", child, processors)
+
+ return processors
+
+ for name, module in self.named_children():
+ if "temporal_transformer" not in name:
+ fn_recursive_add_processors(name, module, processors)
+
+ return processors
+
+ def set_attention_slice(self, slice_size):
+ r"""
+ Enable sliced attention computation.
+
+ When this option is enabled, the attention module will split the input tensor in slices, to compute attention
+ in several steps. This is useful to save some memory in exchange for a small speed decrease.
+
+ Args:
+ slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`):
+ When `"auto"`, halves the input to the attention heads, so attention will be computed in two steps. If
+ `"max"`, maxium amount of memory will be saved by running only one slice at a time. If a number is
+ provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim`
+ must be a multiple of `slice_size`.
+ """
+ sliceable_head_dims = []
+
+ def fn_recursive_retrieve_slicable_dims(module: torch.nn.Module):
+ if hasattr(module, "set_attention_slice"):
+ sliceable_head_dims.append(module.sliceable_head_dim)
+
+ for child in module.children():
+ fn_recursive_retrieve_slicable_dims(child)
+
+ # retrieve number of attention layers
+ for module in self.children():
+ fn_recursive_retrieve_slicable_dims(module)
+
+ num_slicable_layers = len(sliceable_head_dims)
+
+ if slice_size == "auto":
+ # half the attention head size is usually a good trade-off between
+ # speed and memory
+ slice_size = [dim // 2 for dim in sliceable_head_dims]
+ elif slice_size == "max":
+ # make smallest slice possible
+ slice_size = num_slicable_layers * [1]
+
+ slice_size = (
+ num_slicable_layers * [slice_size]
+ if not isinstance(slice_size, list)
+ else slice_size
+ )
+
+ if len(slice_size) != len(sliceable_head_dims):
+ raise ValueError(
+ f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different"
+ f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}."
+ )
+
+ for i in range(len(slice_size)):
+ size = slice_size[i]
+ dim = sliceable_head_dims[i]
+ if size is not None and size > dim:
+ raise ValueError(f"size {size} has to be smaller or equal to {dim}.")
+
+ # Recursively walk through all the children.
+ # Any children which exposes the set_attention_slice method
+ # gets the message
+ def fn_recursive_set_attention_slice(
+ module: torch.nn.Module, slice_size: List[int]
+ ):
+ if hasattr(module, "set_attention_slice"):
+ module.set_attention_slice(slice_size.pop())
+
+ for child in module.children():
+ fn_recursive_set_attention_slice(child, slice_size)
+
+ reversed_slice_size = list(reversed(slice_size))
+ for module in self.children():
+ fn_recursive_set_attention_slice(module, reversed_slice_size)
+
+ def set_use_cross_frame_attention(self, value):
+
+ def fn_recursive_set_use_cf_att(module: torch.nn.Module, value):
+ if hasattr(module, "set_use_cross_frame_attention"):
+ module.set_use_cross_frame_attention(value)
+
+ for child in module.children():
+ fn_recursive_set_use_cf_att(child, value)
+
+ for module in self.children():
+ fn_recursive_set_use_cf_att(module, value)
+
+ def _set_gradient_checkpointing(self, module, value=False):
+ if hasattr(module, "gradient_checkpointing"):
+ module.gradient_checkpointing = value
+
+ # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.set_attn_processor
+ def set_attn_processor(
+ self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]
+ ):
+ r"""
+ Sets the attention processor to use to compute attention.
+
+ Parameters:
+ processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):
+ The instantiated processor class or a dictionary of processor classes that will be set as the processor
+ for **all** `Attention` layers.
+
+ If `processor` is a dict, the key needs to define the path to the corresponding cross attention
+ processor. This is strongly recommended when setting trainable attention processors.
+
+ """
+ count = len(self.attn_processors.keys())
+
+ if isinstance(processor, dict) and len(processor) != count:
+ raise ValueError(
+ f"A dict of processors was passed, but the number of processors {len(processor)} does not match the"
+ f" number of attention layers: {count}. Please make sure to pass {count} processor classes."
+ )
+
+ def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor):
+ if hasattr(module, "set_processor"):
+ if not isinstance(processor, dict):
+ module.set_processor(processor)
+ else:
+ module.set_processor(processor.pop(f"{name}.processor"))
+
+ for sub_name, child in module.named_children():
+ fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor)
+
+ for name, module in self.named_children():
+ fn_recursive_attn_processor(name, module, processor)
+
+ def forward(
+ self,
+ sample: torch.FloatTensor,
+ timestep: Union[torch.Tensor, float, int],
+ encoder_hidden_states: torch.Tensor,
+ pose_cond_fea : torch.Tensor,
+ d00: torch.Tensor,
+ d01: torch.Tensor,
+ d10: torch.Tensor,
+ d11: torch.Tensor,
+ d20: torch.Tensor,
+ d21: torch.Tensor,
+ m: torch.Tensor,
+ u10: torch.Tensor,
+ u11: torch.Tensor,
+ u12: torch.Tensor,
+ u20: torch.Tensor,
+ u21: torch.Tensor,
+ u22: torch.Tensor,
+ u30: torch.Tensor,
+ u31: torch.Tensor,
+ u32: torch.Tensor,
+ class_labels: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None,
+ mid_block_additional_residual: Optional[torch.Tensor] = None,
+ return_dict: bool = True,
+ skip_mm: bool = False,
+ ) -> Union[UNet3DConditionOutput, Tuple]:
+ r"""
+ Args:
+ sample (`torch.FloatTensor`): (batch, channel, height, width) noisy inputs tensor
+ timestep (`torch.FloatTensor` or `float` or `int`): (batch) timesteps
+ encoder_hidden_states (`torch.FloatTensor`): (batch, sequence_length, feature_dim) encoder hidden states
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether or not to return a [`models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain tuple.
+
+ Returns:
+ [`~models.unet_2d_condition.UNet2DConditionOutput`] or `tuple`:
+ [`~models.unet_2d_condition.UNet2DConditionOutput`] if `return_dict` is True, otherwise a `tuple`. When
+ returning a tuple, the first element is the sample tensor.
+ """
+ # By default samples have to be AT least a multiple of the overall upsampling factor.
+ # The overall upsampling factor is equal to 2 ** (# num of upsampling layears).
+ # However, the upsampling interpolation output size can be forced to fit any upsampling size
+ # on the fly if necessary.
+ default_overall_up_factor = 2**self.num_upsamplers
+
+ # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`
+ forward_upsample_size = False
+ upsample_size = None
+
+ if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]):
+ logger.info("Forward upsample size to force interpolation output size.")
+ forward_upsample_size = True
+
+ # prepare attention_mask
+ if attention_mask is not None:
+ attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0
+ attention_mask = attention_mask.unsqueeze(1)
+
+ # center input if necessary
+ if self.config.center_input_sample:
+ sample = 2 * sample - 1.0
+
+ # time
+ timesteps = timestep
+ if not torch.is_tensor(timesteps):
+ # This would be a good case for the `match` statement (Python 3.10+)
+ is_mps = sample.device.type == "mps"
+ if isinstance(timestep, float):
+ dtype = torch.float32 if is_mps else torch.float64
+ else:
+ dtype = torch.int32 if is_mps else torch.int64
+ timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)
+ elif len(timesteps.shape) == 0:
+ timesteps = timesteps[None].to(sample.device)
+
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
+ # try:
+ # timesteps = timesteps.expand(sample.shape[0])
+ # except Exception:
+ # timesteps = timesteps.expand(sample.shape[0] * sample.shape[2])
+
+ t_emb = self.time_proj(timesteps)
+
+ # timesteps does not contain any weights and will always return f32 tensors
+ # but time_embedding might actually be running in fp16. so we need to cast here.
+ # there might be better ways to encapsulate this.
+ t_emb = t_emb.to(dtype=self.dtype)
+ emb = self.time_embedding(t_emb)
+ if self.class_embedding is not None:
+ if class_labels is None:
+ raise ValueError(
+ "class_labels should be provided when num_class_embeds > 0"
+ )
+
+ if self.config.class_embed_type == "timestep":
+ class_labels = self.time_proj(class_labels)
+
+ class_emb = self.class_embedding(class_labels).to(dtype=self.dtype)
+ emb = emb + class_emb
+
+ # pre-process
+ sample = self.conv_in(sample)
+ if pose_cond_fea is not None:
+ sample = sample + pose_cond_fea
+
+ # down
+ down_block_res_samples = (sample,)
+ for i, downsample_block in enumerate(self.down_blocks):
+ down_reference = None
+ if i==0:
+ down_reference = [d00, d01]
+ elif i==1:
+ down_reference = [d10, d11]
+ elif i==2:
+ down_reference = [d20, d21]
+ if (
+ hasattr(downsample_block, "has_cross_attention")
+ and downsample_block.has_cross_attention
+ ):
+ sample, res_samples = downsample_block(
+ hidden_states=sample,
+ temb=emb,
+ encoder_hidden_states=encoder_hidden_states,
+ attention_mask=attention_mask,
+ skip_mm=skip_mm,
+ down_reference=down_reference,
+ )
+ else:
+ sample, res_samples = downsample_block(
+ hidden_states=sample,
+ temb=emb,
+ encoder_hidden_states=encoder_hidden_states,
+ skip_mm=skip_mm,
+ )
+ down_block_res_samples += res_samples
+
+ if down_block_additional_residuals is not None:
+ new_down_block_res_samples = ()
+
+ for down_block_res_sample, down_block_additional_residual in zip(
+ down_block_res_samples, down_block_additional_residuals
+ ):
+ down_block_res_sample = (
+ down_block_res_sample + down_block_additional_residual
+ )
+ new_down_block_res_samples += (down_block_res_sample,)
+
+ down_block_res_samples = new_down_block_res_samples
+
+ # mid
+ sample = self.mid_block(
+ sample,
+ emb,
+ encoder_hidden_states=encoder_hidden_states,
+ attention_mask=attention_mask,
+ skip_mm=skip_mm,
+ mid_reference=m,
+ )
+
+ if mid_block_additional_residual is not None:
+ sample = sample + mid_block_additional_residual
+
+ # up
+ for i, upsample_block in enumerate(self.up_blocks):
+ up_reference = None
+ if i==1:
+ up_reference = [u10, u11, u12]
+ elif i==2:
+ up_reference = [u20, u21, u22]
+ elif i==3:
+ up_reference = [u30, u31, u32]
+
+ is_final_block = i == len(self.up_blocks) - 1
+
+ res_samples = down_block_res_samples[-len(upsample_block.resnets) :]
+ down_block_res_samples = down_block_res_samples[
+ : -len(upsample_block.resnets)
+ ]
+
+ # if we have not reached the final block and need to forward the
+ # upsample size, we do it here
+ if not is_final_block and forward_upsample_size:
+ upsample_size = down_block_res_samples[-1].shape[2:]
+
+ if (
+ hasattr(upsample_block, "has_cross_attention")
+ and upsample_block.has_cross_attention
+ ):
+ sample = upsample_block(
+ hidden_states=sample,
+ temb=emb,
+ res_hidden_states_tuple=res_samples,
+ encoder_hidden_states=encoder_hidden_states,
+ upsample_size=upsample_size,
+ attention_mask=attention_mask,
+ skip_mm=skip_mm,
+ up_reference=up_reference,
+ )
+ else:
+ sample = upsample_block(
+ hidden_states=sample,
+ temb=emb,
+ res_hidden_states_tuple=res_samples,
+ upsample_size=upsample_size,
+ encoder_hidden_states=encoder_hidden_states,
+ skip_mm=skip_mm,
+ )
+
+ # post-process
+ sample = self.conv_norm_out(sample)
+ sample = self.conv_act(sample)
+ sample = self.conv_out(sample)
+
+ return sample
+
+ @classmethod
+ def from_pretrained_2d(
+ cls,
+ pretrained_model_path: PathLike,
+ motion_module_path: PathLike,
+ subfolder=None,
+ unet_additional_kwargs=None,
+ mm_zero_proj_out=False,
+ ):
+ pretrained_model_path = Path(pretrained_model_path)
+ motion_module_path = Path(motion_module_path)
+ if subfolder is not None:
+ pretrained_model_path = pretrained_model_path.joinpath(subfolder)
+ logger.info(
+ f"loaded temporal unet's pretrained weights from {pretrained_model_path} ..."
+ )
+
+ config_file = pretrained_model_path / "config.json"
+ if not (config_file.exists() and config_file.is_file()):
+ raise RuntimeError(f"{config_file} does not exist or is not a file")
+
+ unet_config = cls.load_config(config_file)
+ unet_config["_class_name"] = cls.__name__
+ unet_config["down_block_types"] = [
+ "CrossAttnDownBlock3D",
+ "CrossAttnDownBlock3D",
+ "CrossAttnDownBlock3D",
+ "DownBlock3D",
+ ]
+ unet_config["up_block_types"] = [
+ "UpBlock3D",
+ "CrossAttnUpBlock3D",
+ "CrossAttnUpBlock3D",
+ "CrossAttnUpBlock3D",
+ ]
+ unet_config["mid_block_type"] = "UNetMidBlock3DCrossAttn"
+
+ model = cls.from_config(unet_config, **unet_additional_kwargs)
+ # load the vanilla weights
+ if pretrained_model_path.joinpath(SAFETENSORS_WEIGHTS_NAME).exists():
+ logger.debug(
+ f"loading safeTensors weights from {pretrained_model_path} ..."
+ )
+ state_dict = load_file(
+ pretrained_model_path.joinpath(SAFETENSORS_WEIGHTS_NAME), device="cpu"
+ )
+
+ elif pretrained_model_path.joinpath(WEIGHTS_NAME).exists():
+ logger.debug(f"loading weights from {pretrained_model_path} ...")
+ state_dict = torch.load(
+ pretrained_model_path.joinpath(WEIGHTS_NAME),
+ map_location="cpu",
+ weights_only=True,
+ )
+ else:
+ raise FileNotFoundError(f"no weights file found in {pretrained_model_path}")
+
+ # load the motion module weights
+ if motion_module_path.exists() and motion_module_path.is_file():
+ if motion_module_path.suffix.lower() in [".pth", ".pt", ".ckpt"]:
+ logger.info(f"Load motion module params from {motion_module_path}")
+ motion_state_dict = torch.load(
+ motion_module_path, map_location="cpu", weights_only=True
+ )
+ elif motion_module_path.suffix.lower() == ".safetensors":
+ motion_state_dict = load_file(motion_module_path, device="cpu")
+ else:
+ raise RuntimeError(
+ f"unknown file format for motion module weights: {motion_module_path.suffix}"
+ )
+
+ motion_state_dict = {
+ k.replace('motion_modules.', 'temporal_modules.'): v for k, v in motion_state_dict.items() if not "pos_encoder" in k
+ }
+
+ if mm_zero_proj_out:
+ logger.info(f"Zero initialize proj_out layers in motion module...")
+ new_motion_state_dict = OrderedDict()
+ for k in motion_state_dict:
+ if "proj_out" in k:
+ continue
+ new_motion_state_dict[k] = motion_state_dict[k]
+ motion_state_dict = new_motion_state_dict
+
+ # merge the state dicts
+ state_dict.update(motion_state_dict)
+
+ # load the weights into the model
+ m, u = model.load_state_dict(state_dict, strict=False)
+ logger.debug(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};")
+
+ params = [
+ p.numel() if "temporal_modules" in n else 0
+ for n, p in model.named_parameters()
+ ]
+ mm_params = [
+ p.numel() if "motion_modules" in n else 0
+ for n, p in model.named_parameters()
+ ]
+ logger.info(
+ f"Loaded {sum(mm_params) / 1e6}M-parameter motion module, Loaded {sum(params) / 1e6}M-parameter temporal module"
+ )
+
+ return model
diff --git a/src/pipelines/context.py b/src/pipelines/context.py
new file mode 100644
index 0000000..543f09a
--- /dev/null
+++ b/src/pipelines/context.py
@@ -0,0 +1,76 @@
+from typing import Callable, List, Optional
+
+import numpy as np
+
+
+def ordered_halving(val):
+ bin_str = f"{val:064b}"
+ bin_flip = bin_str[::-1]
+ as_int = int(bin_flip, 2)
+
+ return as_int / (1 << 64)
+
+
+def uniform(
+ step: int = ...,
+ num_steps: Optional[int] = None,
+ num_frames: int = ...,
+ context_size: Optional[int] = None,
+ context_stride: int = 3,
+ context_overlap: int = 4,
+ closed_loop: bool = True,
+ offset=0,
+):
+ if num_frames <= context_size:
+ yield list(range(num_frames))
+ return
+
+ context_stride = min(
+ context_stride, int(np.ceil(np.log2(num_frames / context_size))) + 1
+ )
+
+ for context_step in 1 << np.arange(context_stride):
+ pad = int(round(num_frames * ordered_halving(step)))
+ for j in range(
+ int(ordered_halving(step) * context_step) + pad,
+ num_frames + pad + (0 if closed_loop else -context_overlap),
+ (context_size * context_step - context_overlap),
+ ):
+ yield [
+ (e+offset) % num_frames
+ for e in range(j, j + context_size * context_step, context_step)
+ ]
+
+
+def get_context_scheduler(name: str) -> Callable:
+ if name == "uniform":
+ return uniform
+ else:
+ raise ValueError(f"Unknown context_overlap policy {name}")
+
+
+def get_total_steps(
+ scheduler,
+ timesteps: List[int],
+ num_steps: Optional[int] = None,
+ num_frames: int = ...,
+ context_size: Optional[int] = None,
+ context_stride: int = 3,
+ context_overlap: int = 4,
+ closed_loop: bool = True,
+):
+ return sum(
+ len(
+ list(
+ scheduler(
+ i,
+ num_steps,
+ num_frames,
+ context_size,
+ context_stride,
+ context_overlap,
+ )
+ )
+ )
+ for i in range(len(timesteps))
+ )
diff --git a/src/pipelines/pipeline_pose2vid.py b/src/pipelines/pipeline_pose2vid.py
new file mode 100644
index 0000000..fda15bc
--- /dev/null
+++ b/src/pipelines/pipeline_pose2vid.py
@@ -0,0 +1,699 @@
+import inspect
+import math
+from dataclasses import dataclass
+from typing import Callable, List, Optional, Union
+import random
+import numpy as np
+import torch
+import torch.nn.functional as F
+import torchvision.transforms as transforms
+from diffusers import DiffusionPipeline
+from diffusers import (
+ AutoencoderKL,
+ AutoencoderKLTemporalDecoder,
+)
+from diffusers.image_processor import VaeImageProcessor
+from diffusers.schedulers import (
+ DDIMScheduler,
+ DPMSolverMultistepScheduler,
+ EulerAncestralDiscreteScheduler,
+ EulerDiscreteScheduler,
+ LMSDiscreteScheduler,
+ PNDMScheduler,
+)
+from diffusers.utils import BaseOutput, is_accelerate_available
+from diffusers.utils.torch_utils import randn_tensor
+from einops import rearrange
+from tqdm import tqdm
+from transformers import CLIPImageProcessor
+from src.pipelines.utils import get_tensor_interpolation_method
+from src.models.mutual_self_attention import ReferenceAttentionControl
+from src.utils.util import draw_keypoints
+
+@dataclass
+class Pose2VideoPipelineOutput(BaseOutput):
+ videos: Union[torch.Tensor, np.ndarray]
+
+
+class Pose2VideoPipeline(DiffusionPipeline):
+ _optional_components = []
+
+ def __init__(
+ self,
+ vae,
+ # vae_tiny,
+ image_encoder,
+ reference_unet,
+ denoising_unet,
+ motion_encoder,
+ pose_encoder,
+ pose_guider,
+ scheduler: Union[
+ DDIMScheduler,
+ PNDMScheduler,
+ LMSDiscreteScheduler,
+ EulerDiscreteScheduler,
+ EulerAncestralDiscreteScheduler,
+ DPMSolverMultistepScheduler,
+ ],
+ ):
+ super().__init__()
+
+ self.register_modules(
+ vae=vae,
+ # vae_tiny=vae_tiny,
+ image_encoder=image_encoder,
+ reference_unet=reference_unet,
+ denoising_unet=denoising_unet,
+ motion_encoder=motion_encoder,
+ pose_encoder=pose_encoder,
+ pose_guider=pose_guider,
+ scheduler=scheduler,
+ )
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
+ self.clip_image_processor = CLIPImageProcessor()
+ self.ref_image_processor = VaeImageProcessor(
+ vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True
+ )
+ self.cond_image_processor = VaeImageProcessor(
+ vae_scale_factor=self.vae_scale_factor,
+ do_convert_rgb=True,
+ do_normalize=True,
+ )
+
+ def enable_vae_slicing(self):
+ self.vae.enable_slicing()
+
+ def disable_vae_slicing(self):
+ self.vae.disable_slicing()
+
+ def enable_sequential_cpu_offload(self, gpu_id=0):
+ if is_accelerate_available():
+ from accelerate import cpu_offload
+ else:
+ raise ImportError("Please install accelerate via `pip install accelerate`")
+
+ device = torch.device(f"cuda:{gpu_id}")
+
+ for cpu_offloaded_model in [self.unet, self.text_encoder, self.vae]:
+ if cpu_offloaded_model is not None:
+ cpu_offload(cpu_offloaded_model, device)
+
+ @property
+ def _execution_device(self):
+ if self.device != torch.device("meta") or not hasattr(self.unet, "_hf_hook"):
+ return self.device
+ for module in self.unet.modules():
+ if (
+ hasattr(module, "_hf_hook")
+ and hasattr(module._hf_hook, "execution_device")
+ and module._hf_hook.execution_device is not None
+ ):
+ return torch.device(module._hf_hook.execution_device)
+ return self.device
+
+ def decode_latents_tiny(self, latents, decode_chunk_size=64):
+ video_length = latents.shape[2]
+ latents = rearrange(latents, "b c f h w -> (b f) c h w")
+ video = []
+ for frame_idx in tqdm(range(0, latents.shape[0], decode_chunk_size)):
+ video.append(self.vae_tiny.decode(latents[frame_idx : frame_idx + decode_chunk_size]).sample)
+ video = torch.cat(video)
+ video = rearrange(video, "(b f) c h w -> b c f h w", f=video_length)
+ video = (video / 2 + 0.5).clamp(0, 1)
+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16
+ video = video.cpu().float().numpy()
+ return video
+
+ def decode_latents(self, latents, decode_chunk_size=16):
+ video_length = latents.shape[2]
+ latents = 1 / 0.18215 * latents
+ latents = rearrange(latents, "b c f h w -> (b f) c h w")
+ video = []
+ for frame_idx in range(0, latents.shape[0], decode_chunk_size):
+ video.append(self.vae.decode(latents[frame_idx : frame_idx + decode_chunk_size]).sample)
+ video = torch.cat(video)
+ video = rearrange(video, "(b f) c h w -> b c f h w", f=video_length)
+ video = (video / 2 + 0.5).clamp(0, 1)
+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16
+ video = video.cpu().float().numpy()
+ return video
+
+ def decode_latents_svd(
+ self, latents: torch.FloatTensor, decode_chunk_size: int = 5
+ ):
+ video_length = latents.shape[2]
+ latents = 1 / self.vae.config.scaling_factor * latents
+ latents = rearrange(latents, "b c f h w -> (b f) c h w")
+
+ # decode decode_chunk_size frames at a time to avoid OOM
+ frames = []
+ for i in range(0, latents.shape[0], decode_chunk_size):
+ num_frames_in = latents[i : i + decode_chunk_size].shape[0]
+ frame = self.vae.decode(
+ latents[i : i + decode_chunk_size], num_frames_in
+ ).sample
+ frames.append(frame)
+ video = torch.cat(frames, dim=0)
+ video = rearrange(video, "(b f) c h w -> b c f h w", f=video_length)
+ video = (video / 2 + 0.5).clamp(0, 1)
+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16
+ video = video.cpu().float().numpy()
+ return video
+
+ def prepare_extra_step_kwargs(self, generator, eta):
+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
+ # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
+ # and should be between [0, 1]
+
+ accepts_eta = "eta" in set(
+ inspect.signature(self.scheduler.step).parameters.keys()
+ )
+ extra_step_kwargs = {}
+ if accepts_eta:
+ extra_step_kwargs["eta"] = eta
+
+ # check if the scheduler accepts generator
+ accepts_generator = "generator" in set(
+ inspect.signature(self.scheduler.step).parameters.keys()
+ )
+ if accepts_generator:
+ extra_step_kwargs["generator"] = generator
+ return extra_step_kwargs
+
+ def prepare_latents(
+ self,
+ batch_size,
+ num_channels_latents,
+ width,
+ height,
+ video_length,
+ dtype,
+ device,
+ generator,
+ latents=None,
+ ):
+ shape = (
+ batch_size,
+ num_channels_latents,
+ video_length,
+ height // self.vae_scale_factor,
+ width // self.vae_scale_factor,
+ )
+ if isinstance(generator, list) and len(generator) != batch_size:
+ raise ValueError(
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
+ )
+
+ if latents is None:
+ latents = randn_tensor(
+ shape, generator=generator, device=device, dtype=dtype
+ )
+ else:
+ latents = latents.to(device)
+
+ # scale the initial noise by the standard deviation required by the scheduler
+ latents = latents * self.scheduler.init_noise_sigma
+ return latents
+
+ def _encode_prompt(
+ self,
+ prompt,
+ device,
+ num_videos_per_prompt,
+ do_classifier_free_guidance,
+ negative_prompt,
+ ):
+ batch_size = len(prompt) if isinstance(prompt, list) else 1
+
+ text_inputs = self.tokenizer(
+ prompt,
+ padding="max_length",
+ max_length=self.tokenizer.model_max_length,
+ truncation=True,
+ return_tensors="pt",
+ )
+ text_input_ids = text_inputs.input_ids
+ untruncated_ids = self.tokenizer(
+ prompt, padding="longest", return_tensors="pt"
+ ).input_ids
+
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
+ text_input_ids, untruncated_ids
+ ):
+ removed_text = self.tokenizer.batch_decode(
+ untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]
+ )
+
+ if (
+ hasattr(self.text_encoder.config, "use_attention_mask")
+ and self.text_encoder.config.use_attention_mask
+ ):
+ attention_mask = text_inputs.attention_mask.to(device)
+ else:
+ attention_mask = None
+
+ text_embeddings = self.text_encoder(
+ text_input_ids.to(device),
+ attention_mask=attention_mask,
+ )
+ text_embeddings = text_embeddings[0]
+
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
+ bs_embed, seq_len, _ = text_embeddings.shape
+ text_embeddings = text_embeddings.repeat(1, num_videos_per_prompt, 1)
+ text_embeddings = text_embeddings.view(
+ bs_embed * num_videos_per_prompt, seq_len, -1
+ )
+
+ # get unconditional embeddings for classifier free guidance
+ if do_classifier_free_guidance:
+ uncond_tokens: List[str]
+ if negative_prompt is None:
+ uncond_tokens = [""] * batch_size
+ elif type(prompt) is not type(negative_prompt):
+ raise TypeError(
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
+ f" {type(prompt)}."
+ )
+ elif isinstance(negative_prompt, str):
+ uncond_tokens = [negative_prompt]
+ elif batch_size != len(negative_prompt):
+ raise ValueError(
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
+ " the batch size of `prompt`."
+ )
+ else:
+ uncond_tokens = negative_prompt
+
+ max_length = text_input_ids.shape[-1]
+ uncond_input = self.tokenizer(
+ uncond_tokens,
+ padding="max_length",
+ max_length=max_length,
+ truncation=True,
+ return_tensors="pt",
+ )
+
+ if (
+ hasattr(self.text_encoder.config, "use_attention_mask")
+ and self.text_encoder.config.use_attention_mask
+ ):
+ attention_mask = uncond_input.attention_mask.to(device)
+ else:
+ attention_mask = None
+
+ uncond_embeddings = self.text_encoder(
+ uncond_input.input_ids.to(device),
+ attention_mask=attention_mask,
+ )
+ uncond_embeddings = uncond_embeddings[0]
+
+ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
+ seq_len = uncond_embeddings.shape[1]
+ uncond_embeddings = uncond_embeddings.repeat(1, num_videos_per_prompt, 1)
+ uncond_embeddings = uncond_embeddings.view(
+ batch_size * num_videos_per_prompt, seq_len, -1
+ )
+
+ # For classifier free guidance, we need to do two forward passes.
+ # Here we concatenate the unconditional and text embeddings into a single batch
+ # to avoid doing two forward passes
+ text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
+
+ return text_embeddings
+
+ def downgrade_input(self, init_latents, generator, device, dtype):
+ mask = (
+ torch.rand(
+ *init_latents.shape,
+ generator=generator,
+ device=device,
+ )
+ >= 0.5
+ ).to(dtype=dtype)
+
+ blur = transforms.GaussianBlur(kernel_size=(9, 9), sigma=(18, 18))
+ # first_frame_latents = init_latents[:, :, 0:1, :, :].clone()
+ video_length = init_latents.shape[2]
+ init_latents = rearrange(init_latents, "b c f h w -> (b f) c h w")
+ init_latents_blur = blur(init_latents)
+ init_latents = rearrange(init_latents, "(b f) c h w -> b c f h w", f=video_length)
+ init_latents_blur = rearrange(init_latents_blur, "(b f) c h w -> b c f h w", f=video_length)
+ # init_latents = init_latents * mask + init_latents_blur * (1 - mask)
+ # init_latents[:, :, 0:1, :, :] = first_frame_latents
+
+ return init_latents_blur
+
+ def interpolate_latents(
+ self, latents: torch.Tensor, interpolation_factor: int, device
+ ):
+ if interpolation_factor < 2:
+ return latents
+
+ new_latents = torch.zeros(
+ (
+ latents.shape[0],
+ latents.shape[1],
+ ((latents.shape[2] - 1) * interpolation_factor) + 1,
+ latents.shape[3],
+ latents.shape[4],
+ ),
+ device=latents.device,
+ dtype=latents.dtype,
+ )
+
+ org_video_length = latents.shape[2]
+ rate = [i / interpolation_factor for i in range(interpolation_factor)][1:]
+
+ new_index = 0
+
+ v0 = None
+ v1 = None
+
+ for i0, i1 in zip(range(org_video_length), range(org_video_length)[1:]):
+ v0 = latents[:, :, i0, :, :]
+ v1 = latents[:, :, i1, :, :]
+
+ new_latents[:, :, new_index, :, :] = v0
+ new_index += 1
+
+ for f in rate:
+ v = get_tensor_interpolation_method()(
+ v0.to(device=device), v1.to(device=device), f
+ )
+ new_latents[:, :, new_index, :, :] = v.to(latents.device)
+ new_index += 1
+
+ new_latents[:, :, new_index, :, :] = v1
+ new_index += 1
+
+ return new_latents
+
+ def interpolate_tensors(self, a: torch.Tensor, b: torch.Tensor, num: int = 10) -> torch.Tensor:
+ """
+ Linear interpolation between tensors a and b.
+ input shape: (B, 1, D1, D2, ...)
+ output shape: (B, num, D1, D2, ...)
+ """
+ if a.shape != b.shape:
+ raise ValueError(f"Shape mismatch: a.shape={a.shape}, b.shape={b.shape}")
+
+ B, _, *rest = a.shape
+ alphas = torch.linspace(0, 1, num, device=a.device, dtype=a.dtype)
+ view_shape = (1, num) + (1,) * len(rest)
+ alphas = alphas.view(view_shape) # (1, num, 1, 1, ...)
+
+ # (B, num, D1, D2, ...)
+ result = (1 - alphas) * a + alphas * b
+ return result
+
+ def calculate_dis(self, A, B, threshold=10.):
+ """
+ A: (b, f1, c1, c2) bank
+ B: (b, f2, c1, c2) new data
+ """
+
+ A_flat = A.view(A.size(1), -1).clone()
+ B_flat = B.view(B.size(1), -1).clone()
+
+ dist = torch.cdist(B_flat.to(torch.float32), A_flat.to(torch.float32), p=2)
+
+ min_dist, min_idx = dist.min(dim=1) # (f2,)
+
+ idx_to_add = torch.nonzero(min_dist[:1] > threshold, as_tuple=False).squeeze(1).tolist()
+
+ if len(idx_to_add) > 0:
+ B_to_add = B[:, idx_to_add] # (1, k, c1, c2)
+ A_new = torch.cat([A, B_to_add], dim=1) # (1, f1+k, c1, c2)
+ add_flag = True
+ else:
+ A_new = A
+ add_flag = False
+
+ return add_flag, A_new
+
+ @torch.no_grad()
+ def __call__(
+ self,
+ tgt_images,
+ ref_image,
+ face_images,
+ ref_face_image,
+ width,
+ height,
+ video_length,
+ num_inference_steps,
+ guidance_scale,
+ eta: float = 0.0,
+ generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
+ output_type: Optional[str] = "tensor",
+ return_dict: bool = True,
+ temporal_window_size = 4,
+ temporal_adaptive_step = 4,
+ temporal_kv_cache=True,
+ init_latents=None,
+ **kwargs,
+ ):
+ assert num_inference_steps % temporal_adaptive_step == 0, "temporal_adaptive_step should be divisor of num_inference_steps"
+ assert video_length % temporal_window_size == 0, "temporal_window_size should be divisor of video_length"
+ # Default height and width to unet
+ height = height or self.unet.config.sample_size * self.vae_scale_factor
+ width = width or self.unet.config.sample_size * self.vae_scale_factor
+
+ device = self._execution_device
+
+ do_classifier_free_guidance = guidance_scale > 1.0
+
+ # Prepare timesteps
+ timesteps = torch.tensor([999, 666, 333, 0], device=device).long()
+ self.scheduler.set_step_length(333)
+ jump = num_inference_steps // temporal_adaptive_step
+ windows = video_length // temporal_window_size
+
+ batch_size = 1
+
+ # Prepare clip image embeds
+ clip_image = self.clip_image_processor.preprocess(
+ ref_image.resize((224, 224)), return_tensors="pt"
+ ).pixel_values
+ clip_image_embeds = self.image_encoder(
+ clip_image.to(device, dtype=self.image_encoder.dtype)
+ ).image_embeds
+ image_prompt_embeds = clip_image_embeds.unsqueeze(1)
+
+ uncond_image_prompt_embeds = torch.zeros_like(image_prompt_embeds)
+
+ if do_classifier_free_guidance:
+ image_prompt_embeds = torch.cat(
+ [uncond_image_prompt_embeds, image_prompt_embeds], dim=0
+ )
+
+ reference_control_writer = ReferenceAttentionControl(
+ self.reference_unet,
+ do_classifier_free_guidance=do_classifier_free_guidance,
+ mode="write",
+ batch_size=batch_size,
+ fusion_blocks="full",
+ )
+ reference_control_reader = ReferenceAttentionControl(
+ self.denoising_unet,
+ do_classifier_free_guidance=do_classifier_free_guidance,
+ mode="read",
+ batch_size=batch_size,
+ fusion_blocks="full",
+ cache_kv=temporal_kv_cache,
+ )
+
+ # Prepare extra step kwargs.
+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
+
+ # Prepare ref image latents
+ ref_image_tensor = self.ref_image_processor.preprocess(
+ ref_image, height=height, width=width
+ ) # (bs, c, width, height)
+ ref_image_tensor = ref_image_tensor.to(
+ dtype=self.vae.dtype, device=self.vae.device
+ )
+ ref_image_latents = self.vae.encode(ref_image_tensor).latent_dist.mean
+ ref_image_latents = ref_image_latents * 0.18215 # (b, 4, h, w)
+
+ # initialize latents
+ padding_num = (temporal_adaptive_step - 1) * temporal_window_size
+ noisy_latents_first = []
+ init_latents = ref_image_latents.unsqueeze(2).repeat(1, 1, padding_num, 1, 1)
+ init_timesteps = reversed(timesteps[0::jump]).repeat_interleave(temporal_window_size, dim=0)
+ noise = torch.randn_like(init_latents)
+ noisy_latents_first.append(self.scheduler.add_noise(init_latents, noise, init_timesteps[:padding_num]))
+
+ repeated_latents = ref_image_latents.unsqueeze(2).repeat(1, 1, video_length+padding_num, 1, 1)
+ noise = torch.randn_like(repeated_latents)
+ noisy_latents_first.append(self.scheduler.add_noise(repeated_latents, noise, timesteps[:1]))
+ latents = torch.cat(noisy_latents_first, dim=2)
+
+ # 3D implicit keypoints
+ ref_cond_tensor = self.cond_image_processor.preprocess(
+ ref_image, height=256, width=256
+ ).to(device=device, dtype=self.pose_encoder.dtype) # (1, c, h, w)
+ ref_cond_tensor = ref_cond_tensor / 2 + 0.5 # to [0, 1]
+
+ tgt_cond_tensor = self.cond_image_processor.preprocess(
+ tgt_images, height=256, width=256
+ ).to(device=device, dtype=self.pose_encoder.dtype) # (1, c, h, w)
+ tgt_cond_tensor = tgt_cond_tensor / 2 + 0.5 # to [0, 1]
+
+ mot_bbox_param = self.pose_encoder.interpolate_kps(ref_cond_tensor, tgt_cond_tensor, num_interp=padding_num+1) # (t, c)
+ keypoints = draw_keypoints(mot_bbox_param, device=device).unsqueeze(2)
+ keypoints = rearrange(keypoints, 'f c b h w -> b c f h w')
+ keypoints = keypoints.to(device=device, dtype=self.pose_guider.dtype)
+
+ pose_feas = []
+ for i in range(0, keypoints.shape[2], 256):
+ pose_fea = self.pose_guider(keypoints[:,:, i:i+256, :, :]) # (b, c, f)
+ pose_feas.append(pose_fea)
+ pose_feas = torch.cat(pose_feas, dim=2)
+
+ # motion embeddings
+ face_cond_tensor = self.cond_image_processor.preprocess(
+ face_images, height=224, width=224
+ ).transpose(0, 1)
+ face_cond_tensor = face_cond_tensor.unsqueeze(0) # (1, c, t, h, w)
+ face_cond_tensor = face_cond_tensor.to(
+ device=device, dtype=self.motion_encoder.dtype
+ )
+
+ motion_hidden_states = []
+ for i in range(0, face_cond_tensor.shape[2], 256):
+ motion_hidden_state = self.motion_encoder(face_cond_tensor[:,:, i:i+256, :, :]) # [b, c]
+ motion_hidden_states.append(motion_hidden_state)
+ motion_hidden_states = torch.cat(motion_hidden_states, dim=1) # [b, f, c]
+
+ ref_face_cond_tensor = self.cond_image_processor.preprocess(
+ ref_face_image, height=224, width=224
+ ).to(device=device, dtype=self.motion_encoder.dtype)
+ neg_motion_hidden_states = self.motion_encoder(ref_face_cond_tensor.unsqueeze(2)) # [b, c]
+
+ if do_classifier_free_guidance:
+ ref_motion_emb = mot_bbox_param[:1]
+ ref_keypoints = draw_keypoints(ref_motion_emb).unsqueeze(2)
+ ref_keypoints = ref_keypoints.to(device=device, dtype=self.pose_guider.dtype)
+ ref_pose_fea = self.pose_guider(ref_keypoints)
+
+ init_motion_hidden_states = self.interpolate_tensors(neg_motion_hidden_states, motion_hidden_states[:,:1], num=padding_num+1)[:,:-1]
+ motion_hidden_states = torch.cat([init_motion_hidden_states, motion_hidden_states, motion_hidden_states[:,-1:].repeat(1, padding_num, 1, 1)], dim=1)
+ pose_feas = torch.cat([pose_feas, pose_feas[:,:,-1:].repeat(1,1,padding_num,1,1)], dim=2)
+
+ motion_bank = neg_motion_hidden_states
+
+ # denoising loop
+ with self.progress_bar(total=windows + temporal_adaptive_step - 1) as progress_bar:
+ self.reference_unet(
+ ref_image_latents.repeat((2 if do_classifier_free_guidance else 1), 1, 1, 1),
+ torch.zeros((batch_size,),dtype=torch.float32,device=ref_image_latents.device),
+ encoder_hidden_states=image_prompt_embeds,
+ return_dict=False,
+ )
+ reference_control_reader.update(reference_control_writer)
+
+ for i in range(windows + temporal_adaptive_step - 1):
+ l = i * temporal_window_size
+ r = (i + temporal_adaptive_step) * temporal_window_size
+
+ motion_hidden_state = motion_hidden_states[:, l:r]
+ pose_fea = pose_feas[:, :, l:r]
+
+ add_flag = False
+ if l > temporal_adaptive_step * temporal_window_size * 2 and motion_bank.shape[1] < 5:
+ add_flag, motion_bank = self.calculate_dis(motion_bank, motion_hidden_state, threshold=17.)
+
+ if do_classifier_free_guidance:
+ motion_hidden_state = torch.cat(
+ [
+ neg_motion_hidden_states.unsqueeze(1).expand_as(motion_hidden_state),
+ motion_hidden_state,
+ ],
+ dim=0,
+ )
+
+ pose_fea = torch.cat(
+ [
+ ref_pose_fea.expand_as(pose_fea),
+ pose_fea,
+ ],
+ dim=0,
+ )
+
+ for j in range(jump):
+ latent = latents[:,:,l:r,:,:]
+ latent_model_input = (torch.cat([latent] * 2) if do_classifier_free_guidance else latent)
+ ut = reversed(timesteps[j::jump]).repeat_interleave(temporal_window_size, dim=0)
+ ut = torch.stack([ut] * batch_size * (2 if do_classifier_free_guidance else 1)).to(device)
+ ut = rearrange(ut, 'b f -> (b f)')
+
+ noise_pred = self.denoising_unet(
+ latent_model_input,
+ ut,
+ encoder_hidden_states=[
+ image_prompt_embeds,
+ motion_hidden_state,
+ ],
+ pose_cond_fea=pose_fea,
+ return_dict=False,
+ )[0]
+
+ if do_classifier_free_guidance:
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+ noise_pred = noise_pred_uncond + guidance_scale * (
+ noise_pred_text - noise_pred_uncond
+ )
+
+ clip_length = noise_pred.shape[2]
+ mid_noise_pred = rearrange(noise_pred, 'b c f h w -> (b f) c h w')
+ mid_latents = rearrange(latents[:,:,l:r,:,:], 'b c f h w -> (b f) c h w')
+ ts = ut.chunk(2)[0] if do_classifier_free_guidance else ut
+ mid_latents, pred_original_sample = self.scheduler.step(
+ mid_noise_pred, ts, mid_latents, **extra_step_kwargs, return_dict=False
+ )
+ mid_latents = rearrange(mid_latents, '(b f) c h w -> b c f h w', f=clip_length)
+ pred_original_sample = rearrange(pred_original_sample, '(b f) c h w -> b c f h w', f=clip_length)
+ mid_latents = torch.cat([
+ pred_original_sample[:,:,:temporal_window_size],
+ mid_latents[:,:,temporal_window_size:]], dim=2)
+ latents[:,:,l:r,:,:] = mid_latents
+
+ # history keyframe mechanism
+ if add_flag:
+ reference_control_writer.clear()
+ self.reference_unet(
+ pred_original_sample[:,:,0].repeat(
+ (2 if do_classifier_free_guidance else 1), 1, 1, 1
+ ).to(self.reference_unet.dtype),
+ torch.zeros((batch_size,),dtype=torch.float32,device=ref_image_latents.device),
+ encoder_hidden_states=image_prompt_embeds,
+ return_dict=False,
+ )
+ reference_control_reader.update_hkf(reference_control_writer)
+
+ progress_bar.update()
+ reference_control_reader.clear()
+ reference_control_writer.clear()
+
+ latents = latents[:, :, padding_num:video_length+padding_num, :, :]
+ # Post-processing
+ if isinstance(self.vae, AutoencoderKL):
+ # images = self.decode_latents_tiny(latents) # (b, c, f, h, w)
+ images = self.decode_latents(latents) # (b, c, f, h, w)
+ elif isinstance(self.vae, AutoencoderKLTemporalDecoder):
+ images = self.decode_latents_svd(latents)
+
+ # Convert to tensor
+ if output_type == "tensor":
+ images = torch.from_numpy(images)
+
+ if not return_dict:
+ return images
+
+ return Pose2VideoPipelineOutput(videos=images)
diff --git a/src/pipelines/utils.py b/src/pipelines/utils.py
new file mode 100644
index 0000000..3cd5076
--- /dev/null
+++ b/src/pipelines/utils.py
@@ -0,0 +1,29 @@
+import torch
+
+tensor_interpolation = None
+
+
+def get_tensor_interpolation_method():
+ return tensor_interpolation
+
+
+def set_tensor_interpolation_method(is_slerp):
+ global tensor_interpolation
+ tensor_interpolation = slerp if is_slerp else linear
+
+
+def linear(v1, v2, t):
+ return (1.0 - t) * v1 + t * v2
+
+
+def slerp(
+ v0: torch.Tensor, v1: torch.Tensor, t: float, DOT_THRESHOLD: float = 0.9995
+) -> torch.Tensor:
+ u0 = v0 / v0.norm()
+ u1 = v1 / v1.norm()
+ dot = (u0 * u1).sum()
+ if dot.abs() > DOT_THRESHOLD:
+ # logger.info(f'warning: v0 and v1 close to parallel, using linear interpolation instead.')
+ return (1.0 - t) * v0 + t * v1
+ omega = dot.acos()
+ return (((1.0 - t) * omega).sin() * v0 + (t * omega).sin() * v1) / omega.sin()
diff --git a/src/scheduler/scheduler_ddim.py b/src/scheduler/scheduler_ddim.py
new file mode 100644
index 0000000..cc526d8
--- /dev/null
+++ b/src/scheduler/scheduler_ddim.py
@@ -0,0 +1,550 @@
+# Copyright 2023 Stanford University Team and The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion
+# and https://github.com/hojonathanho/diffusion
+
+import math
+from dataclasses import dataclass
+from typing import List, Optional, Tuple, Union
+
+from einops import rearrange
+import numpy as np
+import torch
+
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.utils import BaseOutput
+from diffusers.utils.torch_utils import randn_tensor
+from diffusers.schedulers.scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin
+
+
+@dataclass
+# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->DDIM
+class DDIMSchedulerOutput(BaseOutput):
+ """
+ Output class for the scheduler's `step` function output.
+
+ Args:
+ prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):
+ Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the
+ denoising loop.
+ pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):
+ The predicted denoised sample `(x_{0})` based on the model output from the current timestep.
+ `pred_original_sample` can be used to preview progress or for guidance.
+ """
+
+ prev_sample: torch.FloatTensor
+ pred_original_sample: Optional[torch.FloatTensor] = None
+
+
+# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar
+def betas_for_alpha_bar(
+ num_diffusion_timesteps,
+ max_beta=0.999,
+ alpha_transform_type="cosine",
+):
+ """
+ Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of
+ (1-beta) over time from t = [0,1].
+
+ Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up
+ to that part of the diffusion process.
+
+
+ Args:
+ num_diffusion_timesteps (`int`): the number of betas to produce.
+ max_beta (`float`): the maximum beta to use; use values lower than 1 to
+ prevent singularities.
+ alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar.
+ Choose from `cosine` or `exp`
+
+ Returns:
+ betas (`np.ndarray`): the betas used by the scheduler to step the model outputs
+ """
+ if alpha_transform_type == "cosine":
+
+ def alpha_bar_fn(t):
+ return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2
+
+ elif alpha_transform_type == "exp":
+
+ def alpha_bar_fn(t):
+ return math.exp(t * -12.0)
+
+ else:
+ raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}")
+
+ betas = []
+ for i in range(num_diffusion_timesteps):
+ t1 = i / num_diffusion_timesteps
+ t2 = (i + 1) / num_diffusion_timesteps
+ betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta))
+ return torch.tensor(betas, dtype=torch.float32)
+
+
+def rescale_zero_terminal_snr(betas):
+ """
+ Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1)
+
+
+ Args:
+ betas (`torch.FloatTensor`):
+ the betas that the scheduler is being initialized with.
+
+ Returns:
+ `torch.FloatTensor`: rescaled betas with zero terminal SNR
+ """
+ # Convert betas to alphas_bar_sqrt
+ alphas = 1.0 - betas
+ alphas_cumprod = torch.cumprod(alphas, dim=0)
+ alphas_bar_sqrt = alphas_cumprod.sqrt()
+
+ # Store old values.
+ alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone()
+ alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone()
+
+ # Shift so the last timestep is zero.
+ alphas_bar_sqrt -= alphas_bar_sqrt_T
+
+ # Scale so the first timestep is back to the old value.
+ alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T)
+
+ # Convert alphas_bar_sqrt to betas
+ alphas_bar = alphas_bar_sqrt**2 # Revert sqrt
+ alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod
+ alphas = torch.cat([alphas_bar[0:1], alphas])
+ betas = 1 - alphas
+
+ return betas
+
+
+class DDIMScheduler(SchedulerMixin, ConfigMixin):
+ """
+ `DDIMScheduler` extends the denoising procedure introduced in denoising diffusion probabilistic models (DDPMs) with
+ non-Markovian guidance.
+
+ This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic
+ methods the library implements for all schedulers such as loading and saving.
+
+ Args:
+ num_train_timesteps (`int`, defaults to 1000):
+ The number of diffusion steps to train the model.
+ beta_start (`float`, defaults to 0.0001):
+ The starting `beta` value of inference.
+ beta_end (`float`, defaults to 0.02):
+ The final `beta` value.
+ beta_schedule (`str`, defaults to `"linear"`):
+ The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from
+ `linear`, `scaled_linear`, or `squaredcos_cap_v2`.
+ trained_betas (`np.ndarray`, *optional*):
+ Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`.
+ clip_sample (`bool`, defaults to `True`):
+ Clip the predicted sample for numerical stability.
+ clip_sample_range (`float`, defaults to 1.0):
+ The maximum magnitude for sample clipping. Valid only when `clip_sample=True`.
+ set_alpha_to_one (`bool`, defaults to `True`):
+ Each diffusion step uses the alphas product value at that step and at the previous one. For the final step
+ there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`,
+ otherwise it uses the alpha value at step 0.
+ steps_offset (`int`, defaults to 0):
+ An offset added to the inference steps. You can use a combination of `offset=1` and
+ `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+ Diffusion.
+ prediction_type (`str`, defaults to `epsilon`, *optional*):
+ Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process),
+ `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen
+ Video](https://imagen.research.google/video/paper.pdf) paper).
+ thresholding (`bool`, defaults to `False`):
+ Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such
+ as Stable Diffusion.
+ dynamic_thresholding_ratio (`float`, defaults to 0.995):
+ The ratio for the dynamic thresholding method. Valid only when `thresholding=True`.
+ sample_max_value (`float`, defaults to 1.0):
+ The threshold value for dynamic thresholding. Valid only when `thresholding=True`.
+ timestep_spacing (`str`, defaults to `"leading"`):
+ The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
+ Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
+ rescale_betas_zero_snr (`bool`, defaults to `False`):
+ Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and
+ dark samples instead of limiting it to samples with medium brightness. Loosely related to
+ [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506).
+ """
+
+ _compatibles = [e.name for e in KarrasDiffusionSchedulers]
+ order = 1
+
+ @register_to_config
+ def __init__(
+ self,
+ num_train_timesteps: int = 1000,
+ beta_start: float = 0.0001,
+ beta_end: float = 0.02,
+ beta_schedule: str = "linear",
+ trained_betas: Optional[Union[np.ndarray, List[float]]] = None,
+ clip_sample: bool = True,
+ set_alpha_to_one: bool = True,
+ steps_offset: int = 0,
+ prediction_type: str = "epsilon",
+ thresholding: bool = False,
+ dynamic_thresholding_ratio: float = 0.995,
+ clip_sample_range: float = 1.0,
+ sample_max_value: float = 1.0,
+ timestep_spacing: str = "leading",
+ rescale_betas_zero_snr: bool = False,
+ ):
+ if trained_betas is not None:
+ self.betas = torch.tensor(trained_betas, dtype=torch.float32)
+ elif beta_schedule == "linear":
+ self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32)
+ elif beta_schedule == "scaled_linear":
+ # this schedule is very specific to the latent diffusion model.
+ self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2
+ elif beta_schedule == "squaredcos_cap_v2":
+ # Glide cosine schedule
+ self.betas = betas_for_alpha_bar(num_train_timesteps)
+ else:
+ raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}")
+
+ # Rescale for zero SNR
+ if rescale_betas_zero_snr:
+ self.betas = rescale_zero_terminal_snr(self.betas)
+
+ self.alphas = 1.0 - self.betas
+ self.alphas_cumprod = torch.cumprod(self.alphas, dim=0)
+
+ # At every step in ddim, we are looking into the previous alphas_cumprod
+ # For the final step, there is no previous alphas_cumprod because we are already at 0
+ # `set_alpha_to_one` decides whether we set this parameter simply to one or
+ # whether we use the final alpha of the "non-previous" one.
+ self.final_alpha_cumprod = torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0]
+
+ # standard deviation of the initial noise distribution
+ self.init_noise_sigma = 1.0
+
+ # setable values
+ self.num_inference_steps = None
+ self.step_length = None
+ self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy().astype(np.int64))
+
+ def to(self, device):
+ self.betas = self.betas.to(device)
+ self.alphas = self.alphas.to(device)
+ self.alphas_cumprod = self.alphas_cumprod.to(device)
+ self.final_alpha_cumprod = self.final_alpha_cumprod.to(device)
+
+ def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor:
+ """
+ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the
+ current timestep.
+
+ Args:
+ sample (`torch.FloatTensor`):
+ The input sample.
+ timestep (`int`, *optional*):
+ The current timestep in the diffusion chain.
+
+ Returns:
+ `torch.FloatTensor`:
+ A scaled input sample.
+ """
+ return sample
+
+ def _get_variance(self, alpha_prod_t, alpha_prod_t_prev):
+ beta_prod_t = 1 - alpha_prod_t
+ beta_prod_t_prev = 1 - alpha_prod_t_prev
+
+ variance = (beta_prod_t_prev / beta_prod_t) * (1 - alpha_prod_t / alpha_prod_t_prev)
+
+ return variance
+
+ # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample
+ def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor:
+ """
+ "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the
+ prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by
+ s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing
+ pixels from saturation at each step. We find that dynamic thresholding results in significantly better
+ photorealism as well as better image-text alignment, especially when using very large guidance weights."
+
+ https://arxiv.org/abs/2205.11487
+ """
+ dtype = sample.dtype
+ batch_size, channels, *remaining_dims = sample.shape
+
+ if dtype not in (torch.float32, torch.float64):
+ sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half
+
+ # Flatten sample for doing quantile calculation along each image
+ sample = sample.reshape(batch_size, channels * np.prod(remaining_dims))
+
+ abs_sample = sample.abs() # "a certain percentile absolute pixel value"
+
+ s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1)
+ s = torch.clamp(
+ s, min=1, max=self.config.sample_max_value
+ ) # When clamped to min=1, equivalent to standard clipping to [-1, 1]
+ s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0
+ sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s"
+
+ sample = sample.reshape(batch_size, channels, *remaining_dims)
+ sample = sample.to(dtype)
+
+ return sample
+
+ def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None, num_train_timesteps = None):
+ """
+ Sets the discrete timesteps used for the diffusion chain (to be run before inference).
+
+ Args:
+ num_inference_steps (`int`):
+ The number of diffusion steps used when generating samples with a pre-trained model.
+ """
+ if num_inference_steps > self.config.num_train_timesteps:
+ raise ValueError(
+ f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:"
+ f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle"
+ f" maximal {self.config.num_train_timesteps} timesteps."
+ )
+
+ self.num_inference_steps = num_inference_steps
+
+ # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891
+ if self.config.timestep_spacing == "linspace":
+ timesteps = (
+ np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps)
+ .round()[::-1]
+ .copy()
+ .astype(np.int64)
+ )
+ elif self.config.timestep_spacing == "leading":
+ step_ratio = self.config.num_train_timesteps // self.num_inference_steps
+ # creates integer timesteps by multiplying by ratio
+ # casting to int to avoid issues when num_inference_step is power of 3
+ timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.int64)
+ timesteps += self.config.steps_offset
+ elif self.config.timestep_spacing == "trailing":
+ step_ratio = self.config.num_train_timesteps / self.num_inference_steps
+ # creates integer timesteps by multiplying by ratio
+ # casting to int to avoid issues when num_inference_step is power of 3
+ timesteps = np.round(np.arange(self.config.num_train_timesteps, 0, -step_ratio)).astype(np.int64)
+ timesteps -= 1
+ else:
+ raise ValueError(
+ f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'leading' or 'trailing'."
+ )
+
+ self.timesteps = torch.from_numpy(timesteps).to(device)
+
+ def set_step_length(self, step_length):
+ self.step_length = step_length
+
+ def step(
+ self,
+ model_output: torch.FloatTensor,
+ timestep: int,
+ sample: torch.FloatTensor,
+ eta: float = 0.0,
+ use_clipped_model_output: bool = False,
+ generator=None,
+ variance_noise: Optional[torch.FloatTensor] = None,
+ return_dict: bool = True,
+ ) -> Union[DDIMSchedulerOutput, Tuple]:
+ """
+ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
+ process from the learned model outputs (most often the predicted noise).
+
+ Args:
+ model_output (`torch.FloatTensor`):
+ The direct output from learned diffusion model.
+ timestep (`float`):
+ The current discrete timestep in the diffusion chain.
+ sample (`torch.FloatTensor`):
+ A current instance of a sample created by the diffusion process.
+ eta (`float`):
+ The weight of noise for added noise in diffusion step.
+ use_clipped_model_output (`bool`, defaults to `False`):
+ If `True`, computes "corrected" `model_output` from the clipped predicted original sample. Necessary
+ because predicted original sample is clipped to [-1, 1] when `self.config.clip_sample` is `True`. If no
+ clipping has happened, "corrected" `model_output` would coincide with the one provided as input and
+ `use_clipped_model_output` has no effect.
+ generator (`torch.Generator`, *optional*):
+ A random number generator.
+ variance_noise (`torch.FloatTensor`):
+ Alternative to generating noise with `generator` by directly providing the noise for the variance
+ itself. Useful for methods such as [`CycleDiffusion`].
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether or not to return a [`~schedulers.scheduling_ddim.DDIMSchedulerOutput`] or `tuple`.
+
+ Returns:
+ [`~schedulers.scheduling_utils.DDIMSchedulerOutput`] or `tuple`:
+ If return_dict is `True`, [`~schedulers.scheduling_ddim.DDIMSchedulerOutput`] is returned, otherwise a
+ tuple is returned where the first element is the sample tensor.
+
+ """
+ if self.num_inference_steps is None and self.step_length is None:
+ raise ValueError(
+ "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler"
+ )
+
+ # See formulas (12) and (16) of DDIM paper https://arxiv.org/pdf/2010.02502.pdf
+ # Ideally, read DDIM paper in-detail understanding
+
+ # Notation (<variable name> -> <name in paper>
+ # - pred_noise_t -> e_theta(x_t, t)
+ # - pred_original_sample -> f_theta(x_t, t) or x_0
+ # - std_dev_t -> sigma_t
+ # - eta -> η
+ # - pred_sample_direction -> "direction pointing to x_t"
+ # - pred_prev_sample -> "x_t-1"
+
+ # 1. get previous step value (=t-1)
+ if self.step_length is not None:
+ prev_timestep = timestep - self.step_length
+ else:
+ prev_timestep = timestep - self.config.num_train_timesteps // self.num_inference_steps
+
+ # 2. compute alphas, betas
+ alpha_prod_t = self.alphas_cumprod[timestep]
+ if isinstance(prev_timestep, torch.Tensor):
+ alpha_prod_t = alpha_prod_t.view(-1, 1, 1, 1)
+ prev_timestep = prev_timestep.to(self.alphas_cumprod.device)
+ alpha_prod_t_prev = torch.where(prev_timestep >= 0, self.alphas_cumprod[prev_timestep], self.final_alpha_cumprod)
+ alpha_prod_t_prev = alpha_prod_t_prev.view(-1, 1, 1, 1)
+ else:
+ alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod
+
+ # alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod
+
+ beta_prod_t = 1 - alpha_prod_t
+
+ # 3. compute predicted original sample from predicted noise also called
+ # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
+ if self.config.prediction_type == "epsilon":
+ pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5)
+ pred_epsilon = model_output
+ elif self.config.prediction_type == "sample":
+ pred_original_sample = model_output
+ pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5)
+ elif self.config.prediction_type == "v_prediction":
+ pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output
+ pred_epsilon = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample
+ else:
+ raise ValueError(
+ f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or"
+ " `v_prediction`"
+ )
+
+ # 4. Clip or threshold "predicted x_0"
+ if self.config.thresholding:
+ pred_original_sample = self._threshold_sample(pred_original_sample)
+ elif self.config.clip_sample:
+ pred_original_sample = pred_original_sample.clamp(
+ -self.config.clip_sample_range, self.config.clip_sample_range
+ )
+
+ # 5. compute variance: "sigma_t(η)" -> see formula (16)
+ # σ_t = sqrt((1 − α_t−1)/(1 − α_t)) * sqrt(1 − α_t/α_t−1)
+ variance = self._get_variance(alpha_prod_t, alpha_prod_t_prev)
+ std_dev_t = eta * variance ** (0.5)
+
+ if use_clipped_model_output:
+ # the pred_epsilon is always re-derived from the clipped x_0 in Glide
+ pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5)
+
+ # 6. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
+ pred_sample_direction = (1 - alpha_prod_t_prev - std_dev_t**2) ** (0.5) * pred_epsilon
+
+ # 7. compute x_t without "random noise" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
+ prev_sample = alpha_prod_t_prev ** (0.5) * pred_original_sample + pred_sample_direction
+
+ if eta > 0:
+ if variance_noise is not None and generator is not None:
+ raise ValueError(
+ "Cannot pass both generator and variance_noise. Please make sure that either `generator` or"
+ " `variance_noise` stays `None`."
+ )
+
+ if variance_noise is None:
+ variance_noise = randn_tensor(
+ model_output.shape, generator=generator, device=model_output.device, dtype=model_output.dtype
+ )
+ variance = std_dev_t * variance_noise
+
+ prev_sample = prev_sample + variance
+
+ if not return_dict:
+ return (prev_sample, pred_original_sample)
+
+ return DDIMSchedulerOutput(prev_sample=prev_sample, pred_original_sample=pred_original_sample)
+
+ # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise
+ def add_noise(
+ self,
+ original_samples: torch.FloatTensor,
+ noise: torch.FloatTensor,
+ timesteps: torch.IntTensor,
+ ) -> torch.FloatTensor:
+ # Make sure alphas_cumprod and timestep have same device and dtype as original_samples
+ # Move the self.alphas_cumprod to device to avoid redundant CPU to GPU data movement
+ # for the subsequent add_noise calls
+ shape_len = len(original_samples.shape)
+ if shape_len == 5:
+ b, c, f, h, w = original_samples.shape
+ original_samples = rearrange(original_samples, "b c f h w -> (b f) c h w")
+ noise = rearrange(noise, "b c f h w -> (b f) c h w")
+
+ self.alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device)
+ alphas_cumprod = self.alphas_cumprod.to(dtype=original_samples.dtype)
+ timesteps = timesteps.to(original_samples.device)
+
+ sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5
+ sqrt_alpha_prod = sqrt_alpha_prod.flatten()
+ while len(sqrt_alpha_prod.shape) < len(original_samples.shape):
+ sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1)
+
+ sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5
+ sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten()
+ while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape):
+ sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1)
+
+ noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise
+
+ if shape_len == 5:
+ noisy_samples = rearrange(noisy_samples, "(b f) c h w -> b c f h w", b=b)
+ return noisy_samples
+
+ # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.get_velocity
+ def get_velocity(
+ self, sample: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor
+ ) -> torch.FloatTensor:
+ # Make sure alphas_cumprod and timestep have same device and dtype as sample
+ self.alphas_cumprod = self.alphas_cumprod.to(device=sample.device)
+ alphas_cumprod = self.alphas_cumprod.to(dtype=sample.dtype)
+ timesteps = timesteps.to(sample.device)
+
+ sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5
+ sqrt_alpha_prod = sqrt_alpha_prod.flatten()
+ while len(sqrt_alpha_prod.shape) < len(sample.shape):
+ sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1)
+
+ sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5
+ sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten()
+ while len(sqrt_one_minus_alpha_prod.shape) < len(sample.shape):
+ sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1)
+
+ velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample
+ return velocity
+
+ def __len__(self):
+ return self.config.num_train_timesteps
diff --git a/src/utils/util.py b/src/utils/util.py
new file mode 100644
index 0000000..7f9ccbf
--- /dev/null
+++ b/src/utils/util.py
@@ -0,0 +1,313 @@
+import importlib
+import os
+import os.path as osp
+import shutil
+import sys
+from pathlib import Path
+
+import av
+import numpy as np
+import torch
+import torchvision
+from einops import rearrange
+from PIL import Image
+import cv2
+
+
+def save_checkpoint(model, save_dir, prefix, ckpt_num, logger, total_limit=None):
+ save_path = osp.join(save_dir, f"{prefix}-{ckpt_num}.pth")
+
+ if total_limit is not None:
+ checkpoints = os.listdir(save_dir)
+ checkpoints = [d for d in checkpoints if d.startswith(prefix)]
+ checkpoints = sorted(
+ checkpoints, key=lambda x: int(x.split("-")[1].split(".")[0])
+ )
+
+ if len(checkpoints) >= total_limit:
+ num_to_remove = len(checkpoints) - total_limit + 1
+ removing_checkpoints = checkpoints[0:num_to_remove]
+ logger.info(
+ f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints"
+ )
+ logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}")
+
+ for removing_checkpoint in removing_checkpoints:
+ removing_checkpoint = os.path.join(save_dir, removing_checkpoint)
+ os.remove(removing_checkpoint)
+
+ state_dict = model.state_dict()
+ torch.save(state_dict, save_path)
+
+
+def create_code_snapshot(root, dst_path, extensions=(".py", ".h", ".cpp", ".cu", ".cc", ".cuh", ".json", ".sh", ".bat", ".yaml"), exclude=()):
+ """Creates tarball with the source code"""
+ import tarfile
+ from pathlib import Path
+ with tarfile.open(str(dst_path), "w:gz") as tar:
+ for path in Path(root).rglob("*"):
+ if '.git' in path.parts:
+ continue
+ exclude_flag = False
+ if len(exclude) > 0:
+ for k in exclude:
+ if k in path.parts:
+ exclude_flag = True
+ if exclude_flag:
+ continue
+ if path.suffix.lower() in extensions:
+ try:
+ tar.add(path.as_posix(), arcname=path.relative_to(
+ root).as_posix(), recursive=True)
+ except:
+ print(path)
+ assert False, 'Error occur in create_code_snapshot'
+
+def seed_everything(seed):
+ import random
+
+ import numpy as np
+
+ torch.manual_seed(seed)
+ torch.cuda.manual_seed_all(seed)
+ np.random.seed(seed % (2**32))
+ random.seed(seed)
+
+
+def import_filename(filename):
+ spec = importlib.util.spec_from_file_location("mymodule", filename)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+def delete_additional_ckpt(base_path, num_keep):
+ dirs = []
+ for d in os.listdir(base_path):
+ if d.startswith("checkpoint-"):
+ dirs.append(d)
+ num_tot = len(dirs)
+ if num_tot <= num_keep:
+ return
+ # ensure ckpt is sorted and delete the ealier!
+ del_dirs = sorted(dirs, key=lambda x: int(x.split("-")[-1]))[: num_tot - num_keep]
+ for d in del_dirs:
+ path_to_dir = osp.join(base_path, d)
+ if osp.exists(path_to_dir):
+ shutil.rmtree(path_to_dir)
+
+
+def save_videos_from_pil(pil_images, path, fps=8, crf=None):
+ import av
+
+ save_fmt = Path(path).suffix
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ width, height = pil_images[0].size
+
+ if save_fmt == ".mp4":
+ if True:
+ codec = "libx264"
+ container = av.open(path, "w")
+ stream = container.add_stream(codec, rate=fps)
+
+ stream.width = width
+ stream.height = height
+ if crf is not None:
+ stream.options = {'crf': str(crf)}
+
+ for pil_image in pil_images:
+ # pil_image = Image.fromarray(image_arr).convert("RGB")
+ av_frame = av.VideoFrame.from_image(pil_image)
+ container.mux(stream.encode(av_frame))
+ container.mux(stream.encode())
+ container.close()
+ else:
+
+ video_writer = cv2.VideoWriter(
+ path.replace('.mp4', '_cv.mp4'), cv2.VideoWriter_fourcc(*'mp4v'), fps, (width, height)
+ )
+ for pil_image in pil_images:
+ img_np = cv2.cvtColor(np.array(pil_image), cv2.COLOR_RGB2BGR)
+ video_writer.write(img_np)
+ video_writer.release()
+
+ elif save_fmt == ".gif":
+ pil_images[0].save(
+ fp=path,
+ format="GIF",
+ append_images=pil_images[1:],
+ save_all=True,
+ duration=(1 / fps * 1000),
+ loop=0,
+ )
+ else:
+ raise ValueError("Unsupported file type. Use .mp4 or .gif.")
+
+
+def save_videos_grid(videos_, path: str, rescale=False, n_rows=6, fps=8, crf=None):
+ if not isinstance(videos_, list): videos_ = [videos_]
+
+ outputs = []
+ vid_len = videos_[0].shape[2]
+ for i in range(vid_len):
+ output = []
+ for videos in videos_:
+ videos = rearrange(videos, "b c t h w -> t b c h w")
+ height, width = videos.shape[-2:]
+
+ x = torchvision.utils.make_grid(videos[i], nrow=n_rows) # (c h w)
+ x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) # (h w c)
+ if rescale:
+ x = (x + 1.0) / 2.0 # -1,1 -> 0,1
+ x = (x * 255).numpy().astype(np.uint8)
+ output.append(x)
+
+ output = Image.fromarray(np.concatenate(output, axis=0))
+ outputs.append(output)
+
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ save_videos_from_pil(outputs, path, fps, crf)
+
+
+def save_videos_grid_ori(videos: torch.Tensor, path: str, rescale=False, n_rows=6, fps=8):
+ videos = rearrange(videos, "b c t h w -> t b c h w")
+ height, width = videos.shape[-2:]
+ outputs = []
+
+ for x in videos:
+ x = torchvision.utils.make_grid(x, nrow=n_rows) # (c h w)
+ x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) # (h w c)
+ if rescale:
+ x = (x + 1.0) / 2.0 # -1,1 -> 0,1
+ x = (x * 255).numpy().astype(np.uint8)
+ x = Image.fromarray(x)
+
+ outputs.append(x)
+
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+
+ save_videos_from_pil(outputs, path, fps)
+
+
+def read_frames(video_path):
+ container = av.open(video_path)
+
+ video_stream = next(s for s in container.streams if s.type == "video")
+ frames = []
+ for packet in container.demux(video_stream):
+ for frame in packet.decode():
+ image = Image.frombytes(
+ "RGB",
+ (frame.width, frame.height),
+ frame.to_rgb().to_ndarray(),
+ )
+ frames.append(image)
+
+ return frames
+
+
+def get_fps(video_path):
+ container = av.open(video_path)
+ video_stream = next(s for s in container.streams if s.type == "video")
+ fps = video_stream.average_rate
+ container.close()
+ return fps
+
+def draw_keypoints(keypoints, height=512, width=512, device="cuda"):
+ colors = torch.tensor([
+ [255, 0, 0],
+ [255, 255, 0],
+ [0, 255, 0],
+ [0, 255, 255],
+ [0, 0, 255],
+ [255, 0, 255],
+ [255, 0, 85],
+ ], device=device, dtype=torch.float32)
+
+ selected = torch.tensor([1, 2, 3, 4, 12, 15, 20], device=device)
+ B = keypoints.shape[0]
+
+ # [B, len(selected), 2]
+ pts = keypoints[:, selected] * 0.5 + 0.5
+ pts[..., 0] *= width
+ pts[..., 1] *= height
+ pts = pts.long()
+
+ canvas = torch.zeros((B, 3, height, width), device=device)
+ radius = 4
+
+ for i, color in enumerate(colors):
+ x = pts[:, i, 0]
+ y = pts[:, i, 1]
+ mask = (
+ (x[:, None, None] - torch.arange(width, device=device)) ** 2
+ + (y[:, None, None] - torch.arange(height, device=device)[:, None]) ** 2
+ ) <= radius**2
+ canvas[:, 0] += color[0] / 255.0 * mask
+ canvas[:, 1] += color[1] / 255.0 * mask
+ canvas[:, 2] += color[2] / 255.0 * mask
+
+ return canvas.clamp(0, 1)
+
+def get_boxes(keypoints, height=512, width=512):
+ selected = torch.tensor([1, 2, 3, 4, 12, 15, 20])
+
+ # [B, len(selected), 2]
+ pts = keypoints[:, selected] * 0.5 + 0.5
+ pts[..., 0] *= width
+ pts[..., 1] *= height
+ pts = pts.long()
+
+ cx = pts[..., 0].float().mean(dim=1) # [B]
+ cy = pts[..., 1].float().mean(dim=1) # [B]
+
+ min_y = pts[..., 1].float().min(dim=1)[0] # [B]
+
+ side = (cy - min_y) * 2.0
+ side = side * 1.7
+
+ x1 = (cx - side / 2 * 0.95).clamp(0, width - 1).long()
+ y1 = (cy - side / 2 * 0.95).clamp(0, height - 1).long()
+ x2 = (cx + side / 2 * 1.05).clamp(0, width - 1).long()
+ y2 = (cy + side / 2 * 1.05).clamp(0, height - 1).long()
+
+ boxes = torch.stack([x1, y1, x2, y2], dim=1) # [B, 4]
+
+ return boxes
+
+def crop_face(image_pil, face_mesh):
+ image = np.array(image_pil)
+ h, w = image.shape[:2]
+ results = face_mesh.process(image)
+ face_landmarks = results.multi_face_landmarks[0]
+ coords = [(int(l.x * w), int(l.y * h)) for l in face_landmarks.landmark]
+ xs, ys = zip(*coords)
+ x1, y1 = min(xs), min(ys)
+ x2, y2 = max(xs), max(ys)
+ face_box = (x1, y1, x2, y2)
+
+ left, top, right, bot = scale_bb(face_box, scale=1.1, size=image.shape[:2])
+
+ face_patch = image[int(top) : int(bot), int(left) : int(right)]
+
+ return face_patch
+
+def scale_bb(bbox, scale, size):
+ left, top, right, bot = bbox
+ width = right - left
+ height = bot - top
+ length = max(width, height) * scale
+ center_X = (left + right) * 0.5
+ center_Y = (top + bot) * 0.5
+ left, top, right, bot = [
+ center_X - length / 2,
+ center_Y - length / 2,
+ center_X + length / 2,
+ center_Y + length / 2,
+ ]
+ left = max(0, left)
+ top = max(0, top)
+ right = min(size[1] - 1, right)
+ bot = min(size[0] - 1, bot)
+ return np.array([left, top, right, bot])
\ No newline at end of file
diff --git a/src/wrapper.py b/src/wrapper.py
new file mode 100644
index 0000000..c6de821
--- /dev/null
+++ b/src/wrapper.py
@@ -0,0 +1,400 @@
+from omegaconf import OmegaConf
+import os
+import torch
+import numpy as np
+from PIL import Image
+import time
+import gc
+import cv2
+from src.models.unet_2d_condition import UNet2DConditionModel
+from src.models.pose_guider import PoseGuider
+from src.models.motion_encoder.encoder import MotEncoder
+from src.models.unet_3d import UNet3DConditionModel
+from src.models.mutual_self_attention import ReferenceAttentionControl
+from src.scheduler.scheduler_ddim import DDIMScheduler
+from src.liveportrait.motion_extractor import MotionExtractor
+from diffusers import AutoencoderKL
+from diffusers.image_processor import VaeImageProcessor
+from transformers import CLIPVisionModelWithProjection, CLIPImageProcessor
+from collections import deque
+from threading import Lock, Thread
+from torchvision import transforms as T
+from einops import rearrange
+from src.utils.util import draw_keypoints, get_boxes
+import torch.nn.functional as F
+
+def map_device(device_or_str):
+ return device_or_str if isinstance(device_or_str, torch.device) else torch.device(device_or_str)
+
+class PersonaLive:
+ def __init__(self, config_path, device=None):
+ cfg = OmegaConf.load(config_path)
+ if(device is None):
+ self.device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+ else:
+ self.device = map_device(device)
+
+ self.temporal_adaptive_step = cfg.temporal_adaptive_step
+ self.temporal_window_size = cfg.temporal_window_size
+
+ if cfg.dtype == "fp16":
+ self.numpy_dtype = np.float16
+ self.dtype = torch.float16
+ elif cfg.dtype == "fp32":
+ self.numpy_dtype = np.float32
+ self.dtype = torch.float32
+
+ infer_config = OmegaConf.load(cfg.inference_config)
+ sched_kwargs = OmegaConf.to_container(
+ infer_config.noise_scheduler_kwargs
+ )
+
+ self.num_inference_steps = cfg.num_inference_steps
+
+ # initialize models
+ self.pose_guider = PoseGuider().to(device=self.device, dtype=self.dtype)
+ pose_guider_state_dict = torch.load(cfg.pose_guider_path, map_location="cpu")
+ self.pose_guider.load_state_dict(pose_guider_state_dict)
+ del pose_guider_state_dict
+
+ self.motion_encoder = MotEncoder().to(dtype=self.dtype, device=self.device).eval()
+ motion_encoder_state_dict = torch.load(cfg.motion_encoder_path, map_location="cpu")
+ self.motion_encoder.load_state_dict(motion_encoder_state_dict)
+ del motion_encoder_state_dict
+
+ self.pose_encoder = MotionExtractor(num_kp=21).to(device=self.device, dtype=self.dtype).eval()
+ pose_encoder_state_dict = torch.load(cfg.pose_encoder_path, map_location="cpu")
+ self.pose_encoder.load_state_dict(pose_encoder_state_dict, strict=False)
+ del pose_encoder_state_dict
+
+ self.denoising_unet = UNet3DConditionModel.from_pretrained_2d(
+ cfg.pretrained_base_model_path,
+ "",
+ subfolder="unet",
+ unet_additional_kwargs=infer_config.unet_additional_kwargs,
+ ).to(dtype=self.dtype, device=self.device)
+
+ self.reference_unet = UNet2DConditionModel.from_pretrained(
+ cfg.pretrained_base_model_path,
+ subfolder="unet",
+ ).to(dtype=self.dtype, device=self.device)
+ reference_unet_state_dict = torch.load(cfg.reference_unet_weight_path, map_location="cpu")
+ self.reference_unet.load_state_dict(reference_unet_state_dict)
+ del reference_unet_state_dict
+
+ self.denoising_unet.load_state_dict(
+ torch.load(cfg.denoising_unet_path, map_location="cpu"), strict=False
+ )
+ self.denoising_unet.load_state_dict(
+ torch.load(
+ cfg.temporal_module_path,
+ map_location="cpu",
+ ),
+ strict=False,
+ )
+
+ self.reference_control_writer = ReferenceAttentionControl(
+ self.reference_unet,
+ do_classifier_free_guidance=False,
+ mode="write",
+ batch_size=cfg.batch_size,
+ fusion_blocks="full",
+ )
+ self.reference_control_reader = ReferenceAttentionControl(
+ self.denoising_unet,
+ do_classifier_free_guidance=False,
+ mode="read",
+ batch_size=cfg.batch_size,
+ fusion_blocks="full",
+ cache_kv=True,
+ )
+
+ self.vae = AutoencoderKL.from_pretrained(cfg.vae_model_path).to(
+ device=self.device, dtype=self.dtype
+ )
+ self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(
+ cfg.image_encoder_path,
+ ).to(device=self.device, dtype=self.dtype)
+
+ # miscellaneous
+ self.scheduler = DDIMScheduler(**sched_kwargs)
+ self.timesteps = torch.tensor([999, 666, 333, 0], device=self.device).long()
+ self.scheduler.set_step_length(333)
+
+ self.generator = torch.Generator(self.device)
+ self.generator.manual_seed(cfg.seed)
+
+ self.batch_size = cfg.batch_size
+ self.vae_scale_factor = 8
+ self.ref_image_processor = VaeImageProcessor(
+ vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True
+ )
+ self.clip_image_processor = CLIPImageProcessor()
+ self.cond_image_processor = VaeImageProcessor(
+ vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True, do_normalize=True)
+
+ self.first_frame = True
+ self.motion_bank = None
+ self.count = 0
+ self.num_khf = 0
+
+ self.latents_pile = deque([])
+ self.pose_pile = deque([])
+ self.motion_pile = deque([])
+
+ self.cfg = cfg
+ torch.cuda.empty_cache()
+
+ self.enable_xformers_memory_efficient_attention()
+
+ def enable_xformers_memory_efficient_attention(self):
+ self.reference_unet.enable_xformers_memory_efficient_attention()
+ self.denoising_unet.enable_xformers_memory_efficient_attention()
+
+ def fast_resize(self, images, target_width, target_height) -> torch.Tensor:
+ tgt_cond_tensor = F.interpolate(
+ images,
+ size=(target_width, target_height),
+ mode="bilinear",
+ align_corners=False,
+ )
+ return tgt_cond_tensor
+
+ @torch.no_grad()
+ def fuse_reference(self, ref_image): # pil input
+ clip_image = self.clip_image_processor.preprocess(
+ ref_image, return_tensors="pt"
+ ).pixel_values
+ ref_image_tensor = self.ref_image_processor.preprocess(
+ ref_image, height=self.cfg.reference_image_height, width=self.cfg.reference_image_width
+ ) # (bs, c, width, height)
+ clip_image_embeds = self.image_encoder(
+ clip_image.to(self.image_encoder.device, dtype=self.image_encoder.dtype)
+ ).image_embeds
+ self.encoder_hidden_states = clip_image_embeds.unsqueeze(1)
+ ref_image_tensor = ref_image_tensor.to(
+ dtype=self.vae.dtype, device=self.vae.device
+ )
+ self.ref_image_tensor = ref_image_tensor.squeeze(0)
+ ref_image_latents = self.vae.encode(ref_image_tensor).latent_dist.mean
+ ref_image_latents = ref_image_latents * 0.18215 # (b, 4, h, w)
+ self.reference_unet(
+ ref_image_latents.to(self.reference_unet.device),
+ torch.zeros((self.batch_size,),dtype=self.dtype,device=self.reference_unet.device),
+ encoder_hidden_states=self.encoder_hidden_states,
+ return_dict=False,
+ )
+ self.reference_control_reader.update(self.reference_control_writer)
+ self.encoder_hidden_states = self.encoder_hidden_states.to(self.device)
+
+ ref_cond_tensor = self.cond_image_processor.preprocess(
+ ref_image, height=256, width=256
+ ).to(device=self.device, dtype=self.pose_encoder.dtype) # (1, c, h, w)
+ self.ref_cond_tensor = ref_cond_tensor / 2 + 0.5 # to [0, 1]
+ self.ref_image_latents = ref_image_latents
+
+ padding_num = (self.temporal_adaptive_step - 1) * self.temporal_window_size
+ init_latents = ref_image_latents.unsqueeze(2).repeat(1, 1, padding_num, 1, 1)
+ noise = torch.randn_like(init_latents)
+ init_timesteps = reversed(self.timesteps).repeat_interleave(self.temporal_window_size, dim=0)
+ noisy_latents_first = self.scheduler.add_noise(init_latents, noise, init_timesteps[:padding_num])
+ for i in range(self.temporal_adaptive_step-1):
+ l = i * self.temporal_window_size
+ r = (i+1) * self.temporal_window_size
+ self.latents_pile.append(noisy_latents_first[:,:,l:r])
+
+ def crop_face(self, image_pil, boxes):
+ image = np.array(image_pil)
+
+ left, top, right, bot = boxes
+
+ face_patch = image[int(top) : int(bot), int(left) : int(right)]
+ face_patch = Image.fromarray(face_patch).convert("RGB")
+ return face_patch
+
+ def crop_face_tensor(self, image_tensor, boxes):
+ left, top, right, bot = boxes
+ left, top, right, bottom = map(int, (left, top, right, bot))
+
+ face_patch = image_tensor[:, top:bottom, left:right]
+ face_patch = F.interpolate(
+ face_patch.unsqueeze(0),
+ size=(224, 224),
+ mode="bilinear",
+ align_corners=False,
+ )
+ return face_patch
+
+ def interpolate_tensors(self, a: torch.Tensor, b: torch.Tensor, num: int = 10) -> torch.Tensor:
+ """
+ 在张量 a 和 b 之间线性插值。
+ 输入 shape: (B, 1, D1, D2, ...)
+ 输出 shape: (B, num, D1, D2, ...)
+ """
+ if a.shape != b.shape:
+ raise ValueError(f"Shape mismatch: a.shape={a.shape}, b.shape={b.shape}")
+
+ B, _, *rest = a.shape
+ # 插值系数 (num,) → reshape 成 (1, num, 1, 1, ...)
+ alphas = torch.linspace(0, 1, num, device=a.device, dtype=a.dtype)
+ view_shape = (1, num) + (1,) * len(rest)
+ alphas = alphas.view(view_shape) # (1, num, 1, 1, ...)
+
+ # 插值 (B, num, D1, D2, ...)
+ result = (1 - alphas) * a + alphas * b
+ return result
+
+ def calculate_dis(self, A, B, threshold=10.):
+ """
+ A: (b, f1, c1, c2) bank
+ B: (b, f2, c1, c2) new data
+ """
+
+ A_flat = A.view(A.size(1), -1).clone()
+ B_flat = B.view(B.size(1), -1).clone()
+
+ dist = torch.cdist(B_flat.to(torch.float32), A_flat.to(torch.float32), p=2)
+
+ min_dist, min_idx = dist.min(dim=1) # (f2,)
+
+ idx_to_add = torch.nonzero(min_dist[:1] > threshold, as_tuple=False).squeeze(1).tolist()
+
+ if len(idx_to_add) > 0: # 有需要添加的元素
+ B_to_add = B[:, idx_to_add] # (1, k, c1, c2)
+ A_new = torch.cat([A, B_to_add], dim=1) # (1, f1+k, c1, c2)
+ else:
+ A_new = A # 没有需要添加的
+
+ return idx_to_add, A_new, min_idx
+
+ @torch.no_grad()
+ def process_input(self, images):
+ batch_size = self.batch_size
+ device = self.device
+ temporal_window_size = self.temporal_window_size
+ temporal_adaptive_step = self.temporal_adaptive_step
+
+ tgt_cond_tensor = self.fast_resize(images, 256, 256)
+ tgt_cond_tensor = tgt_cond_tensor / 2 + 0.5
+
+ if self.first_frame:
+ mot_bbox_param, kps_ref, kps_frame1, kps_dri = self.pose_encoder.interpolate_kps_online(self.ref_cond_tensor, tgt_cond_tensor, num_interp=12+1)
+ self.kps_ref = kps_ref
+ self.kps_frame1 = kps_frame1
+ else:
+ mot_bbox_param, kps_dri = self.pose_encoder.get_kps(self.kps_ref, self.kps_frame1, tgt_cond_tensor)
+
+ keypoints = draw_keypoints(mot_bbox_param, device=device)
+ boxes = get_boxes(kps_dri)
+ keypoints = rearrange(keypoints.unsqueeze(2), 'f c b h w -> b c f h w')
+ keypoints = keypoints.to(device=device, dtype=self.pose_guider.dtype)
+
+ if self.first_frame:
+ ref_box = get_boxes(mot_bbox_param[:1])
+ ref_face = self.crop_face_tensor(self.ref_image_tensor, ref_box[0])
+ motion_face = [ref_face]
+ for i, frame in enumerate(images):
+ motion_face.append(self.crop_face_tensor(frame, boxes[i]))
+ pose_cond_tensor = torch.cat(motion_face, dim=0).transpose(0, 1)
+ pose_cond_tensor = pose_cond_tensor.unsqueeze(0)
+ # pose_cond_tensor = pose_cond_tensor.to(
+ # device=device, dtype=self.motion_encoder.dtype
+ # )
+ motion_hidden_states = self.motion_encoder(pose_cond_tensor)
+ ref_motion = motion_hidden_states[:, :1]
+ dri_motion = motion_hidden_states[:, 1:]
+
+ init_motion_hidden_states = self.interpolate_tensors(ref_motion, dri_motion[:,:1], num=12+1)[:,:-1]
+ for i in range(temporal_adaptive_step-1):
+ l = i * temporal_window_size
+ r = (i+1) * temporal_window_size
+ self.motion_pile.append(init_motion_hidden_states[:,l:r])
+ self.motion_pile.append(dri_motion)
+
+ self.motion_bank = ref_motion
+ else:
+ motion_face = []
+ for i, frame in enumerate(images):
+ motion_face.append(self.crop_face_tensor(frame, boxes[i]))
+ pose_cond_tensor = torch.cat(motion_face, dim=0).transpose(0, 1)
+ pose_cond_tensor = pose_cond_tensor.unsqueeze(0)
+ motion_hidden_states = self.motion_encoder(pose_cond_tensor)
+ self.motion_pile.append(motion_hidden_states)
+
+ pose_fea = self.pose_guider(keypoints)
+ if self.first_frame:
+ for i in range(temporal_adaptive_step):
+ l = i * temporal_window_size
+ r = (i+1) * temporal_window_size
+ self.pose_pile.append(pose_fea[:,:,l:r])
+ self.first_frame = False
+ else:
+ self.pose_pile.append(pose_fea)
+
+ latents = self.ref_image_latents.unsqueeze(2).repeat(1, 1, temporal_window_size, 1, 1)
+ noise = torch.randn_like(latents)
+ latents = self.scheduler.add_noise(latents, noise, self.timesteps[:1])
+ self.latents_pile.append(latents)
+
+ jump = 1
+ motion_hidden_state = torch.cat(list(self.motion_pile), dim=1)
+ pose_cond_fea=torch.cat(list(self.pose_pile), dim=2)
+
+ idx_to_add = []
+ if self.count > 8:
+ idx_to_add, self.motion_bank, idx_his = self.calculate_dis(self.motion_bank, motion_hidden_state, threshold=17.)
+
+ latents_model_input = torch.cat(list(self.latents_pile), dim=2)
+ for j in range(jump):
+ timesteps = reversed(self.timesteps[j::jump]).repeat_interleave(temporal_window_size, dim=0)
+ timesteps = torch.stack([timesteps] * batch_size)#.to(device)
+ timesteps = rearrange(timesteps, 'b f -> (b f)')
+ noise_pred = self.denoising_unet(
+ latents_model_input,
+ timesteps,
+ encoder_hidden_states=[self.encoder_hidden_states,
+ motion_hidden_state],
+ pose_cond_fea=pose_cond_fea,
+ return_dict=False,
+ )[0]
+
+ clip_length = noise_pred.shape[2]
+ mid_noise_pred = rearrange(noise_pred, 'b c f h w -> (b f) c h w')
+ mid_latents = rearrange(latents_model_input, 'b c f h w -> (b f) c h w')
+ latents_model_input, pred_original_sample = self.scheduler.step(
+ mid_noise_pred, timesteps, mid_latents, generator=self.generator, return_dict=False
+ )
+ latents_model_input = rearrange(latents_model_input, '(b f) c h w -> b c f h w', f=clip_length)
+ pred_original_sample = rearrange(pred_original_sample, '(b f) c h w -> b c f h w', f=clip_length)
+ latents_model_input = torch.cat([
+ pred_original_sample[:,:,:temporal_window_size],
+ latents_model_input[:,:,temporal_window_size:]], dim=2)
+ latents_model_input = latents_model_input.to(dtype=self.dtype)
+
+ if len(idx_to_add) > 0 and self.num_khf < 3:
+ self.reference_control_writer.clear()
+ self.reference_unet(
+ pred_original_sample[:,:,0].to(self.reference_unet.dtype),
+ torch.zeros((batch_size,),dtype=self.dtype,device=self.reference_unet.device),
+ encoder_hidden_states=self.encoder_hidden_states,
+ return_dict=False,
+ )
+ self.reference_control_reader.update_hkf(self.reference_control_writer)
+ print('add_keyframes')
+ self.num_khf += 1
+
+
+ for i in range(len(self.latents_pile)):
+ self.latents_pile[i] = latents_model_input[:, :, i * temporal_adaptive_step : (i + 1) * temporal_adaptive_step, :, :]
+
+ self.pose_pile.popleft()
+ self.motion_pile.popleft()
+ latents = self.latents_pile.popleft()
+ latents = 1 / 0.18215 * latents
+ latents = rearrange(latents, "b c f h w -> (b f) c h w")
+ video = self.vae.decode(latents).sample
+ video = rearrange(video, "b c h w -> b h w c")
+ video = (video / 2 + 0.5).clamp(0, 1)
+ video = video.cpu().numpy()
+ self.count += 1
+ return video
\ No newline at end of file
diff --git a/src/wrapper_trt.py b/src/wrapper_trt.py
new file mode 100644
index 0000000..fa1b534
--- /dev/null
+++ b/src/wrapper_trt.py
@@ -0,0 +1,336 @@
+from omegaconf import OmegaConf
+import os
+import torch
+import numpy as np
+from PIL import Image
+import time
+import gc
+import cv2
+from src.models.unet_2d_condition import UNet2DConditionModel
+from src.models.pose_guider import PoseGuider
+from src.models.motion_encoder.encoder import MotEncoder
+from src.models.unet_3d import UNet3DConditionModel
+from src.models.mutual_self_attention import ReferenceAttentionControl
+from src.scheduler.scheduler_ddim import DDIMScheduler
+from src.liveportrait.motion_extractor import MotionExtractor
+from diffusers import AutoencoderKL
+from diffusers.image_processor import VaeImageProcessor
+from transformers import CLIPVisionModelWithProjection, CLIPImageProcessor
+from collections import deque
+from threading import Lock, Thread
+from torchvision import transforms as T
+from einops import rearrange
+from src.utils.util import draw_keypoints, get_boxes
+import torch.nn.functional as F
+from src.modeling.engine_model import EngineModel
+
+def map_device(device_or_str):
+ return device_or_str if isinstance(device_or_str, torch.device) else torch.device(device_or_str)
+
+class PersonaLive:
+ def __init__(self, config_path, device=None):
+ cfg = OmegaConf.load(config_path)
+ if(device is None):
+ self.device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+ else:
+ self.device = map_device(device)
+
+ self.temporal_adaptive_step = cfg.temporal_adaptive_step
+ self.temporal_window_size = cfg.temporal_window_size
+
+ if cfg.dtype == "fp16":
+ self.numpy_dtype = np.float16
+ self.dtype = torch.float16
+ elif cfg.dtype == "fp32":
+ self.numpy_dtype = np.float32
+ self.dtype = torch.float32
+
+ infer_config = OmegaConf.load(cfg.inference_config)
+ sched_kwargs = OmegaConf.to_container(
+ infer_config.noise_scheduler_kwargs
+ )
+
+ self.num_inference_steps = cfg.num_inference_steps
+
+ # initialize models
+ self.pose_guider = PoseGuider().to(device=self.device, dtype=self.dtype)
+ pose_guider_state_dict = torch.load(cfg.pose_guider_path, map_location="cpu")
+ self.pose_guider.load_state_dict(pose_guider_state_dict)
+ del pose_guider_state_dict
+
+ self.motion_encoder = MotEncoder().to(dtype=self.dtype, device=self.device).eval()
+ motion_encoder_state_dict = torch.load(cfg.motion_encoder_path, map_location="cpu")
+ self.motion_encoder.load_state_dict(motion_encoder_state_dict)
+ del motion_encoder_state_dict
+
+ self.pose_encoder = MotionExtractor(num_kp=21).to(device=self.device, dtype=self.dtype).eval()
+ pose_encoder_state_dict = torch.load(cfg.pose_encoder_path, map_location="cpu")
+ self.pose_encoder.load_state_dict(pose_encoder_state_dict, strict=False)
+ del pose_encoder_state_dict
+
+ self.reference_unet = UNet2DConditionModel.from_pretrained(
+ cfg.pretrained_base_model_path,
+ subfolder="unet",
+ ).to(dtype=self.dtype, device=self.device)
+ reference_unet_state_dict = torch.load(cfg.reference_unet_weight_path, map_location="cpu")
+ self.reference_unet.load_state_dict(reference_unet_state_dict)
+ del reference_unet_state_dict
+
+ self.reference_control_writer = ReferenceAttentionControl(
+ self.reference_unet,
+ do_classifier_free_guidance=False,
+ mode="write",
+ batch_size=cfg.batch_size,
+ fusion_blocks="full",
+ )
+
+ self.vae = AutoencoderKL.from_pretrained(cfg.vae_model_path).to(
+ device=self.device, dtype=self.dtype
+ )
+ self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(
+ cfg.image_encoder_path,
+ ).to(device=self.device, dtype=self.dtype)
+
+ #----------------------- TensorRT -----------------------#
+ self.unet_work = EngineModel(engine_file_path=cfg.tensorrt_target_model, device_int=self.device.index)
+ self.unet_work.bind({
+ "motion_hidden_states_out": "motion_hidden_states",
+ "pose_cond_fea_out": "pose_cond_fea",
+ "latents" : "sample",
+ })
+ #------------------------------------------------------------#
+
+ # miscellaneous
+ self.scheduler = DDIMScheduler(**sched_kwargs)
+ timesteps = torch.tensor([0, 333, 666, 999], device=self.device)
+ self.timesteps = timesteps.repeat_interleave(cfg.temporal_window_size, dim=0).long()
+ self.scheduler.set_step_length(333)
+
+ self.batch_size = cfg.batch_size
+ self.vae_scale_factor = 8
+ self.ref_image_processor = VaeImageProcessor(
+ vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True
+ )
+ self.clip_image_processor = CLIPImageProcessor()
+ self.cond_image_processor = VaeImageProcessor(
+ vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True, do_normalize=True)
+
+ self.first_frame = True
+ self.motion_bank = None
+ self.count = 0
+ self.num_khf = 0
+
+ self.cfg = cfg
+ self.reference_hidden_states_names = ["d00", "d01", "d10", "d11",
+ "d20", "d21", "m", "u10", "u11", "u12",
+ "u20", "u21", "u22", "u30", "u31", "u32"]
+ torch.cuda.empty_cache()
+
+ self.enable_xformers_memory_efficient_attention()
+
+ def enable_xformers_memory_efficient_attention(self):
+ self.reference_unet.enable_xformers_memory_efficient_attention()
+
+ def fast_resize(self, images, target_width, target_height) -> torch.Tensor:
+ tgt_cond_tensor = F.interpolate(
+ images,
+ size=(target_width, target_height),
+ mode="bilinear",
+ align_corners=False,
+ )
+ return tgt_cond_tensor
+
+ @torch.no_grad()
+ def fuse_reference(self, ref_image): # pil input
+ clip_image = self.clip_image_processor.preprocess(
+ ref_image, return_tensors="pt"
+ ).pixel_values
+ ref_image_tensor = self.ref_image_processor.preprocess(
+ ref_image, height=self.cfg.reference_image_height, width=self.cfg.reference_image_width
+ ) # (bs, c, width, height)
+ clip_image_embeds = self.image_encoder(
+ clip_image.to(self.image_encoder.device, dtype=self.image_encoder.dtype)
+ ).image_embeds
+ encoder_hidden_states = clip_image_embeds.unsqueeze(1)
+ self.unet_work.prefill(encoder_hidden_states = encoder_hidden_states)
+ self.encoder_hidden_states = encoder_hidden_states
+
+ ref_image_tensor = ref_image_tensor.to(
+ dtype=self.vae.dtype, device=self.vae.device
+ )
+ self.ref_image_tensor = ref_image_tensor.squeeze(0)
+ ref_image_latents = self.vae.encode(ref_image_tensor).latent_dist.mean
+ ref_image_latents = ref_image_latents * 0.18215 # (b, 4, h, w)
+ self.reference_unet(
+ ref_image_latents.to(self.reference_unet.device),
+ torch.zeros((self.batch_size,),dtype=self.dtype,device=self.reference_unet.device),
+ encoder_hidden_states=encoder_hidden_states,
+ return_dict=False,
+ )
+ self.reference_hidden_states = self.reference_control_writer.output()
+ self.unet_work.prefill(**{name: self.reference_hidden_states[name] for name in self.reference_hidden_states_names})
+
+ ref_cond_tensor = self.cond_image_processor.preprocess(
+ ref_image, height=256, width=256
+ ).to(device=self.device, dtype=self.pose_encoder.dtype) # (1, c, h, w)
+ self.ref_cond_tensor = ref_cond_tensor / 2 + 0.5 # to [0, 1]
+ self.ref_image_latents = ref_image_latents.unsqueeze(2).repeat(1, 1, self.temporal_window_size, 1, 1)
+
+ padding_num = (self.temporal_adaptive_step - 1) * self.temporal_window_size
+ init_latents = ref_image_latents.unsqueeze(2).repeat(1, 1, padding_num, 1, 1)
+ noise = torch.randn_like(init_latents)
+ self.noisy_latents_first = self.scheduler.add_noise(init_latents, noise, self.timesteps[:padding_num])
+
+ def crop_face(self, image_pil, boxes):
+ image = np.array(image_pil)
+
+ left, top, right, bot = boxes
+
+ face_patch = image[int(top) : int(bot), int(left) : int(right)]
+ face_patch = Image.fromarray(face_patch).convert("RGB")
+ return face_patch
+
+ def crop_face_tensor(self, image_tensor, boxes):
+ left, top, right, bot = boxes
+ left, top, right, bottom = map(int, (left, top, right, bot))
+
+ face_patch = image_tensor[:, top:bottom, left:right]
+ face_patch = F.interpolate(
+ face_patch.unsqueeze(0),
+ size=(224, 224),
+ mode="bilinear",
+ align_corners=False,
+ )
+ return face_patch
+
+ def interpolate_tensors(self, a: torch.Tensor, b: torch.Tensor, num: int = 10) -> torch.Tensor:
+ """
+ 在张量 a 和 b 之间线性插值。
+ 输入 shape: (B, 1, D1, D2, ...)
+ 输出 shape: (B, num, D1, D2, ...)
+ """
+ if a.shape != b.shape:
+ raise ValueError(f"Shape mismatch: a.shape={a.shape}, b.shape={b.shape}")
+
+ B, _, *rest = a.shape
+ # 插值系数 (num,) → reshape 成 (1, num, 1, 1, ...)
+ alphas = torch.linspace(0, 1, num, device=a.device, dtype=a.dtype)
+ view_shape = (1, num) + (1,) * len(rest)
+ alphas = alphas.view(view_shape) # (1, num, 1, 1, ...)
+
+ # 插值 (B, num, D1, D2, ...)
+ result = (1 - alphas) * a + alphas * b
+ return result
+
+ def calculate_dis(self, A, B, threshold=10.):
+ """
+ A: (b, f1, c1, c2) bank
+ B: (b, f2, c1, c2) new data
+ """
+
+ A_flat = A.view(A.size(1), -1).clone()
+ B_flat = B.view(B.size(1), -1).clone()
+
+ dist = torch.cdist(B_flat.to(torch.float32), A_flat.to(torch.float32), p=2)
+
+ min_dist, min_idx = dist.min(dim=1) # (f2,)
+
+ idx_to_add = torch.nonzero(min_dist[:1] > threshold, as_tuple=False).squeeze(1).tolist()
+
+ if len(idx_to_add) > 0: # 有需要添加的元素
+ B_to_add = B[:, idx_to_add] # (1, k, c1, c2)
+ A_new = torch.cat([A, B_to_add], dim=1) # (1, f1+k, c1, c2)
+ else:
+ A_new = A # 没有需要添加的
+
+ return idx_to_add, A_new, min_idx
+
+ @torch.no_grad()
+ def process_input(self, images):
+ batch_size = self.batch_size
+ device = self.device
+
+ tgt_cond_tensor = self.fast_resize(images, 256, 256)
+ tgt_cond_tensor = tgt_cond_tensor / 2 + 0.5
+
+ if self.first_frame:
+ mot_bbox_param, kps_ref, kps_frame1, kps_dri = self.pose_encoder.interpolate_kps_online(self.ref_cond_tensor, tgt_cond_tensor, num_interp=12+1)
+ self.kps_ref = kps_ref
+ self.kps_frame1 = kps_frame1
+ else:
+ mot_bbox_param, kps_dri = self.pose_encoder.get_kps(self.kps_ref, self.kps_frame1, tgt_cond_tensor)
+
+ keypoints = draw_keypoints(mot_bbox_param, device=device)
+ boxes = get_boxes(kps_dri)
+ keypoints = rearrange(keypoints.unsqueeze(2), 'f c b h w -> b c f h w')
+ keypoints = keypoints.to(device=device, dtype=self.pose_guider.dtype)
+
+ if self.first_frame:
+ pose_cond_fea = self.pose_guider(keypoints[:,:, :12])
+ pose = keypoints[:,:,12:]
+
+ ref_box = get_boxes(mot_bbox_param[:1])
+ ref_face = self.crop_face_tensor(self.ref_image_tensor, ref_box[0])
+ motion_face = [ref_face]
+ for i, frame in enumerate(images):
+ motion_face.append(self.crop_face_tensor(frame, boxes[i]))
+ motion_cond_tensor = torch.cat(motion_face, dim=0).transpose(0, 1)
+ motion_cond_tensor = motion_cond_tensor.unsqueeze(0)
+ motion = motion_cond_tensor[:,:,1:]
+ motion_hidden_states = self.motion_encoder(motion_cond_tensor[:,:,:2])
+ ref_motion = motion_hidden_states[:, :1]
+ dri_motion = motion_hidden_states[:, 1:]
+
+ motion_hidden_states = self.interpolate_tensors(ref_motion, dri_motion[:,:1], num=12+1)[:,:-1]
+ self.motion_bank = ref_motion
+
+ latents = self.ref_image_latents
+ noise = torch.randn_like(latents)
+ latents = self.scheduler.add_noise(latents, noise, self.timesteps[-1:])
+ sample = torch.cat([self.noisy_latents_first, latents], dim=2)
+
+ self.unet_work.prefill(latents=sample)
+ self.unet_work.prefill(motion_hidden_states_out=motion_hidden_states)
+ self.unet_work.prefill(pose_cond_fea_out=pose_cond_fea)
+ self.first_frame = False
+ else:
+ pose = keypoints
+
+ motion_face = []
+ for i, frame in enumerate(images):
+ motion_face.append(self.crop_face_tensor(frame, boxes[i]))
+ motion = torch.cat(motion_face, dim=0).transpose(0, 1)
+ motion = motion.unsqueeze(0)
+
+ motion = motion.to(dtype = self.dtype)
+ latents = self.ref_image_latents
+ noise = torch.randn_like(latents)
+ new_noise = self.scheduler.add_noise(latents, noise, self.timesteps[-1:])
+
+ results = self.unet_work(output_list=["pred_video", "motion_out", "latent_first"], return_tensor=True, pose=pose, motion=motion, new_noise=new_noise)
+ video = results['pred_video'].cpu().numpy()
+ motion_out = results['motion_out']
+
+ idx_to_add = []
+ if self.count > 8:
+ idx_to_add, self.motion_bank, idx_his = self.calculate_dis(self.motion_bank, motion_out, threshold=17.)
+
+ if len(idx_to_add) > 0 and self.num_khf < 3:
+ latents_first = results['latent_first']
+ self.reference_control_writer.clear()
+ self.reference_unet(
+ latents_first.to(self.reference_unet.dtype),
+ torch.zeros((batch_size,),dtype=self.dtype,device=self.reference_unet.device),
+ encoder_hidden_states=self.encoder_hidden_states,
+ return_dict=False,
+ )
+ reference_hidden_states = self.reference_control_writer.output()
+ for name in self.reference_hidden_states_names:
+ self.reference_hidden_states[name] = torch.cat([self.reference_hidden_states[name], reference_hidden_states[name]], dim=1)
+
+ self.unet_work.prefill(**{name: self.reference_hidden_states[name] for name in self.reference_hidden_states_names})
+ print('add_keyframes')
+ self.num_khf += 1
+
+ self.count += 1
+ return video
\ No newline at end of file