[CVPR 2026] PersonaLive! : Expressive Portrait Image Animation for Live Streaming
update src
Zhiyuan Li12c7f7d9f399
diff --git a/README.md b/README.mdindex 0a6d66f..42d3792 100644--- a/README.md+++ b/README.md@@ -31,4 +31,4 @@ If you find PersonaLive useful for your research, welcome to 🌟 this repo and## ❤️ Acknowledgement-This project is based on [X-NeMo](https://byteaigc.github.io/X-Portrait2/), [StreamDiffusion](https://github.com/cumulo-autumn/StreamDiffusion) and [RAIN](https://pscgylotti.github.io/pages/RAIN/), thanks to their invaluable contributions.+This project is based on [Moore-AnimateAnyone](https://github.com/MooreThreads/Moore-AnimateAnyone), [X-NeMo](https://byteaigc.github.io/X-Portrait2/), [StreamDiffusion](https://github.com/cumulo-autumn/StreamDiffusion), [RAIN](https://pscgylotti.github.io/pages/RAIN/) and [LivePortrait](https://github.com/KlingTeam/LivePortrait), thanks to their invaluable contributions.diff --git a/pretrained_weights/onnx/readme.txt b/pretrained_weights/onnx/readme.txtnew file mode 100644index 0000000..84baf61--- /dev/null+++ b/pretrained_weights/onnx/readme.txt@@ -0,0 +1 @@+onnx models\ No newline at end of filediff --git a/pretrained_weights/personalive/readme.txt b/pretrained_weights/personalive/readme.txtnew file mode 100644index 0000000..a78b831--- /dev/null+++ b/pretrained_weights/personalive/readme.txt@@ -0,0 +1 @@+pre_trained models\ No newline at end of filediff --git a/pretrained_weights/tensorrt/readme.txt b/pretrained_weights/tensorrt/readme.txtnew file mode 100644index 0000000..33bdb41--- /dev/null+++ b/pretrained_weights/tensorrt/readme.txt@@ -0,0 +1 @@+tensorrt models\ No newline at end of filediff --git a/src/liveportrait/camera.py b/src/liveportrait/camera.pynew file mode 100644index 0000000..a3dd942--- /dev/null+++ b/src/liveportrait/camera.py@@ -0,0 +1,73 @@+# coding: utf-8++"""+functions for processing and transforming 3D facial keypoints+"""++import numpy as np+import torch+import torch.nn.functional as F++PI = np.pi+++def headpose_pred_to_degree(pred):+ """+ pred: (bs, 66) or (bs, 1) or others+ """+ if pred.ndim > 1 and pred.shape[1] == 66:+ # NOTE: note that the average is modified to 97.5+ device = pred.device+ idx_tensor = [idx for idx in range(0, 66)]+ idx_tensor = torch.FloatTensor(idx_tensor).to(device)+ pred = F.softmax(pred, dim=1)+ degree = torch.sum(pred*idx_tensor, axis=1) * 3 - 97.5++ return degree++ return pred+++def get_rotation_matrix(pitch_, yaw_, roll_):+ """ the input is in degree+ """+ # transform to radian+ pitch = pitch_ / 180 * PI+ yaw = yaw_ / 180 * PI+ roll = roll_ / 180 * PI++ device = pitch.device++ if pitch.ndim == 1:+ pitch = pitch.unsqueeze(1)+ if yaw.ndim == 1:+ yaw = yaw.unsqueeze(1)+ if roll.ndim == 1:+ roll = roll.unsqueeze(1)++ # calculate the euler matrix+ bs = pitch.shape[0]+ ones = torch.ones([bs, 1]).to(device)+ zeros = torch.zeros([bs, 1]).to(device)+ x, y, z = pitch, yaw, roll++ rot_x = torch.cat([+ ones, zeros, zeros,+ zeros, torch.cos(x), -torch.sin(x),+ zeros, torch.sin(x), torch.cos(x)+ ], dim=1).reshape([bs, 3, 3])++ rot_y = torch.cat([+ torch.cos(y), zeros, torch.sin(y),+ zeros, ones, zeros,+ -torch.sin(y), zeros, torch.cos(y)+ ], dim=1).reshape([bs, 3, 3])++ rot_z = torch.cat([+ torch.cos(z), -torch.sin(z), zeros,+ torch.sin(z), torch.cos(z), zeros,+ zeros, zeros, ones+ ], dim=1).reshape([bs, 3, 3])++ rot = rot_z @ rot_y @ rot_x+ return rot.permute(0, 2, 1) # transposediff --git a/src/liveportrait/convnextv2.py b/src/liveportrait/convnextv2.pynew file mode 100644index 0000000..8cb7fc9--- /dev/null+++ b/src/liveportrait/convnextv2.py@@ -0,0 +1,216 @@+# coding: utf-8++"""+This moudle is adapted to the ConvNeXtV2 version for the extraction of implicit keypoints, poses, and expression deformation.+"""++import torch+import torch.nn as nn+# from timm.models.layers import trunc_normal_, DropPath+from src.liveportrait.util import LayerNorm, DropPath, trunc_normal_, GRN+from einops import rearrange++__all__ = ['convnextv2_tiny']+++class Block(nn.Module):+ """ ConvNeXtV2 Block.++ Args:+ dim (int): Number of input channels.+ drop_path (float): Stochastic depth rate. Default: 0.0+ """++ def __init__(self, dim, drop_path=0.):+ super().__init__()+ self.dwconv = nn.Conv2d(dim, dim, kernel_size=7, padding=3, groups=dim) # depthwise conv+ self.norm = LayerNorm(dim, eps=1e-6)+ self.pwconv1 = nn.Linear(dim, 4 * dim) # pointwise/1x1 convs, implemented with linear layers+ self.act = nn.GELU()+ self.grn = GRN(4 * dim)+ self.pwconv2 = nn.Linear(4 * dim, dim)+ self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()++ def forward(self, x):+ input = x+ x = self.dwconv(x)+ x = x.permute(0, 2, 3, 1) # (N, C, H, W) -> (N, H, W, C)+ x = self.norm(x)+ x = self.pwconv1(x)+ x = self.act(x)+ x = self.grn(x)+ x = self.pwconv2(x)+ x = x.permute(0, 3, 1, 2) # (N, H, W, C) -> (N, C, H, W)+ x = input + self.drop_path(x)+ return x+++class ConvNeXtV2(nn.Module):+ """ ConvNeXt V2++ Args:+ in_chans (int): Number of input image channels. Default: 3+ num_classes (int): Number of classes for classification head. Default: 1000+ depths (tuple(int)): Number of blocks at each stage. Default: [3, 3, 9, 3]+ dims (int): Feature dimension at each stage. Default: [96, 192, 384, 768]+ drop_path_rate (float): Stochastic depth rate. Default: 0.+ head_init_scale (float): Init scaling value for classifier weights and biases. Default: 1.+ """++ def __init__(+ self,+ in_chans=3,+ depths=[3, 3, 9, 3],+ dims=[96, 192, 384, 768],+ drop_path_rate=0.,+ **kwargs+ ):+ super().__init__()+ self.depths = depths+ self.downsample_layers = nn.ModuleList() # stem and 3 intermediate downsampling conv layers+ stem = nn.Sequential(+ nn.Conv2d(in_chans, dims[0], kernel_size=4, stride=4),+ LayerNorm(dims[0], eps=1e-6, data_format="channels_first")+ )+ self.downsample_layers.append(stem)+ for i in range(3):+ downsample_layer = nn.Sequential(+ LayerNorm(dims[i], eps=1e-6, data_format="channels_first"),+ nn.Conv2d(dims[i], dims[i+1], kernel_size=2, stride=2),+ )+ self.downsample_layers.append(downsample_layer)++ self.stages = nn.ModuleList() # 4 feature resolution stages, each consisting of multiple residual blocks+ dp_rates = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]+ cur = 0+ for i in range(4):+ stage = nn.Sequential(+ *[Block(dim=dims[i], drop_path=dp_rates[cur + j]) for j in range(depths[i])]+ )+ self.stages.append(stage)+ cur += depths[i]++ self.norm = nn.LayerNorm(dims[-1], eps=1e-6) # final norm layer++ # NOTE: the output semantic items+ num_bins = kwargs.get('num_bins', 66)+ num_kp = kwargs.get('num_kp', 24) # the number of implicit keypoints+ self.fc_kp = nn.Linear(dims[-1], 3 * num_kp) # implicit keypoints++ # print('dims[-1]: ', dims[-1])+ self.fc_scale = nn.Linear(dims[-1], 1) # scale+ self.fc_pitch = nn.Linear(dims[-1], num_bins) # pitch bins+ self.fc_yaw = nn.Linear(dims[-1], num_bins) # yaw bins+ self.fc_roll = nn.Linear(dims[-1], num_bins) # roll bins+ self.fc_t = nn.Linear(dims[-1], 3) # translation+ self.fc_exp = nn.Linear(dims[-1], 3 * num_kp) # expression / delta++ def _init_weights(self, m):+ if isinstance(m, (nn.Conv2d, nn.Linear)):+ trunc_normal_(m.weight, std=.02)+ nn.init.constant_(m.bias, 0)++ def forward_features(self, x):+ for i in range(4):+ x = self.downsample_layers[i](x)+ x = self.stages[i](x)+ return self.norm(x.mean([-2, -1])) # global average pooling, (N, C, H, W) -> (N, C)++ def forward(self, x):+ x = self.forward_features(x)++ # implicit keypoints+ kp = self.fc_kp(x)++ # pose and expression deformation+ pitch = self.fc_pitch(x)+ yaw = self.fc_yaw(x)+ roll = self.fc_roll(x)+ t = self.fc_t(x)+ # exp = self.fc_exp(x)+ scale = self.fc_scale(x)++ ret_dct = {+ 'pitch': pitch,+ 'yaw': yaw,+ 'roll': roll,+ 't': t,+ # 'exp': exp,+ 'scale': scale,++ 'kp': kp, # canonical keypoint+ }++ return ret_dct+++def convnextv2_tiny(**kwargs):+ model = ConvNeXtV2(depths=[3, 3, 9, 3], dims=[96, 192, 384, 768], **kwargs)+ return model++class ConvNeXt(nn.Module):+ """ ConvNeXt V2++ Args:+ in_chans (int): Number of input image channels. Default: 3+ num_classes (int): Number of classes for classification head. Default: 1000+ depths (tuple(int)): Number of blocks at each stage. Default: [3, 3, 9, 3]+ dims (int): Feature dimension at each stage. Default: [96, 192, 384, 768]+ drop_path_rate (float): Stochastic depth rate. Default: 0.+ head_init_scale (float): Init scaling value for classifier weights and biases. Default: 1.+ """++ def __init__(+ self,+ in_chans=3,+ depths=[3, 3, 9, 3],+ dims=[96, 192, 384, 768],+ drop_path_rate=0.,+ **kwargs+ ):+ super().__init__()+ self.depths = depths+ self.downsample_layers = nn.ModuleList() # stem and 3 intermediate downsampling conv layers+ stem = nn.Sequential(+ nn.Conv2d(in_chans, dims[0], kernel_size=4, stride=4),+ LayerNorm(dims[0], eps=1e-6, data_format="channels_first")+ )+ self.downsample_layers.append(stem)+ for i in range(3):+ downsample_layer = nn.Sequential(+ LayerNorm(dims[i], eps=1e-6, data_format="channels_first"),+ nn.Conv2d(dims[i], dims[i+1], kernel_size=2, stride=2),+ )+ self.downsample_layers.append(downsample_layer)++ self.stages = nn.ModuleList() # 4 feature resolution stages, each consisting of multiple residual blocks+ dp_rates = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]+ cur = 0+ for i in range(4):+ stage = nn.Sequential(+ *[Block(dim=dims[i], drop_path=dp_rates[cur + j]) for j in range(depths[i])]+ )+ self.stages.append(stage)+ cur += depths[i]++ self.norm = nn.LayerNorm(dims[-1], eps=1e-6) # final norm layer++ def _init_weights(self, m):+ if isinstance(m, (nn.Conv2d, nn.Linear)):+ trunc_normal_(m.weight, std=.02)+ nn.init.constant_(m.bias, 0)++ def forward_features(self, x):+ for i in range(4):+ x = self.downsample_layers[i](x)+ x = self.stages[i](x)+ return self.norm(x.mean([-2, -1])) # global average pooling, (N, C, H, W) -> (N, C)++ def forward(self, x):+ x = self.forward_features(x)+ return x+++def convnextv2(**kwargs):+ model = ConvNeXt(depths=[3, 3, 9, 3], dims=[96, 192, 384, 768], **kwargs)+ return model\ No newline at end of filediff --git a/src/liveportrait/motion_extractor.py b/src/liveportrait/motion_extractor.pynew file mode 100644index 0000000..ace8477--- /dev/null+++ b/src/liveportrait/motion_extractor.py@@ -0,0 +1,212 @@+# coding: utf-8++"""+Motion extractor(M), which directly predicts the canonical keypoints, head pose and expression deformation of the input image+"""++from torch import nn+import torch+from diffusers.models.modeling_utils import ModelMixin+from src.liveportrait.convnextv2 import convnextv2_tiny+from src.liveportrait.util import filter_state_dict+from src.liveportrait.camera import headpose_pred_to_degree, get_rotation_matrix++model_dict = {+ 'convnextv2_tiny': convnextv2_tiny,+}+++class MotionExtractor(ModelMixin):+ def __init__(self, **kwargs):+ super(MotionExtractor, self).__init__()++ # default is convnextv2_base+ backbone = kwargs.get('backbone', 'convnextv2_tiny')+ self.detector = model_dict.get(backbone)(**kwargs)+ self.register_buffer('idx_tensor', torch.arange(66, dtype=torch.float32))++ def headpose_pred_to_degree(self, pred):+ """+ pred: (bs, 66) or (bs, 1) or others+ """+ if pred.ndim > 1 and pred.shape[1] == 66:+ # NOTE: note that the average is modified to 97.5+ prob = torch.nn.functional.softmax(pred, dim=1)+ degree = torch.matmul(prob, self.idx_tensor)+ degree = degree * 3 - 97.5++ return degree++ return pred++ def load_pretrained(self, init_path: str):+ if init_path not in (None, ''):+ state_dict = torch.load(init_path, map_location=lambda storage, loc: storage)['model']+ state_dict = filter_state_dict(state_dict, remove_name='head')+ ret = self.detector.load_state_dict(state_dict, strict=False)+ print(f'Load pretrained model from {init_path}, ret: {ret}')++ def forward(self, x):+ kp_info = self.detector(x)+ return self.get_kp(kp_info)++ def get_kp(self, kp_info):+ bs = kp_info['kp'].shape[0]++ angles_raw = torch.cat([kp_info['pitch'], kp_info['yaw'], kp_info['roll']], dim=0) # (3, 66)+ angles_deg = self.headpose_pred_to_degree(angles_raw)[:, None] # (B, 3)+ pitch, yaw, roll = torch.chunk(angles_deg, chunks=3, dim=0)+++ kp = kp_info['kp'].reshape(bs, -1, 3) # BxNx3+ t, scale = kp_info['t'], kp_info['scale']++ rot_mat = get_rotation_matrix(pitch, yaw, roll).to(self.dtype) # (bs, 3, 3)++ if kp.ndim == 2:+ num_kp = kp.shape[1] // 3 # Bx(num_kpx3)+ else:+ num_kp = kp.shape[1] # Bxnum_kpx3++ # Eqn.2: s * (R * x_c,s) + t+ kp_transformed = kp.view(bs, num_kp, 3) @ rot_mat# + exp.view(bs, num_kp, 3)+ kp_transformed *= scale[..., None] # (bs, k, 3) * (bs, 1, 1) = (bs, k, 3)+ kp_transformed[:, :, 0:2] += t[:, None, 0:2] # remove z, only apply tx ty++ return kp_transformed++ def interpolate_tensors(self, a: torch.Tensor, b: torch.Tensor, num: int = 10) -> torch.Tensor:+ if a.shape != b.shape:+ raise ValueError(f"Shape mismatch: a.shape={a.shape}, b.shape={b.shape}")++ B, *rest = a.shape+ alphas = torch.linspace(0, 1, num, device=a.device, dtype=a.dtype)+ view_shape = (num,) + (1,) * len(rest)+ alphas = alphas.view(view_shape) # (1, num, 1, 1, ...)++ result = (1 - alphas) * a + alphas * b+ return result[:-1]++ def interpolate_kps(self, ref, motion, num_interp, t_scale=0.5, s_scale=0):+ kp1 = self.detector(ref.to(self.dtype))+ kp2_list = []+ for i in range(0, motion.shape[0], 256):+ motion_chunk = motion[i:i+256]+ kp2_chunk = self.detector(motion_chunk.to(self.dtype))+ kp2_list.append(kp2_chunk)+ kp2 = {}+ for key in kp2_list[0].keys():+ kp2[key] = torch.cat([kp2_chunk[key] for kp2_chunk in kp2_list], dim=0)++ angles_raw = torch.cat([kp1['pitch'], kp1['yaw'], kp1['roll']], dim=0) # (3, 66)+ angles_deg = self.headpose_pred_to_degree(angles_raw) # (B, 3)+ pitch_1, yaw_1, roll_1 = torch.chunk(angles_deg, chunks=3, dim=0)++ angles_raw = torch.cat([kp2['pitch'], kp2['yaw'], kp2['roll']], dim=0) # (3, 66)+ angles_deg = self.headpose_pred_to_degree(angles_raw) # (B, 3)+ pitch_2, yaw_2, roll_2 = torch.chunk(angles_deg, chunks=3, dim=0)++ pitch_interp = self.interpolate_tensors(pitch_1, pitch_2[:1], num_interp) # Bx(num_interp)x1+ yaw_interp = self.interpolate_tensors(yaw_1, yaw_2[:1], num_interp) # Bx(num_interp)x1+ roll_interp = self.interpolate_tensors(roll_1, roll_2[:1], num_interp) # Bx(num_interp)x1++ t_1 = kp1['t']+ t_2 = kp2['t']+ t_2 = (t_2 - t_2[0]) * t_scale + t_1+ t_interp = self.interpolate_tensors(t_1, t_2[:1], num_interp)++ s_1 = kp1['scale']+ s_2 = kp2['scale']+ s_2 = s_2 * s_scale + s_1+ s_interp = self.interpolate_tensors(s_1, s_2[:1], num_interp)++ kp = kp1['kp'].repeat(num_interp+motion.shape[0]-1, 1)++ kps_interp = {+ 'pitch': torch.cat([pitch_interp, pitch_2], dim=0),+ 'yaw': torch.cat([yaw_interp, yaw_2], dim=0),+ 'roll': torch.cat([roll_interp, roll_2], dim=0),+ 't': torch.cat([t_interp, t_2], dim=0),+ 'scale': torch.cat([s_interp, s_2], dim=0),+ 'kp': kp+ }++ kp_intrep = self.get_kp(kps_interp)++ return kp_intrep+++ def interpolate_kps_online(self, ref, motion, num_interp, t_scale=0.5, s_scale=0):+ kp1 = self.detector(ref.to(self.dtype))+ kp_frame1 = self.detector(motion[:1].to(self.dtype))+ kp2 = self.detector(motion.to(self.dtype))++ angles_raw = torch.cat([kp1['pitch'], kp1['yaw'], kp1['roll']], dim=0) # (3, 66)+ angles_deg = self.headpose_pred_to_degree(angles_raw) # (B, 3)+ pitch_1, yaw_1, roll_1 = torch.chunk(angles_deg, chunks=3, dim=0)++ angles_raw = torch.cat([kp2['pitch'], kp2['yaw'], kp2['roll']], dim=0) # (3, 66)+ angles_deg = self.headpose_pred_to_degree(angles_raw) # (B, 3)+ pitch_2, yaw_2, roll_2 = torch.chunk(angles_deg, chunks=3, dim=0)++ pitch_interp = self.interpolate_tensors(pitch_1, pitch_2[:1], num_interp) # Bx(num_interp)x1+ yaw_interp = self.interpolate_tensors(yaw_1, yaw_2[:1], num_interp) # Bx(num_interp)x1+ roll_interp = self.interpolate_tensors(roll_1, roll_2[:1], num_interp) # Bx(num_interp)x1++ t_1 = kp1['t']+ t_2 = kp2['t']+ t_2 = (t_2 - t_2[0]) * t_scale + t_1+ t_interp = self.interpolate_tensors(t_1, t_2[:1], num_interp)++ s_1 = kp1['scale']+ s_2 = kp2['scale']+ s_2 = s_2 * s_scale + s_1+ s_interp = self.interpolate_tensors(s_1, s_2[:1], num_interp)++ kp = kp1['kp'].repeat(num_interp+motion.shape[0]-1, 1)++ kps_interp = {+ 'pitch': torch.cat([pitch_interp, pitch_2], dim=0),+ 'yaw': torch.cat([yaw_interp, yaw_2], dim=0),+ 'roll': torch.cat([roll_interp, roll_2], dim=0),+ 't': torch.cat([t_interp, t_2], dim=0),+ 'scale': torch.cat([s_interp, s_2], dim=0),+ 'kp': kp+ }++ kp_intrep = self.get_kp(kps_interp)++ kp_dri = self.get_kp(kp2)++ return kp_intrep, kp1, kp_frame1, kp_dri++ def get_kps(self, kp_ref, kp_frame1, motion, t_scale=0.5, s_scale=0):+ kps_motion = self.detector(motion.to(self.dtype))++ kps_dri = self.get_kp(kps_motion)++ t_ref = kp_ref['t']+ t_frame1 = kp_frame1['t']+ t_motion = kps_motion['t']+ kps_motion['t'] = (t_motion - t_frame1) * t_scale + t_ref++ s_ref = kp_ref['scale']+ s_motion = kps_motion['scale']+ kps_motion['scale'] = s_motion * s_scale + s_ref+++ kps_motion['kp'] = kp_ref['kp'].repeat(motion.shape[0], 1)++ kps_motion = self.get_kp(kps_motion)++ return kps_motion, kps_dri++ def inference(self, ref, motion):+ kps_ref = self.detector(ref.to(self.dtype))+ kps_motion = self.detector(motion.to(self.dtype))+ kps_motion['kp'] = kps_ref['kp']++ kp_s = self.get_kp(kps_ref)+ kp_d = self.get_kp(kps_motion)++ return kp_s, kp_d\ No newline at end of filediff --git a/src/liveportrait/util.py b/src/liveportrait/util.pynew file mode 100644index 0000000..a58ce58--- /dev/null+++ b/src/liveportrait/util.py@@ -0,0 +1,492 @@+# coding: utf-8++"""+This file defines various neural network modules and utility functions, including convolutional and residual blocks,+normalizations, and functions for spatial transformation and tensor manipulation.+"""++from torch import nn+import torch.nn.functional as F+import torch+import torch.nn.utils.spectral_norm as spectral_norm+import math+import warnings+import collections.abc+from itertools import repeat++def kp2gaussian(kp, spatial_size, kp_variance):+ """+ Transform a keypoint into gaussian like representation+ """+ mean = kp++ coordinate_grid = make_coordinate_grid(spatial_size, mean)+ number_of_leading_dimensions = len(mean.shape) - 1+ shape = (1,) * number_of_leading_dimensions + coordinate_grid.shape+ coordinate_grid = coordinate_grid.view(*shape)+ repeats = mean.shape[:number_of_leading_dimensions] + (1, 1, 1, 1)+ coordinate_grid = coordinate_grid.repeat(*repeats)++ # Preprocess kp shape+ shape = mean.shape[:number_of_leading_dimensions] + (1, 1, 1, 3)+ mean = mean.view(*shape)++ mean_sub = (coordinate_grid - mean)++ out = torch.exp(-0.5 * (mean_sub ** 2).sum(-1) / kp_variance)++ return out+++def make_coordinate_grid(spatial_size, ref, **kwargs):+ d, h, w = spatial_size+ x = torch.arange(w).type(ref.dtype).to(ref.device)+ y = torch.arange(h).type(ref.dtype).to(ref.device)+ z = torch.arange(d).type(ref.dtype).to(ref.device)++ # NOTE: must be right-down-in+ x = (2 * (x / (w - 1)) - 1) # the x axis faces to the right+ y = (2 * (y / (h - 1)) - 1) # the y axis faces to the bottom+ z = (2 * (z / (d - 1)) - 1) # the z axis faces to the inner++ yy = y.view(1, -1, 1).repeat(d, 1, w)+ xx = x.view(1, 1, -1).repeat(d, h, 1)+ zz = z.view(-1, 1, 1).repeat(1, h, w)++ meshed = torch.cat([xx.unsqueeze_(3), yy.unsqueeze_(3), zz.unsqueeze_(3)], 3)++ return meshed+++class ConvT2d(nn.Module):+ """+ Upsampling block for use in decoder.+ """++ def __init__(self, in_features, out_features, kernel_size=3, stride=2, padding=1, output_padding=1):+ super(ConvT2d, self).__init__()++ self.convT = nn.ConvTranspose2d(in_features, out_features, kernel_size=kernel_size, stride=stride,+ padding=padding, output_padding=output_padding)+ self.norm = nn.InstanceNorm2d(out_features)++ def forward(self, x):+ out = self.convT(x)+ out = self.norm(out)+ out = F.leaky_relu(out)+ return out+++class ResBlock3d(nn.Module):+ """+ Res block, preserve spatial resolution.+ """++ def __init__(self, in_features, kernel_size, padding):+ super(ResBlock3d, self).__init__()+ self.conv1 = nn.Conv3d(in_channels=in_features, out_channels=in_features, kernel_size=kernel_size, padding=padding)+ self.conv2 = nn.Conv3d(in_channels=in_features, out_channels=in_features, kernel_size=kernel_size, padding=padding)+ self.norm1 = nn.BatchNorm3d(in_features, affine=True)+ self.norm2 = nn.BatchNorm3d(in_features, affine=True)++ def forward(self, x):+ out = self.norm1(x)+ out = F.relu(out)+ out = self.conv1(out)+ out = self.norm2(out)+ out = F.relu(out)+ out = self.conv2(out)+ out += x+ return out+++class UpBlock3d(nn.Module):+ """+ Upsampling block for use in decoder.+ """++ def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1):+ super(UpBlock3d, self).__init__()++ self.conv = nn.Conv3d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size,+ padding=padding, groups=groups)+ self.norm = nn.BatchNorm3d(out_features, affine=True)++ def forward(self, x):+ out = F.interpolate(x, scale_factor=(1, 2, 2))+ out = self.conv(out)+ out = self.norm(out)+ out = F.relu(out)+ return out+++class DownBlock2d(nn.Module):+ """+ Downsampling block for use in encoder.+ """++ def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1):+ super(DownBlock2d, self).__init__()+ self.conv = nn.Conv2d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size, padding=padding, groups=groups)+ self.norm = nn.BatchNorm2d(out_features, affine=True)+ self.pool = nn.AvgPool2d(kernel_size=(2, 2))++ def forward(self, x):+ out = self.conv(x)+ out = self.norm(out)+ out = F.relu(out)+ out = self.pool(out)+ return out+++class DownBlock3d(nn.Module):+ """+ Downsampling block for use in encoder.+ """++ def __init__(self, in_features, out_features, kernel_size=3, padding=1, groups=1):+ super(DownBlock3d, self).__init__()+ '''+ self.conv = nn.Conv3d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size,+ padding=padding, groups=groups, stride=(1, 2, 2))+ '''+ self.conv = nn.Conv3d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size,+ padding=padding, groups=groups)+ self.norm = nn.BatchNorm3d(out_features, affine=True)+ self.pool = nn.AvgPool3d(kernel_size=(1, 2, 2))++ def forward(self, x):+ out = self.conv(x)+ out = self.norm(out)+ out = F.relu(out)+ out = self.pool(out)+ return out+++class SameBlock2d(nn.Module):+ """+ Simple block, preserve spatial resolution.+ """++ def __init__(self, in_features, out_features, groups=1, kernel_size=3, padding=1, lrelu=False):+ super(SameBlock2d, self).__init__()+ self.conv = nn.Conv2d(in_channels=in_features, out_channels=out_features, kernel_size=kernel_size, padding=padding, groups=groups)+ self.norm = nn.BatchNorm2d(out_features, affine=True)+ if lrelu:+ self.ac = nn.LeakyReLU()+ else:+ self.ac = nn.ReLU()++ def forward(self, x):+ out = self.conv(x)+ out = self.norm(out)+ out = self.ac(out)+ return out+++class Encoder(nn.Module):+ """+ Hourglass Encoder+ """++ def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256):+ super(Encoder, self).__init__()++ down_blocks = []+ for i in range(num_blocks):+ down_blocks.append(DownBlock3d(in_features if i == 0 else min(max_features, block_expansion * (2 ** i)), min(max_features, block_expansion * (2 ** (i + 1))), kernel_size=3, padding=1))+ self.down_blocks = nn.ModuleList(down_blocks)++ def forward(self, x):+ outs = [x]+ for down_block in self.down_blocks:+ outs.append(down_block(outs[-1]))+ return outs+++class Decoder(nn.Module):+ """+ Hourglass Decoder+ """++ def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256):+ super(Decoder, self).__init__()++ up_blocks = []++ for i in range(num_blocks)[::-1]:+ in_filters = (1 if i == num_blocks - 1 else 2) * min(max_features, block_expansion * (2 ** (i + 1)))+ out_filters = min(max_features, block_expansion * (2 ** i))+ up_blocks.append(UpBlock3d(in_filters, out_filters, kernel_size=3, padding=1))++ self.up_blocks = nn.ModuleList(up_blocks)+ self.out_filters = block_expansion + in_features++ self.conv = nn.Conv3d(in_channels=self.out_filters, out_channels=self.out_filters, kernel_size=3, padding=1)+ self.norm = nn.BatchNorm3d(self.out_filters, affine=True)++ def forward(self, x):+ out = x.pop()+ for up_block in self.up_blocks:+ out = up_block(out)+ skip = x.pop()+ out = torch.cat([out, skip], dim=1)+ out = self.conv(out)+ out = self.norm(out)+ out = F.relu(out)+ return out+++class Hourglass(nn.Module):+ """+ Hourglass architecture.+ """++ def __init__(self, block_expansion, in_features, num_blocks=3, max_features=256):+ super(Hourglass, self).__init__()+ self.encoder = Encoder(block_expansion, in_features, num_blocks, max_features)+ self.decoder = Decoder(block_expansion, in_features, num_blocks, max_features)+ self.out_filters = self.decoder.out_filters++ def forward(self, x):+ return self.decoder(self.encoder(x))+++class SPADE(nn.Module):+ def __init__(self, norm_nc, label_nc):+ super().__init__()++ self.param_free_norm = nn.InstanceNorm2d(norm_nc, affine=False)+ nhidden = 128++ self.mlp_shared = nn.Sequential(+ nn.Conv2d(label_nc, nhidden, kernel_size=3, padding=1),+ nn.ReLU())+ self.mlp_gamma = nn.Conv2d(nhidden, norm_nc, kernel_size=3, padding=1)+ self.mlp_beta = nn.Conv2d(nhidden, norm_nc, kernel_size=3, padding=1)++ def forward(self, x, segmap):+ normalized = self.param_free_norm(x)+ segmap = F.interpolate(segmap, size=x.size()[2:], mode='nearest')+ actv = self.mlp_shared(segmap)+ gamma = self.mlp_gamma(actv)+ beta = self.mlp_beta(actv)+ out = normalized * (1 + gamma) + beta+ return out+++class SPADEResnetBlock(nn.Module):+ def __init__(self, fin, fout, norm_G, label_nc, use_se=False, dilation=1):+ super().__init__()+ # Attributes+ self.learned_shortcut = (fin != fout)+ fmiddle = min(fin, fout)+ self.use_se = use_se+ # create conv layers+ self.conv_0 = nn.Conv2d(fin, fmiddle, kernel_size=3, padding=dilation, dilation=dilation)+ self.conv_1 = nn.Conv2d(fmiddle, fout, kernel_size=3, padding=dilation, dilation=dilation)+ if self.learned_shortcut:+ self.conv_s = nn.Conv2d(fin, fout, kernel_size=1, bias=False)+ # apply spectral norm if specified+ if 'spectral' in norm_G:+ self.conv_0 = spectral_norm(self.conv_0)+ self.conv_1 = spectral_norm(self.conv_1)+ if self.learned_shortcut:+ self.conv_s = spectral_norm(self.conv_s)+ # define normalization layers+ self.norm_0 = SPADE(fin, label_nc)+ self.norm_1 = SPADE(fmiddle, label_nc)+ if self.learned_shortcut:+ self.norm_s = SPADE(fin, label_nc)++ def forward(self, x, seg1):+ x_s = self.shortcut(x, seg1)+ dx = self.conv_0(self.actvn(self.norm_0(x, seg1)))+ dx = self.conv_1(self.actvn(self.norm_1(dx, seg1)))+ out = x_s + dx+ return out++ def shortcut(self, x, seg1):+ if self.learned_shortcut:+ x_s = self.conv_s(self.norm_s(x, seg1))+ else:+ x_s = x+ return x_s++ def actvn(self, x):+ return F.leaky_relu(x, 2e-1)+++def filter_state_dict(state_dict, remove_name='fc'):+ new_state_dict = {}+ for key in state_dict:+ if remove_name in key:+ continue+ new_state_dict[key] = state_dict[key]+ return new_state_dict+++class GRN(nn.Module):+ """ GRN (Global Response Normalization) layer+ """++ def __init__(self, dim):+ super().__init__()+ self.gamma = nn.Parameter(torch.zeros(1, 1, 1, dim))+ self.beta = nn.Parameter(torch.zeros(1, 1, 1, dim))++ def forward(self, x):+ Gx = torch.norm(x, p=2, dim=(1, 2), keepdim=True)+ Nx = Gx / (Gx.mean(dim=-1, keepdim=True) + 1e-6)+ return self.gamma * (x * Nx) + self.beta + x+++class LayerNorm(nn.Module):+ r""" LayerNorm that supports two data formats: channels_last (default) or channels_first.+ The ordering of the dimensions in the inputs. channels_last corresponds to inputs with+ shape (batch_size, height, width, channels) while channels_first corresponds to inputs+ with shape (batch_size, channels, height, width).+ """++ def __init__(self, normalized_shape, eps=1e-6, data_format="channels_last"):+ super().__init__()+ self.weight = nn.Parameter(torch.ones(normalized_shape, dtype=torch.float32))+ self.bias = nn.Parameter(torch.zeros(normalized_shape, dtype=torch.float32))+ self.eps = float(eps)+ self.data_format = data_format+ if self.data_format not in ["channels_last", "channels_first"]:+ raise NotImplementedError+ self.normalized_shape = (normalized_shape, )++ def _apply(self, fn):+ """+ 重写 _apply,完全接管参数的转换逻辑。+ 拦截所有 .cuda(), .cpu(), .half(), .to() 操作。+ """++ for name, param in self._parameters.items():+ if param is not None:+ dummy_probe = param.data.view(-1)[:1]++ try:+ target_tensor = fn(dummy_probe)++ target_device = target_tensor.device+ target_dtype = target_tensor.dtype+ except:+ target_device = param.device+ target_dtype = param.dtype++ if name in ['weight', 'bias']:+ # 核心逻辑:如果是 weight/bias,且目标是半精度,则强制保持 FP32+ if target_dtype in [torch.float16, torch.bfloat16]:+ new_data = param.data.to(device=target_device, dtype=torch.float32)+ else:+ new_data = fn(param.data)+ else:+ new_data = fn(param.data)++ param.data = new_data++ if param.grad is not None:+ param.grad.data = param.grad.data.to(device=new_data.device, dtype=new_data.dtype)++ for name, buf in self._buffers.items():+ if buf is not None:+ self._buffers[name] = fn(buf)++ return self++ def forward(self, x):+ dtype = x.dtype+ x = x.float()+ if self.data_format == "channels_last":+ x = F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)+ elif self.data_format == "channels_first":+ x = x.permute(0, 2, 3, 1) # BCHW → BHWC+ x = F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)+ x = x.permute(0, 3, 1, 2) # BHWC → BCHW+ return x.to(dtype)+++def _no_grad_trunc_normal_(tensor, mean, std, a, b):+ # Cut & paste from PyTorch official master until it's in a few official releases - RW+ # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf+ def norm_cdf(x):+ # Computes standard normal cumulative distribution function+ return (1. + math.erf(x / math.sqrt(2.))) / 2.++ if (mean < a - 2 * std) or (mean > b + 2 * std):+ warnings.warn("mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "+ "The distribution of values may be incorrect.",+ stacklevel=2)++ with torch.no_grad():+ # Values are generated by using a truncated uniform distribution and+ # then using the inverse CDF for the normal distribution.+ # Get upper and lower cdf values+ l = norm_cdf((a - mean) / std)+ u = norm_cdf((b - mean) / std)++ # Uniformly fill tensor with values from [l, u], then translate to+ # [2l-1, 2u-1].+ tensor.uniform_(2 * l - 1, 2 * u - 1)++ # Use inverse cdf transform for normal distribution to get truncated+ # standard normal+ tensor.erfinv_()++ # Transform to proper mean, std+ tensor.mul_(std * math.sqrt(2.))+ tensor.add_(mean)++ # Clamp to ensure it's in the proper range+ tensor.clamp_(min=a, max=b)+ return tensor+++def drop_path(x, drop_prob=0., training=False, scale_by_keep=True):+ """ Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).++ This is the same as the DropConnect impl I created for EfficientNet, etc networks, however,+ the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...+ See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for+ changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use+ 'survival rate' as the argument.++ """+ if drop_prob == 0. or not training:+ return x+ keep_prob = 1 - drop_prob+ shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets+ random_tensor = x.new_empty(shape).bernoulli_(keep_prob)+ if keep_prob > 0.0 and scale_by_keep:+ random_tensor.div_(keep_prob)+ return x * random_tensor+++class DropPath(nn.Module):+ """ Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).+ """++ def __init__(self, drop_prob=None, scale_by_keep=True):+ super(DropPath, self).__init__()+ self.drop_prob = drop_prob+ self.scale_by_keep = scale_by_keep++ def forward(self, x):+ return drop_path(x, self.drop_prob, self.training, self.scale_by_keep)+++def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):+ return _no_grad_trunc_normal_(tensor, mean, std, a, b)++# From PyTorch internals+def _ntuple(n):+ def parse(x):+ if isinstance(x, collections.abc.Iterable) and not isinstance(x, str):+ return tuple(x)+ return tuple(repeat(x, n))+ return parse++to_2tuple = _ntuple(2)diff --git a/src/modeling/engine_model.py b/src/modeling/engine_model.pynew file mode 100644index 0000000..ac41ba4--- /dev/null+++ b/src/modeling/engine_model.py@@ -0,0 +1,308 @@+import tensorrt as trt+import pycuda.driver as cuda+import pycuda.autoinit+import numpy as np+import torch+import traceback+import os+from PIL import Image++TRT_LOGGER = trt.Logger()+SKIP_ENGINE_MODEL_CHECK = True++def get_engine(engine_file_path):+ if os.path.exists(engine_file_path):+ print(f"Loading engine from file {engine_file_path}...")+ with open(engine_file_path, "rb") as f, trt.Runtime(TRT_LOGGER) as runtime:+ return runtime.deserialize_cuda_engine(f.read())+ else:+ print(f"No file named {engine_file_path}! Please check the input.")+ return None+++def numpy_to_torch_dtype(np_dtype):+ mapping = {+ np.float32: torch.float,+ np.float64: torch.double,+ np.float16: torch.half,+ np.int32: torch.int32,+ np.int64: torch.int64,+ np.int16: torch.int16,+ np.int8: torch.int8,+ np.uint8: torch.uint8,+ np.bool_: torch.bool+ }+ return mapping.get(np_dtype, None)++def match_shape(a, b):+ if(len(a) == len(b)):+ return tuple(a) == tuple(b)+ elif len(a) > len(b):+ if(a[0] == 1):+ return match_shape(a[1:], b)+ else:+ if(b[0] == 1):+ return match_shape(a, b[1:])+ return False++def match_dtype(a, b):+ if(a.__class__ == torch.dtype):+ a = torch.tensor(0,dtype=a).numpy().dtype+ return a == b+++class EngineModel:+ def __init__(self, engine_file_path, stream = None, device_int = 0, extra_lock = None):+ self.device_int = device_int+ self.extra_lock = extra_lock+ if not(self.extra_lock is None):+ self.extra_lock.acquire()+ assert os.path.exists(engine_file_path), "Engine model path not exists!"+ self.ctx = cuda.Device(self.device_int).make_context()+ try:+ self.engine = get_engine(engine_file_path) # 载入TensorRT引擎+ input_nvars = 0+ output_nvars = 0+ self.input_names = []+ self.output_names = []++ # 【辅助函数】用于获取安全的 Shape (消除 -1)+ def get_safe_shape(engine, name):+ shape = engine.get_tensor_shape(name)+ # 如果形状里包含 -1 (动态维度)+ if -1 in shape:+ # 获取 Profile 0 的 (min, opt, max)+ # 取下标 [2] 即 Max Shape,确保分配足够大的显存+ profile = engine.get_tensor_profile_shape(name, 0)+ if profile:+ print(f"[EngineModel] Detected dynamic shape for {name}: {shape} -> Using Max Profile: {profile[2]}")+ return profile[2]+ else:+ # 如果获取不到 Profile (通常发生在 Output),这是一个风险点+ # 这里为了防止报错,可以尝试打印警告+ print(f"[EngineModel] Warning: Dynamic output {name} has no profile. Mem alloc might fail.")+ return shape++ for binding in self.engine: # 遍历所有tensor,区分Input/Output+ mode = self.engine.get_tensor_mode(binding)+ if(mode== trt.TensorIOMode.INPUT):+ input_nvars += 1+ self.input_names.append(binding)+ elif(mode == trt.TensorIOMode.OUTPUT):+ output_nvars += 1+ self.output_names.append(binding)++ self.input_nvars = input_nvars # input的数量+ self.output_nvars = output_nvars # output的数量++ self.input_shapes = {name : get_safe_shape(self.engine, name) for name in self.input_names} # 获取每个 I/O 的 shape 和 dtype+ self.input_dtypes = {name : self.engine.get_tensor_dtype(name) for name in self.input_names}+ self.input_nbytes = {+ name : trt.volume(self.input_shapes[name]) * trt.nptype(self.input_dtypes[name])().itemsize+ for name in self.input_names+ } # nbytes = tensor 占多少 CUDA 内存(字节数)+ self.output_shapes = {name : get_safe_shape(self.engine, name) for name in self.output_names}+ self.output_dtypes = {name : self.engine.get_tensor_dtype(name) for name in self.output_names}+ self.output_nbytes = {+ name : trt.volume(self.output_shapes[name]) * trt.nptype(self.output_dtypes[name])().itemsize+ for name in self.output_names+ }+ self.dinputs = {name : cuda.mem_alloc(self.input_nbytes[name]) for name in self.input_names} # 为每个输入/输出分配 CUDA 设备内存+ self.doutputs = {name :cuda.mem_alloc(self.output_nbytes[name]) for name in self.output_names}+ self.context = self.engine.create_execution_context() # 创建 ExecutionContext(执行上下文)+ if stream is None:+ self.stream = cuda.Stream()+ else:+ self.stream = stream+ for name in self.input_names: # 绑定 tensor 到 context+ self.context.set_tensor_address(name, int(self.dinputs[name]))+ for name in self.output_names:+ self.context.set_tensor_address(name, int(self.doutputs[name]))+ self.houtputs = {+ name :+ cuda.pagelocked_empty(+ trt.volume(self.output_shapes[name]), dtype=trt.nptype(self.output_dtypes[name])+ ) for name in self.output_names+ } # 分配 page-locked host 内存以存储输出+ except:+ self.ctx.pop()+ raise Exception("CUDA Initialization Failed!")+ self.ctx.pop()+ if not(self.extra_lock is None):+ self.extra_lock.release()++ def __call__(self, skip_check=SKIP_ENGINE_MODEL_CHECK, output_list=[], return_tensor=False, **inputs):+ if not skip_check:+ for name in inputs:+ assert name in self.input_names+ assert match_shape(inputs[name].shape, self.input_shapes[name])+ assert match_dtype(inputs[name].dtype, trt.nptype(self.input_dtypes[name]))+ if not(self.extra_lock is None):+ self.extra_lock.acquire()+ self.ctx.push()+ r = {}+ try:++ for name in inputs:+ hinput = inputs[name]+ if (isinstance(hinput,torch.Tensor) and hinput.device.type=="cuda" and hinput.device.index==self.device_int):+ hinput_con = hinput.contiguous()+ ptr = hinput_con.data_ptr()+ cuda.memcpy_dtod_async(self.dinputs[name], ptr, self.input_nbytes[name], self.stream)+ else:+ hinput_con = np.ascontiguousarray(hinput)+ cuda.memcpy_htod_async(self.dinputs[name], hinput_con, self.stream)+ for name in self.input_names:+ if name not in inputs:+ self.context.set_input_shape(name, self.input_shapes[name])+ self.context.execute_async_v3(self.stream.handle)+ if(return_tensor):+ for name in output_list:+ t = torch.zeros(trt.volume(self.output_shapes[name]), device=f"cuda:{self.device_int}", dtype=numpy_to_torch_dtype(trt.nptype(self.output_dtypes[name])))+ ptr = t.data_ptr()+ cuda.memcpy_dtod_async(ptr, self.doutputs[name], self.output_nbytes[name], self.stream)+ t = t.reshape(tuple(self.output_shapes[name]))+ r[name] = t+ else:+ for name in output_list:+ cuda.memcpy_dtoh_async(self.houtputs[name], self.doutputs[name], self.stream)+ r[name] = self.houtputs[name]+ self.stream.synchronize()+ except Exception as e:+ print("TensorRT Execution Failed!")+ traceback.print_exc()+ self.ctx.pop()+ if not(self.extra_lock is None):+ self.extra_lock.release()+ return None+ self.ctx.pop()+ if not(self.extra_lock is None):+ self.extra_lock.release()+ return r+++ def prefill(self, skip_check=SKIP_ENGINE_MODEL_CHECK, **inputs):+ if not (skip_check):+ for name in inputs:+ in_input = (name in self.input_names)+ assert in_input or (name in self.output_names)+ assert match_shape(inputs[name].shape, self.input_shapes[name] if in_input else self.output_shapes[name])+ assert match_dtype(inputs[name].dtype, trt.nptype(self.input_dtypes[name] if in_input else self.output_dtypes[name]))+ if not(self.extra_lock is None):+ self.extra_lock.acquire()+ self.ctx.push()+ try:+ for name in inputs:+ in_input = (name in self.input_names)+ hinput = inputs[name]++ dst_ptr = self.dinputs[name] if in_input else self.doutputs[name]+ real_nbytes = 0+ if isinstance(hinput, torch.Tensor):+ real_nbytes = hinput.numel() * hinput.element_size()+ else:+ # 假设是 numpy+ real_nbytes = hinput.nbytes++ if (isinstance(hinput,torch.Tensor) and hinput.device.type=="cuda" and hinput.device.index==self.device_int):+ hinput_con = hinput.contiguous()+ ptr = hinput_con.data_ptr()+ cuda.memcpy_dtod_async(dst_ptr, ptr, real_nbytes, self.stream)+ else:+ hinput_con = np.ascontiguousarray(hinput)+ cuda.memcpy_htod_async(dst_ptr, hinput, self.stream)+ self.stream.synchronize()+ except Exception as e:+ traceback.print_exc()+ self.ctx.pop()+ if not(self.extra_lock is None):+ self.extra_lock.release()+ return False+ self.ctx.pop()+ if not(self.extra_lock is None):+ self.extra_lock.release()+ return True++ def __repr__(self):+ r = "TensorRTEngineModel(\n\tInput=[\n"+ for name in self.input_names:+ r += f"\t\t{name}: \t{trt.nptype(self.input_dtypes[name]).__name__}{self.input_shapes[name]},\n"+ r += "\t],Output=[\n"+ for name in self.output_names:+ r += f"\t\t{name}: \t{trt.nptype(self.output_dtypes[name]).__name__}{self.output_shapes[name]},\n"+ r+="\t]\n)"+ return r++ def link(self, other, var_map, skip_check=SKIP_ENGINE_MODEL_CHECK):+ assert self.device_int == other.device_int+ if not (skip_check):+ for source in var_map:+ assert source in other.output_names+ target = var_map[source]+ assert target in self.input_names+ assert match_shape(other.output_shapes[source], self.input_shapes[target])+ assert match_dtype(other.output_dtypes[source], self.input_dtypes[target])++ if not(self.extra_lock is None):+ self.extra_lock.acquire()+ self.ctx.push()+ try:+ for source in var_map:+ target = var_map[source]+ self.context.set_tensor_address(target, int(other.doutputs[source]))+ except Exception as e:+ traceback.print_exc()+ self.ctx.pop()+ if not(self.extra_lock is None):+ self.extra_lock.release()+ return False+ self.ctx.pop()+ if not(self.extra_lock is None):+ self.extra_lock.release()+ return True++ def bind(self, var_map, skip_check=SKIP_ENGINE_MODEL_CHECK):+ if not (skip_check):+ for source in var_map:+ assert source in self.output_names+ target = var_map[source]+ assert target in self.input_names+ assert match_shape(self.output_shapes[source], self.input_shapes[target])+ assert match_dtype(self.output_dtypes[source], self.input_dtypes[target])++ if not(self.extra_lock is None):+ self.extra_lock.acquire()+ self.ctx.push()+ try:+ for source in var_map:+ target = var_map[source]+ self.context.set_tensor_address(target, int(self.doutputs[source]))+ except Exception as e:+ traceback.print_exc()+ self.ctx.pop()+ if not(self.extra_lock is None):+ self.extra_lock.release()+ return False+ self.ctx.pop()+ if not(self.extra_lock is None):+ self.extra_lock.release()+ return True++ def unlink(self):++ if not(self.extra_lock is None):+ self.extra_lock.acquire()+ self.ctx.push()+ try:+ for name in self.input_names:+ self.context.set_tensor_address(name, int(self.dinputs[name]))+ except:+ self.ctx.pop()+ if not(self.extra_lock is None):+ self.extra_lock.release()+ return False+ self.ctx.pop()+ if not(self.extra_lock is None):+ self.extra_lock.release()+ return True+diff --git a/src/modeling/framed_models.py b/src/modeling/framed_models.pynew file mode 100644index 0000000..a0508f8--- /dev/null+++ b/src/modeling/framed_models.py@@ -0,0 +1,177 @@+import torch+from torch import nn+from einops import rearrange+from polygraphy.backend.trt import Profile++class unet_work(nn.Module): # Ugly Power Strip+ def __init__(self, pose_guider, motion_encoder, unet, vae, scheduler, timestep):+ super().__init__()+ self.pose_guider = pose_guider+ self.motion_encoder = motion_encoder+ self.unet = unet+ self.vae = vae+ self.scheduler = scheduler+ self.timesteps = timestep++ def decode_slice(self, vae, x):+ x = x / 0.18215+ x = vae.decode(x).sample+ x = rearrange(x, "b c h w -> b h w c")+ x = (x / 2 + 0.5).clamp(0, 1)+ return x++ def forward(self, sample, encoder_hidden_states, motion_hidden_states, motion, pose_cond_fea, pose, new_noise,+ d00, d01, d10, d11, d20, d21, m, u10, u11, u12, u20, u21, u22, u30, u31, u32+ ):+ new_pose_cond_fea = self.pose_guider(pose)+ pose_cond_fea = torch.cat([pose_cond_fea, new_pose_cond_fea], dim=2)+ new_motion_hidden_states = self.motion_encoder(motion)+ motion_hidden_states = torch.cat([motion_hidden_states, new_motion_hidden_states], dim=1)+ encoder_hidden_states = [encoder_hidden_states, motion_hidden_states]+ score = self.unet(sample, self.timesteps, encoder_hidden_states, pose_cond_fea, d00, d01, d10, d11, d20, d21, m, u10, u11, u12, u20, u21, u22, u30, u31, u32)+ score = rearrange(score, 'b c f h w -> (b f) c h w')+ sample = rearrange(sample, 'b c f h w -> (b f) c h w')+ latents_model_input, pred_original_sample = self.scheduler.step(+ score, self.timesteps, sample, return_dict=False+ )+ latents_model_input = latents_model_input.to(sample.dtype)+ pred_original_sample = pred_original_sample.to(sample.dtype)+ latents_model_input = rearrange(latents_model_input, '(b f) c h w -> b c f h w', f=16)+ pred_video = self.decode_slice(self.vae, pred_original_sample[:4])+ latents = torch.cat([latents_model_input[:, :, 4:, :, :], new_noise], dim=2)+ pose_cond_fea_out = pose_cond_fea[:, :, 4:, :, :]+ motion_hidden_states_out = motion_hidden_states[:, 4:, :, :]+ motion_out = motion_hidden_states[:, :1, :, :]+ return pred_video, latents, pose_cond_fea_out, motion_hidden_states_out, motion_out, pred_original_sample[:1]++ def get_sample_input(self, batchsize, height, width, dtype, device):+ tw, ts, tb = 4, 4, 16 # temporal window size| temporal adaptive steps | temporal batch size+ ml, mc, mh, mw= 32, 16, 224, 224 # motion latent size | motion channels+ b, h, w = batchsize, height, width+ lh, lw = height // 8, width // 8 # latent height | width+ cd0, cd1, cd2, cm, cu1, cu2, cu3 = 320, 640, 1280, 1280, 1280, 640, 320 # unet channels+ emb = 768 # CLIP Embedding Dims | TAESDV Channels+ lc, ic = 4, 3 # latent | image channels+ profile = {+ "sample" : [b, lc, tb, lh, lw],+ "encoder_hidden_states" : [b, 1, emb],+ "motion_hidden_states" : [b, tw * (ts - 1), ml, mc],+ "motion": [b, ic, tw, mh, mw],+ "pose_cond_fea" : [b, cd0, tw * (ts - 1), lh, lw],+ "pose" : [b, ic, tw, h, w],+ "new_noise" : [b, lc, tw, lh, lw],+ "d00" : [b, lh * lw, cd0],+ "d01" : [b, lh * lw, cd0],+ "d10" : [b, lh * lw // 4, cd1],+ "d11" : [b, lh * lw // 4, cd1],+ "d20" : [b, lh * lw // 16, cd2],+ "d21" : [b, lh * lw // 16, cd2],+ "m" : [b, lh * lw // 64, cm],+ "u10" : [b, lh * lw // 16, cu1],+ "u11" : [b, lh * lw // 16, cu1],+ "u12" : [b, lh * lw // 16, cu1],+ "u20" : [b, lh * lw // 4, cu2],+ "u21" : [b, lh * lw // 4, cu2],+ "u22" : [b, lh * lw // 4, cu2],+ "u30" : [b, lh * lw, cu3],+ "u31" : [b, lh * lw, cu3],+ "u32" : [b, lh * lw, cu3],+ }+ return {k: torch.randn(profile[k], dtype=dtype, device=device) for k in profile}++ def get_input_names(self):+ return ["sample", "encoder_hidden_states", "motion_hidden_states",+ "motion", "pose_cond_fea", "pose", "new_noise",+ "d00", "d01", "d10", "d11", "d20", "d21", "m", "u10", "u11", "u12",+ "u20", "u21", "u22", "u30", "u31", "u32"]++ def get_output_names(self):+ return ["pred_video", "latents", "pose_cond_fea_out",+ "motion_hidden_states_out", "motion_out", "latent_first"]++ def get_dynamic_axes(self):+ dynamic_axes = {+ "sample": {3:"h_64", 4:"w_64"},+ "pose_cond_fea": {3:"h_64", 4:"w_64"},+ "pose": {3:"h_512", 4:"h_512"},+ "new_noise": {3: "h_64", 4: "w_64"},+ "d00" : {1: "len_4096"},+ "d01" : {1: "len_4096"},+ "u30" : {1: "len_4096"},+ "u31" : {1: "len_4096"},+ "u32" : {1: "len_4096"},+ "d10" : {1: "len_1024"},+ "d11" : {1: "len_1024"},+ "u20" : {1: "len_1024"},+ "u21" : {1: "len_1024"},+ "u22" : {1: "len_1024"},+ "d20" : {1: "len_256"},+ "d21" : {1: "len_256"},+ "u10" : {1: "len_256"},+ "u11" : {1: "len_256"},+ "u12" : {1: "len_256"},+ "m" : {1: "len_64"},+ }+ return dynamic_axes++ def get_dynamic_map(self, batchsize, height, width):+ tw, ts, tb = 4, 4, 16 # temporal window size| temporal adaptive steps | temporal batch size+ ml, mc, mh, mw= 32, 16, 224, 224 # motion latent size | motion channels+ b, h, w = batchsize, height, width+ lh, lw = height // 8, width // 8 # latent height | width+ cd0, cd1, cd2, cm, cu1, cu2, cu3 = 320, 640, 1280, 1280, 1280, 640, 320 # unet channels+ emb = 768 # CLIP Embedding Dims | TAESDV Channels+ lc, ic = 4, 3 # latent | image channels++ fixed_inputs_map = {+ "sample": (b, lc, tb, lh, lw),+ "encoder_hidden_states": (b, 1, emb),+ "motion_hidden_states": (b, tw * (ts - 1), ml, mc),+ "motion": (b, ic, tw, mh, mw),+ "pose_cond_fea": (b, cd0, tw * (ts - 1), lh, lw),+ "pose": (b, ic, tw, h, w),+ "new_noise": (b, lc, tw, lh, lw),+ }++ dynamic_inputs_map = {+ "d00": (b, lh * lw, cd0),+ "d01": (b, lh * lw, cd0),+ "d10": (b, lh * lw // 4, cd1),+ "d11": (b, lh * lw // 4, cd1),+ "d20": (b, lh * lw // 16, cd2),+ "d21": (b, lh * lw // 16, cd2),+ "m": (b, lh * lw // 64, cm),+ "u10": (b, lh * lw // 16, cu1),+ "u11": (b, lh * lw // 16, cu1),+ "u12": (b, lh * lw // 16, cu1),+ "u20": (b, lh * lw // 4, cu2),+ "u21": (b, lh * lw // 4, cu2),+ "u22": (b, lh * lw // 4, cu2),+ "u30": (b, lh * lw, cu3),+ "u31": (b, lh * lw, cu3),+ "u32": (b, lh * lw, cu3),+ }++ profile = Profile()++ for name, shape in fixed_inputs_map.items():+ shape_tuple = tuple(shape)+ profile.add(name, min=shape_tuple, opt=shape_tuple, max=shape_tuple)++ for name, base_shape in dynamic_inputs_map.items():++ dim0, dim1_base, dim2 = base_shape++ val_1x = dim1_base * 1+ val_2x = dim1_base * 2+ val_4x = dim1_base * 4++ min_shape = (dim0, val_1x, dim2)+ opt_shape = (dim0, val_2x, dim2)+ max_shape = (dim0, val_4x, dim2)++ profile.add(name, min=min_shape, opt=opt_shape, max=max_shape)++ print(f"Dynamic: {name:<5} | Base(1x): {dim1_base:<5} | Range: {val_1x} ~ {val_4x} | Opt: {val_2x}")++ return profile\ No newline at end of filediff --git a/src/modeling/onnx_export.py b/src/modeling/onnx_export.pynew file mode 100644index 0000000..c8b2069--- /dev/null+++ b/src/modeling/onnx_export.py@@ -0,0 +1,102 @@+# adapted from https://github.com/NVIDIA/TensorRT/blob/main/demo/Diffusion/utilities.py+#+# Copyright 2022 The HuggingFace Inc. team.+# SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++import onnx+import gc+import onnx_graphsurgeon as gs+import torch+from onnx import shape_inference+from polygraphy.backend.onnx.loader import fold_constants+import os+from onnxsim import simplify+[email protected]_grad()+def export_onnx(+ model,+ onnx_path: str,+ opt_image_height: int,+ opt_image_width: int,+ opt_batch_size: int,+ onnx_opset: int,+ dtype,+ device,+ auto_cast: bool = True,+):+ from contextlib import contextmanager++ @contextmanager+ def auto_cast_manager(enabled):+ if enabled:+ with torch.inference_mode(), torch.autocast("cuda"):+ yield+ else:+ yield++ # 确保父目录存在+ os.makedirs(os.path.dirname(onnx_path), exist_ok=True)++ with auto_cast_manager(auto_cast):+ inputs = model.get_sample_input(opt_batch_size, opt_image_height, opt_image_width, dtype, device)++ print(model.get_output_names())+ print(f"开始导出 ONNX 模型到: {onnx_path} ...")+ torch.onnx.utils.export(+ model,+ inputs,+ onnx_path,+ export_params=True,+ opset_version=onnx_opset,+ do_constant_folding=True,+ input_names=model.get_input_names(),+ output_names=model.get_output_names(),+ dynamic_axes=model.get_dynamic_axes(),+ )++ del model+ gc.collect()+ torch.cuda.empty_cache()++def optimize_onnx(onnx_path, onnx_opt_path):+ model = onnx.load(onnx_path)+ name = os.path.splitext(os.path.basename(onnx_opt_path))[0]+ model_opt = model++ print(f"Saving to {onnx_opt_path}...")+ onnx.save(+ model_opt,+ onnx_opt_path,+ save_as_external_data=True,+ all_tensors_to_one_file=True,+ location=f"{name}.onnx.data",+ size_threshold=1024+ )+ print("Optimization done.")++def handle_onnx_batch_norm(onnx_path: str):+ onnx_model = onnx.load(onnx_path)+ for node in onnx_model.graph.node:+ if node.op_type == "BatchNormalization":+ for attribute in node.attribute:+ if attribute.name == "training_mode":+ if attribute.i == 1:+ node.output.remove(node.output[1])+ node.output.remove(node.output[1])+ attribute.i = 0++ onnx.save_model(onnx_model, onnx_path)\ No newline at end of filediff --git a/src/models/attention.py b/src/models/attention.pynew file mode 100644index 0000000..fd8b435--- /dev/null+++ b/src/models/attention.py@@ -0,0 +1,463 @@+from typing import Any, Dict, Optional++import torch+from diffusers.models.attention import AdaLayerNorm, Attention, FeedForward+from diffusers.models.embeddings import SinusoidalPositionalEmbedding+from einops import rearrange+from torch import nn++from diffusers.models.attention import *+from diffusers.models.attention_processor import *++class BasicTransformerBlock(nn.Module):+ r"""+ A basic Transformer block.++ Parameters:+ dim (`int`): The number of channels in the input and output.+ num_attention_heads (`int`): The number of heads to use for multi-head attention.+ attention_head_dim (`int`): The number of channels in each head.+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.+ cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention.+ activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward.+ num_embeds_ada_norm (:+ obj: `int`, *optional*): The number of diffusion steps used during training. See `Transformer2DModel`.+ attention_bias (:+ obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter.+ only_cross_attention (`bool`, *optional*):+ Whether to use only cross-attention layers. In this case two cross attention layers are used.+ double_self_attention (`bool`, *optional*):+ Whether to use two self-attention layers. In this case no cross attention layers are used.+ upcast_attention (`bool`, *optional*):+ Whether to upcast the attention computation to float32. This is useful for mixed precision training.+ norm_elementwise_affine (`bool`, *optional*, defaults to `True`):+ Whether to use learnable elementwise affine parameters for normalization.+ norm_type (`str`, *optional*, defaults to `"layer_norm"`):+ The normalization layer to use. Can be `"layer_norm"`, `"ada_norm"` or `"ada_norm_zero"`.+ final_dropout (`bool` *optional*, defaults to False):+ Whether to apply a final dropout after the last feed-forward layer.+ attention_type (`str`, *optional*, defaults to `"default"`):+ The type of attention to use. Can be `"default"` or `"gated"` or `"gated-text-image"`.+ positional_embeddings (`str`, *optional*, defaults to `None`):+ The type of positional embeddings to apply to.+ num_positional_embeddings (`int`, *optional*, defaults to `None`):+ The maximum number of positional embeddings to apply.+ """++ def __init__(+ self,+ dim: int,+ num_attention_heads: int,+ attention_head_dim: int,+ dropout=0.0,+ cross_attention_dim: Optional[int] = None,+ activation_fn: str = "geglu",+ num_embeds_ada_norm: Optional[int] = None,+ attention_bias: bool = False,+ only_cross_attention: bool = False,+ double_self_attention: bool = False,+ upcast_attention: bool = False,+ norm_elementwise_affine: bool = True,+ norm_type: str = "layer_norm", # 'layer_norm', 'ada_norm', 'ada_norm_zero', 'ada_norm_single'+ norm_eps: float = 1e-5,+ final_dropout: bool = False,+ attention_type: str = "default",+ positional_embeddings: Optional[str] = None,+ num_positional_embeddings: Optional[int] = None,+ ):+ super().__init__()+ self.only_cross_attention = only_cross_attention++ self.use_ada_layer_norm_zero = (+ num_embeds_ada_norm is not None+ ) and norm_type == "ada_norm_zero"+ self.use_ada_layer_norm = (+ num_embeds_ada_norm is not None+ ) and norm_type == "ada_norm"+ self.use_ada_layer_norm_single = norm_type == "ada_norm_single"+ self.use_layer_norm = norm_type == "layer_norm"++ if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:+ raise ValueError(+ f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"+ f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."+ )++ if positional_embeddings and (num_positional_embeddings is None):+ raise ValueError(+ "If `positional_embedding` type is defined, `num_positition_embeddings` must also be defined."+ )++ if positional_embeddings == "sinusoidal":+ self.pos_embed = SinusoidalPositionalEmbedding(+ dim, max_seq_length=num_positional_embeddings+ )+ else:+ self.pos_embed = None++ # Define 3 blocks. Each block has its own normalization layer.+ # 1. Self-Attn+ if self.use_ada_layer_norm:+ self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm)+ elif self.use_ada_layer_norm_zero:+ self.norm1 = AdaLayerNormZero(dim, num_embeds_ada_norm)+ else:+ self.norm1 = nn.LayerNorm(+ dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps+ )++ self.attn1 = Attention(+ query_dim=dim,+ heads=num_attention_heads,+ dim_head=attention_head_dim,+ dropout=dropout,+ bias=attention_bias,+ cross_attention_dim=cross_attention_dim if only_cross_attention else None,+ upcast_attention=upcast_attention,+ )++ # 2. Cross-Attn+ if cross_attention_dim is not None or double_self_attention:+ # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.+ # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during+ # the second cross attention block.+ self.norm2 = (+ AdaLayerNorm(dim, num_embeds_ada_norm)+ if self.use_ada_layer_norm+ else nn.LayerNorm(+ dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps+ )+ )+ self.attn2 = Attention(+ query_dim=dim,+ cross_attention_dim=cross_attention_dim+ if not double_self_attention+ else None,+ heads=num_attention_heads,+ dim_head=attention_head_dim,+ dropout=dropout,+ bias=attention_bias,+ upcast_attention=upcast_attention,+ ) # is self-attn if encoder_hidden_states is none+ else:+ self.norm2 = None+ self.attn2 = None++ # 3. Feed-forward+ if not self.use_ada_layer_norm_single:+ self.norm3 = nn.LayerNorm(+ dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps+ )++ self.ff = FeedForward(+ dim,+ dropout=dropout,+ activation_fn=activation_fn,+ final_dropout=final_dropout,+ )++ # 4. Fuser+ if attention_type == "gated" or attention_type == "gated-text-image":+ self.fuser = GatedSelfAttentionDense(+ dim, cross_attention_dim, num_attention_heads, attention_head_dim+ )++ # 5. Scale-shift for PixArt-Alpha.+ if self.use_ada_layer_norm_single:+ self.scale_shift_table = nn.Parameter(torch.randn(6, dim) / dim**0.5)++ # let chunk size default to None+ self._chunk_size = None+ self._chunk_dim = 0++ def set_chunk_feed_forward(self, chunk_size: Optional[int], dim: int = 0):+ # Sets chunk feed-forward+ self._chunk_size = chunk_size+ self._chunk_dim = dim++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ attention_mask: Optional[torch.FloatTensor] = None,+ encoder_hidden_states: Optional[torch.FloatTensor] = None,+ encoder_attention_mask: Optional[torch.FloatTensor] = None,+ timestep: Optional[torch.LongTensor] = None,+ cross_attention_kwargs: Dict[str, Any] = None,+ class_labels: Optional[torch.LongTensor] = None,+ ) -> torch.FloatTensor:+ # Notice that normalization is always applied before the real computation in the following blocks.+ # 0. Self-Attention+ batch_size = hidden_states.shape[0]++ if self.use_ada_layer_norm:+ norm_hidden_states = self.norm1(hidden_states, timestep)+ elif self.use_ada_layer_norm_zero:+ norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(+ hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype+ )+ elif self.use_layer_norm:+ norm_hidden_states = self.norm1(hidden_states)+ elif self.use_ada_layer_norm_single:+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (+ self.scale_shift_table[None] + timestep.reshape(batch_size, 6, -1)+ ).chunk(6, dim=1)+ norm_hidden_states = self.norm1(hidden_states)+ norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa+ norm_hidden_states = norm_hidden_states.squeeze(1)+ else:+ raise ValueError("Incorrect norm used")++ if self.pos_embed is not None:+ norm_hidden_states = self.pos_embed(norm_hidden_states)++ # 1. Retrieve lora scale.+ lora_scale = (+ cross_attention_kwargs.get("scale", 1.0)+ if cross_attention_kwargs is not None+ else 1.0+ )++ # 2. Prepare GLIGEN inputs+ cross_attention_kwargs = (+ cross_attention_kwargs.copy() if cross_attention_kwargs is not None else {}+ )+ gligen_kwargs = cross_attention_kwargs.pop("gligen", None)++ attn_output = self.attn1(+ norm_hidden_states,+ encoder_hidden_states=encoder_hidden_states+ if self.only_cross_attention+ else None,+ attention_mask=attention_mask,+ **cross_attention_kwargs,+ )+ if self.use_ada_layer_norm_zero:+ attn_output = gate_msa.unsqueeze(1) * attn_output+ elif self.use_ada_layer_norm_single:+ attn_output = gate_msa * attn_output++ hidden_states = attn_output + hidden_states+ if hidden_states.ndim == 4:+ hidden_states = hidden_states.squeeze(1)++ # 2.5 GLIGEN Control+ if gligen_kwargs is not None:+ hidden_states = self.fuser(hidden_states, gligen_kwargs["objs"])++ # 3. Cross-Attention+ if self.attn2 is not None:+ if self.use_ada_layer_norm:+ norm_hidden_states = self.norm2(hidden_states, timestep)+ elif self.use_ada_layer_norm_zero or self.use_layer_norm:+ norm_hidden_states = self.norm2(hidden_states)+ elif self.use_ada_layer_norm_single:+ # For PixArt norm2 isn't applied here:+ # https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L70C1-L76C103+ norm_hidden_states = hidden_states+ else:+ raise ValueError("Incorrect norm")++ if self.pos_embed is not None and self.use_ada_layer_norm_single is False:+ norm_hidden_states = self.pos_embed(norm_hidden_states)++ attn_output = self.attn2(+ norm_hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ attention_mask=encoder_attention_mask,+ **cross_attention_kwargs,+ )+ hidden_states = attn_output + hidden_states++ # 4. Feed-forward+ if not self.use_ada_layer_norm_single:+ norm_hidden_states = self.norm3(hidden_states)++ if self.use_ada_layer_norm_zero:+ norm_hidden_states = (+ norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]+ )++ if self.use_ada_layer_norm_single:+ norm_hidden_states = self.norm2(hidden_states)+ norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp++ ff_output = self.ff(norm_hidden_states, scale=lora_scale)++ if self.use_ada_layer_norm_zero:+ ff_output = gate_mlp.unsqueeze(1) * ff_output+ elif self.use_ada_layer_norm_single:+ ff_output = gate_mlp * ff_output++ hidden_states = ff_output + hidden_states+ if hidden_states.ndim == 4:+ hidden_states = hidden_states.squeeze(1)++ return hidden_states+++class TemporalBasicTransformerBlock(nn.Module):+ def __init__(+ self,+ dim: int,+ num_attention_heads: int,+ attention_head_dim: int,+ dropout=0.0,+ cross_attention_dim: Optional[int] = None,+ activation_fn: str = "geglu",+ num_embeds_ada_norm: Optional[int] = None,+ attention_bias: bool = False,+ only_cross_attention: bool = False,+ upcast_attention: bool = False,+ unet_use_cross_frame_attention=None,+ unet_use_temporal_attention=None,+ ):+ super().__init__()+ self.only_cross_attention = only_cross_attention+ self.use_ada_layer_norm = num_embeds_ada_norm is not None+ self.unet_use_cross_frame_attention = unet_use_cross_frame_attention+ self.unet_use_temporal_attention = unet_use_temporal_attention++ # SC-Attn+ self.attn1 = Attention(+ query_dim=dim,+ heads=num_attention_heads,+ dim_head=attention_head_dim,+ dropout=dropout,+ bias=attention_bias,+ upcast_attention=upcast_attention,+ )+ self.norm1 = (+ AdaLayerNorm(dim, num_embeds_ada_norm)+ if self.use_ada_layer_norm+ else nn.LayerNorm(dim)+ )++ # Cross-Attn+ if cross_attention_dim is not None:+ self.attn2 = Attention(+ query_dim=dim,+ cross_attention_dim=cross_attention_dim,+ heads=num_attention_heads,+ dim_head=attention_head_dim,+ dropout=dropout,+ bias=attention_bias,+ upcast_attention=upcast_attention,+ )+ else:+ self.attn2 = None++ if cross_attention_dim is not None:+ self.norm2 = (+ AdaLayerNorm(dim, num_embeds_ada_norm)+ if self.use_ada_layer_norm+ else nn.LayerNorm(dim)+ )+ else:+ self.norm2 = None++ # Feed-forward+ self.ff = FeedForward(dim, dropout=dropout, activation_fn=activation_fn)+ self.norm3 = nn.LayerNorm(dim)+ self.use_ada_layer_norm_zero = False++ # Temp-Attn+ assert unet_use_temporal_attention is not None+ if unet_use_temporal_attention:+ self.attn_temp = Attention(+ query_dim=dim,+ heads=num_attention_heads,+ dim_head=attention_head_dim,+ dropout=dropout,+ bias=attention_bias,+ upcast_attention=upcast_attention,+ )+ nn.init.zeros_(self.attn_temp.to_out[0].weight.data)+ self.norm_temp = (+ AdaLayerNorm(dim, num_embeds_ada_norm)+ if self.use_ada_layer_norm+ else nn.LayerNorm(dim)+ )++ def set_use_cross_frame_attention(self, value):+ self.unet_use_cross_frame_attention = value++ def forward(+ self,+ hidden_states,+ encoder_hidden_states=None,+ timestep=None,+ attention_mask=None,+ video_length=None,+ reference=None,+ ):+ norm_hidden_states = (+ self.norm1(hidden_states, timestep)+ if self.use_ada_layer_norm+ else self.norm1(hidden_states)+ )++ if self.unet_use_cross_frame_attention:+ hidden_states = (+ self.attn1(+ norm_hidden_states,+ attention_mask=attention_mask,+ video_length=video_length,+ )+ + hidden_states+ )+ else:+ if reference is not None:+ bank_fea = [+ rearrange(+ reference.unsqueeze(1).repeat(1, video_length, 1, 1),+ "b t l c -> (b t) l c",+ )+ ]+ modify_norm_hidden_states = torch.cat(+ [norm_hidden_states] + bank_fea, dim=1+ )++ hidden_states = (+ self.attn1(norm_hidden_states, encoder_hidden_states=modify_norm_hidden_states, attention_mask=attention_mask)+ + hidden_states+ )+ else:+ hidden_states = (+ self.attn1(norm_hidden_states, attention_mask=attention_mask)+ + hidden_states+ )++ if self.attn2 is not None:+ # Cross-Attention+ norm_hidden_states = (+ self.norm2(hidden_states, timestep)+ if self.use_ada_layer_norm+ else self.norm2(hidden_states)+ )+ hidden_states = (+ self.attn2(+ norm_hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ attention_mask=attention_mask,+ )+ + hidden_states+ )++ # Feed-forward+ hidden_states = self.ff(self.norm3(hidden_states)) + hidden_states++ # Temporal-Attention+ if self.unet_use_temporal_attention:+ d = hidden_states.shape[1]+ hidden_states = rearrange(+ hidden_states, "(b f) d c -> (b d) f c", f=video_length+ )+ norm_hidden_states = (+ self.norm_temp(hidden_states, timestep)+ if self.use_ada_layer_norm+ else self.norm_temp(hidden_states)+ )+ hidden_states = self.attn_temp(norm_hidden_states) + hidden_states+ hidden_states = rearrange(hidden_states, "(b d) f c -> (b f) d c", d=d)++ return hidden_statesdiff --git a/src/models/motion_encoder/FAN_feature_extractor.py b/src/models/motion_encoder/FAN_feature_extractor.pynew file mode 100644index 0000000..1fca87d--- /dev/null+++ b/src/models/motion_encoder/FAN_feature_extractor.py@@ -0,0 +1,334 @@+import torch+import torch.nn as nn+import torch.nn.functional as F+from typing import Dict, Union+from diffusers.models.attention_processor import AttentionProcessor+++def conv3x3(in_planes, out_planes, strd=1, padding=1, bias=False):+ "3x3 convolution with padding"+ return nn.Conv2d(in_planes, out_planes, kernel_size=3,+ stride=strd, padding=padding, bias=bias)+++class ConvBlock(nn.Module):+ def __init__(self, in_planes, out_planes):+ super(ConvBlock, self).__init__()+ self.bn1 = nn.BatchNorm2d(in_planes)+ self.conv1 = conv3x3(in_planes, int(out_planes / 2))+ self.bn2 = nn.BatchNorm2d(int(out_planes / 2))+ self.conv2 = conv3x3(int(out_planes / 2), int(out_planes / 4))+ self.bn3 = nn.BatchNorm2d(int(out_planes / 4))+ self.conv3 = conv3x3(int(out_planes / 4), int(out_planes / 4))++ if in_planes != out_planes:+ self.downsample = nn.Sequential(+ nn.BatchNorm2d(in_planes, eps=1e-4),+ nn.ReLU(True),+ nn.Conv2d(in_planes, out_planes,+ kernel_size=1, stride=1, bias=False),+ )+ else:+ self.downsample = None++ def forward(self, x):+ residual = x++ out1 = self.bn1(x)+ out1 = F.relu(out1, True)+ out1 = self.conv1(out1)++ out2 = self.bn2(out1)+ out2 = F.relu(out2, True)+ out2 = self.conv2(out2)++ out3 = self.bn3(out2)+ out3 = F.relu(out3, True)+ out3 = self.conv3(out3)++ out3 = torch.cat((out1, out2, out3), 1)++ if self.downsample is not None:+ residual = self.downsample(residual)++ out3 += residual++ return out3+++class HourGlass(nn.Module):+ def __init__(self, num_modules, depth, num_features):+ super(HourGlass, self).__init__()+ self.num_modules = num_modules+ self.depth = depth+ self.features = num_features+ self.dropout = nn.Dropout(0.5)++ self._generate_network(self.depth)++ def _generate_network(self, level):+ self.add_module('b1_' + str(level), ConvBlock(256, 256))++ self.add_module('b2_' + str(level), ConvBlock(256, 256))++ if level > 1:+ self._generate_network(level - 1)+ else:+ self.add_module('b2_plus_' + str(level), ConvBlock(256, 256))++ self.add_module('b3_' + str(level), ConvBlock(256, 256))++ def _forward(self, level, inp):+ # Upper branch+ up1 = inp+ up1 = self._modules['b1_' + str(level)](up1)+ up1 = self.dropout(up1)+ # Lower branch+ low1 = F.max_pool2d(inp, 2, stride=2)+ low1 = self._modules['b2_' + str(level)](low1)++ if level > 1:+ low2 = self._forward(level - 1, low1)+ else:+ low2 = low1+ low2 = self._modules['b2_plus_' + str(level)](low2)++ low3 = low2+ low3 = self._modules['b3_' + str(level)](low3)+ up1size = up1.size()+ rescale_size = (up1size[2], up1size[3])+ up2 = F.upsample(low3, size=rescale_size, mode='bilinear')++ return up1 + up2++ def forward(self, x):+ return self._forward(self.depth, x)+++class FAN_use(nn.Module):+ def __init__(self):+ super(FAN_use, self).__init__()+ self.num_modules = 1++ # Base part+ self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3)+ self.bn1 = nn.BatchNorm2d(64)+ self.conv2 = ConvBlock(64, 128)+ self.conv3 = ConvBlock(128, 128)+ self.conv4 = ConvBlock(128, 256)++ # Stacking part+ hg_module = 0+ self.add_module('m' + str(hg_module), HourGlass(1, 4, 256))+ self.add_module('top_m_' + str(hg_module), ConvBlock(256, 256))+ self.add_module('conv_last' + str(hg_module), nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0))+ self.add_module('l' + str(hg_module), nn.Conv2d(256, 68, kernel_size=1, stride=1, padding=0))+ self.add_module('bn_end' + str(hg_module), nn.BatchNorm2d(256))++ if hg_module < self.num_modules - 1:+ self.add_module('bl' + str(hg_module), nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0))+ self.add_module('al' + str(hg_module), nn.Conv2d(68, 256, kernel_size=1, stride=1, padding=0))++ self.avgpool = nn.MaxPool2d((2, 2), 2)+ self.conv6 = nn.Conv2d(68, 1, 3, 2, 1)+ self.fc = nn.Linear(28 * 28, 512)+ self.bn5 = nn.BatchNorm2d(68)+ self.relu = nn.ReLU(True)++ def forward(self, x, return_featmap=False):+ x = F.relu(self.bn1(self.conv1(x)), True) # 112+ x = F.max_pool2d(self.conv2(x), 2) # 56 # [B, 128, 112, 112]+ x = self.conv3(x)+ x = self.conv4(x) # [B, 256, 56, 56]++ previous = x++ i = 0+ hg = self._modules['m' + str(i)](previous)++ ll = hg+ ll = self._modules['top_m_' + str(i)](ll)++ ll = self._modules['bn_end' + str(i)](self._modules['conv_last' + str(i)](ll)) # [B, 256, 56, 56]+ if return_featmap:+ return ll+ tmp_out = self._modules['l' + str(i)](F.relu(ll))++ net = self.relu(self.bn5(tmp_out)) # [B, 68, 56, 56]+ net = self.conv6(net) # 28 # [B, 1, 28, 28]+ net = net.view(-1, net.shape[-2] * net.shape[-1])+ net = self.relu(net)+ net = self.fc(net)+ return net++from .FAN_temporal_feature_extractor import TemporalTransformer3DModel+from einops import rearrange+++class FAN_SA(nn.Module):+ def __init__(self):+ super(FAN_SA, self).__init__()+ self.num_modules = 1++ # Base part+ self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3)+ self.bn1 = nn.BatchNorm2d(64)+ self.conv2 = ConvBlock(64, 128)+ self.conv3 = ConvBlock(128, 128)+ self.conv4 = ConvBlock(128, 256)++ # Stacking part+ hg_module = 0+ self.add_module('m' + str(hg_module), HourGlass(1, 4, 256))+ self.add_module('top_m_' + str(hg_module), ConvBlock(256, 256))+ self.add_module(+ 'conv_last' + str(hg_module),+ nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0),+ )+ self.add_module(+ 'l' + str(hg_module), nn.Conv2d(256, 68, kernel_size=1, stride=1, padding=0)+ )+ self.add_module('bn_end' + str(hg_module), nn.BatchNorm2d(256))++ if hg_module < self.num_modules - 1:+ self.add_module(+ 'bl' + str(hg_module),+ nn.Conv2d(256, 256, kernel_size=1, stride=1, padding=0),+ )+ self.add_module(+ 'al' + str(hg_module),+ nn.Conv2d(68, 256, kernel_size=1, stride=1, padding=0),+ )++ self.avgpool = nn.MaxPool2d((2, 2), 2)+ self.conv6 = nn.Conv2d(68, 1, 3, 2, 1)+ self.fc = nn.Linear(28 * 28, 512)+ # self.conv6 = nn.Conv2d(68, 2, 3, 2, 1)+ # self.fc = nn.Linear(28 * 28 * 2, 1024)+ self.bn5 = nn.BatchNorm2d(68)+ self.relu = nn.ReLU(True)++ # Add by zxc+ self.att_1 = TemporalTransformer3DModel(+ in_channels=128,+ sample_size=112,+ patch_size=4,+ attention_block_types=("Spatial_Self",),+ zero_initialize=True,+ )+ self.att_2 = TemporalTransformer3DModel(+ in_channels=256,+ sample_size=56,+ patch_size=2,+ attention_block_types=("Spatial_Self",),+ zero_initialize=True,+ )+ self.att_3 = TemporalTransformer3DModel(+ in_channels=256,+ sample_size=56,+ patch_size=2,+ attention_block_types=("Spatial_Self",),+ zero_initialize=True,+ )++ @property+ # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors+ def attn_processors(self) -> Dict[str, AttentionProcessor]:+ r"""+ Returns:+ `dict` of attention processors: A dictionary containing all attention processors used in the model with+ indexed by its weight name.+ """+ # set recursively+ processors = {}++ def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]):+ if hasattr(module, "get_processor"):+ processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True)++ for sub_name, child in module.named_children():+ fn_recursive_add_processors(f"{name}.{sub_name}", child, processors)++ return processors++ for name, module in self.named_children():+ fn_recursive_add_processors(name, module, processors)++ return processors++ def set_attn_processor(+ self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]+ ):+ r"""+ Sets the attention processor to use to compute attention.++ Parameters:+ processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):+ The instantiated processor class or a dictionary of processor classes that will be set as the processor+ for **all** `Attention` layers.++ If `processor` is a dict, the key needs to define the path to the corresponding cross attention+ processor. This is strongly recommended when setting trainable attention processors.++ """+ count = len(self.attn_processors.keys())++ if isinstance(processor, dict) and len(processor) != count:+ raise ValueError(+ f"A dict of processors was passed, but the number of processors {len(processor)} does not match the"+ f" number of attention layers: {count}. Please make sure to pass {count} processor classes."+ )++ def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor):+ if hasattr(module, "set_processor"):+ if not isinstance(processor, dict):+ module.set_processor(processor)+ else:+ module.set_processor(processor.pop(f"{name}.processor"))++ for sub_name, child in module.named_children():+ fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor)++ for name, module in self.named_children():+ fn_recursive_attn_processor(name, module, processor)++ def forward(self, x):+ x = F.relu(self.bn1(self.conv1(x)), True) # 112+ x = self.conv2(x) # [B, 128, 112, 112]++ # Temp Self-Att: [B*h*w, T, 128*p*p] [B*28*28, T, 1024]+ x = rearrange(x, "(b f) c h w -> b c f h w", f=1)+ x = self.att_1(x, skip=True)[:, :, 0]++ x = F.max_pool2d(x, 2) # 56+ x = self.conv3(x)+ x = self.conv4(x) # [B, 256, 56, 56]++ # Temp Self-Att: [B*h*w, T, 256*p*p] [B*28*28, T, 1024]+ x = rearrange(x, "(b f) c h w -> b c f h w", f=1)+ x = self.att_2(x, skip=True)[:, :, 0]++ previous = x++ i = 0+ hg = self._modules['m' + str(i)](previous)++ ll = hg+ ll = self._modules['top_m_' + str(i)](ll)++ ll = self._modules['bn_end' + str(i)](+ self._modules['conv_last' + str(i)](ll)+ ) # [B, 256, 56, 56]+ # Temp Cross-Att: [B*28*28, 1, 1024]*[B*28*28, T, 1024]+ ll = rearrange(ll, "(b f) c h w -> b c f h w", f=1)+ ll = self.att_3(ll, skip=True)[:, :, 0] # "b c 1 h w -> b c h w"+ # print('att3', torch.abs(ll).mean().item())++ tmp_out = self._modules['l' + str(i)](F.relu(ll))++ net = self.relu(self.bn5(tmp_out)) # [B, 68, 56, 56]+ net = self.conv6(net) # 28 # [B, 1, 28, 28]+ net = net.view(-1, net.shape[-2] * net.shape[-1] * net.shape[1])+ net = self.relu(net)+ net = self.fc(net)+ return netdiff --git a/src/models/motion_encoder/FAN_temporal_feature_extractor.py b/src/models/motion_encoder/FAN_temporal_feature_extractor.pynew file mode 100644index 0000000..01676dd--- /dev/null+++ b/src/models/motion_encoder/FAN_temporal_feature_extractor.py@@ -0,0 +1,231 @@+import torch+import torch.nn as nn+import torch.nn.functional as F+from diffusers.models.embeddings import PatchEmbed++from einops import rearrange+from src.models.motion_module import TemporalTransformerBlock, zero_module, random_module++def conv3x3(in_planes, out_planes, strd=1, padding=1, bias=False):+ "3x3 convolution with padding"+ return nn.Conv2d(in_planes, out_planes, kernel_size=3,+ stride=strd, padding=padding, bias=bias)+++class ConvBlock(nn.Module):+ def __init__(self, in_planes, out_planes):+ super(ConvBlock, self).__init__()+ self.bn1 = nn.BatchNorm2d(in_planes)+ self.conv1 = conv3x3(in_planes, int(out_planes / 2))+ self.bn2 = nn.BatchNorm2d(int(out_planes / 2))+ self.conv2 = conv3x3(int(out_planes / 2), int(out_planes / 4))+ self.bn3 = nn.BatchNorm2d(int(out_planes / 4))+ self.conv3 = conv3x3(int(out_planes / 4), int(out_planes / 4))++ if in_planes != out_planes:+ self.downsample = nn.Sequential(+ nn.BatchNorm2d(in_planes),+ nn.ReLU(True),+ nn.Conv2d(in_planes, out_planes,+ kernel_size=1, stride=1, bias=False),+ )+ else:+ self.downsample = None++ def forward(self, x):+ residual = x++ out1 = self.bn1(x)+ out1 = F.relu(out1, True)+ out1 = self.conv1(out1)++ out2 = self.bn2(out1)+ out2 = F.relu(out2, True)+ out2 = self.conv2(out2)++ out3 = self.bn3(out2)+ out3 = F.relu(out3, True)+ out3 = self.conv3(out3)++ out3 = torch.cat((out1, out2, out3), 1)++ if self.downsample is not None:+ residual = self.downsample(residual)++ out3 += residual++ return out3+++class HourGlass(nn.Module):+ def __init__(self, num_modules, depth, num_features):+ super(HourGlass, self).__init__()+ self.num_modules = num_modules+ self.depth = depth+ self.features = num_features+ self.dropout = nn.Dropout(0.5)++ self._generate_network(self.depth)++ def _generate_network(self, level):+ self.add_module('b1_' + str(level), ConvBlock(256, 256))++ self.add_module('b2_' + str(level), ConvBlock(256, 256))++ if level > 1:+ self._generate_network(level - 1)+ else:+ self.add_module('b2_plus_' + str(level), ConvBlock(256, 256))++ self.add_module('b3_' + str(level), ConvBlock(256, 256))++ def _forward(self, level, inp):+ # Upper branch+ up1 = inp+ up1 = self._modules['b1_' + str(level)](up1)+ up1 = self.dropout(up1)+ # Lower branch+ low1 = F.max_pool2d(inp, 2, stride=2)+ low1 = self._modules['b2_' + str(level)](low1)++ if level > 1:+ low2 = self._forward(level - 1, low1)+ else:+ low2 = low1+ low2 = self._modules['b2_plus_' + str(level)](low2)++ low3 = low2+ low3 = self._modules['b3_' + str(level)](low3)+ up1size = up1.size()+ rescale_size = (up1size[2], up1size[3])+ up2 = F.upsample(low3, size=rescale_size, mode='bilinear')++ return up1 + up2++ def forward(self, x):+ return self._forward(self.depth, x)++class TemporalTransformer3DModel(nn.Module):+ def __init__(+ self,+ in_channels,+ num_attention_heads=8,+ attention_head_dim=128,+ num_layers=1,+ attention_block_types=("Temporal_Self",),+ sample_size=56,+ patch_size=2,+ dropout=0.0,+ norm_num_groups=32,+ cross_attention_dim=0,+ activation_fn="geglu",+ attention_bias=False,+ upcast_attention=False,+ cross_frame_attention_mode=None,+ temporal_position_encoding=True,+ temporal_position_encoding_max_len=24,+ zero_initialize=True+ ):+ super().__init__()+ self.in_channels = in_channels+ inner_dim = num_attention_heads * attention_head_dim++ self.norm = torch.nn.GroupNorm(+ num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True+ )+ # self.proj_in = nn.Linear(in_channels, inner_dim) # patchEmbed中有+ if cross_attention_dim == 0 and any([block_name.endswith("_Cross") for block_name in attention_block_types]):+ cross_attention_dim = inner_dim+ self.cross_frame_attention_mode = cross_frame_attention_mode+ if cross_frame_attention_mode is not None:+ assert cross_frame_attention_mode in ['Spatial', 'Temporal'], cross_attention_dim+ assert any([block_name.endswith("_Cross") for block_name in attention_block_types])+ self.transformer_blocks = nn.ModuleList(+ [+ TemporalTransformerBlock(+ dim=inner_dim,+ num_attention_heads=num_attention_heads,+ attention_head_dim=attention_head_dim,+ attention_block_types=attention_block_types,+ dropout=dropout,+ norm_num_groups=norm_num_groups,+ cross_attention_dim=cross_attention_dim,+ activation_fn=activation_fn,+ attention_bias=attention_bias,+ upcast_attention=upcast_attention,+ cross_frame_attention_mode=None,+ temporal_position_encoding=temporal_position_encoding,+ temporal_position_encoding_max_len=temporal_position_encoding_max_len,+ )+ for _ in range(num_layers)+ ]+ )+ self.proj_out = nn.Linear(inner_dim // (patch_size**2), in_channels)++ self.patch_size = patch_size+ self.sample_size = sample_size+ self.pos_embed = PatchEmbed(+ height=sample_size,+ width=sample_size,+ patch_size=patch_size,+ in_channels=in_channels,+ embed_dim=inner_dim,+ interpolation_scale=1,+ )+ # 在转onnx模型时,需要设置为False+ self.pos_embed.pos_embed.requires_grad = False+ self.proj_out = zero_module(self.proj_out) if zero_initialize else random_module(self.proj_out)++ def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, skip=True):+ assert hidden_states.dim() == 5, f"Expected hidden_states to have ndim=5, but got ndim={hidden_states.dim()}."+ batch, _, video_length = hidden_states.shape[:3]+ residual = hidden_states+ hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w")++ _, _, height, width = hidden_states.shape++ hidden_states = self.norm(hidden_states) # 需要先norm,再加posEmb,否则Emb偏执太大了+ grid_h, grid_w = height // self.patch_size, width // self.patch_size,++ hidden_states = self.pos_embed(hidden_states) # [(bf), l, c] # 偏执太大了+ # print(+ # round(torch.abs(hidden_states).mean().item(), 6),+ # round(torch.abs(norm_hidden_states).mean().item(), 6),+ # )+ if self.cross_frame_attention_mode is not None:+ assert encoder_hidden_states is None+ encoder_hidden_states = rearrange(hidden_states, "(b f) d c -> b f d c", b=batch)++ hidden_states = encoder_hidden_states[:, -1] # [b, d, c]+ residual = residual[:, :, -1:]+ video_length = 1+ if self.cross_frame_attention_mode == 'Temporal': # 用所有帧来condition最后一帧(当前帧)+ encoder_hidden_states = rearrange(encoder_hidden_states, "b f d c -> (b d) f c")+ elif self.cross_frame_attention_mode == 'Spatial':+ encoder_hidden_states = rearrange(encoder_hidden_states, "b f d c -> b (f d) c")++ # Transformer Blocks+ for block in self.transformer_blocks:+ hidden_states = block(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ video_length=video_length,+ att_flag=False+ )++ hidden_states = hidden_states.reshape(+ shape=(batch * video_length, grid_h, grid_w, self.patch_size, self.patch_size, -1)+ )+ hidden_states = self.proj_out(hidden_states)+ hidden_states = torch.einsum("nhwpqc->nchpwq", hidden_states)+ hidden_states = hidden_states.reshape(shape=(batch * video_length, self.in_channels, height, width))++ hidden_states = rearrange(hidden_states, "(b f) c h w -> b c f h w", b=batch)+ # print(+ # 'out',+ # round(torch.abs(hidden_states).mean().item(), 6),+ # round(torch.abs(residual).mean().item(), 6),+ # )+ output = (hidden_states + residual) if skip else hidden_states++ return outputdiff --git a/src/models/motion_encoder/encoder.py b/src/models/motion_encoder/encoder.pynew file mode 100644index 0000000..55efbcd--- /dev/null+++ b/src/models/motion_encoder/encoder.py@@ -0,0 +1,41 @@+import torch+import torch.nn as nn+import torch.nn.functional as F+import numpy as np+from .FAN_feature_extractor import FAN_SA+from einops import rearrange+from diffusers.models.embeddings import get_1d_sincos_pos_embed_from_grid+from diffusers.models.modeling_utils import ModelMixin++def zero_module(module):+ # Zero out the parameters of a module and return it.+ assert isinstance(module, nn.Conv2d) or isinstance(module, nn.Linear), type(module)+ for p in module.parameters():+ p.detach().zero_()+ return module++class MotEncoder(ModelMixin):+ def __init__(self, out_ch=16):+ super().__init__()+ self.model = FAN_SA()+ self.out_drop = None #nn.Dropout(p=0.4)+ self.out_ch = out_ch+ expr_dim = 512+ extra_pos_embed = get_1d_sincos_pos_embed_from_grid(out_ch, np.arange(expr_dim//out_ch))+ self.register_buffer("pe", torch.from_numpy(extra_pos_embed).float().unsqueeze(0))+ self.final_proj = nn.Linear(expr_dim, expr_dim)+ self.out_bn = None++ def change_out_dim(self, out_ch):+ self.out_proj = nn.Linear(self.out_ch, out_ch)++ def set_attn_processor(self, processor):+ self.model.set_attn_processor(processor)++ def forward(self, x):+ x = x.to(self.dtype)+ latent = self.model(rearrange(x, "b c f h w -> (b f) c h w"))+ latent = self.final_proj(latent)+ latent = rearrange(latent, "b (l c) -> b l c", c=self.out_ch) + self.pe+ latent = rearrange(latent, "(b f) l c -> b f l c", f=x.shape[2])+ return latent\ No newline at end of filediff --git a/src/models/motion_module.py b/src/models/motion_module.pynew file mode 100644index 0000000..f3f0ba7--- /dev/null+++ b/src/models/motion_module.py@@ -0,0 +1,472 @@+import math+from dataclasses import dataclass+from typing import Callable, Optional++import torch+from diffusers.models.attention import FeedForward+from diffusers.models.attention_processor import Attention, AttnProcessor+from diffusers.utils import BaseOutput+from diffusers.utils.import_utils import is_xformers_available+from einops import rearrange, repeat+from torch import nn+++def zero_module(module):+ # Zero out the parameters of a module and return it.+ assert isinstance(module, nn.Conv2d) or isinstance(module, nn.Linear), type(module)+ for p in module.parameters():+ p.detach().zero_()+ return module++def random_module(m):+ assert isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear), type(m)+ # Initialize weights with He initialization and zero out the biases+ n = (m.kernel_size[0] * m.kernel_size[1] * m.in_channels) if isinstance(m, nn.Conv2d) else m.in_features+ nn.init.normal_(m.weight, mean=0.0, std=math.sqrt(2. / n))+ if m.bias is not None:+ nn.init.zeros_(m.bias)+ return m+++@dataclass+class TemporalTransformer3DModelOutput(BaseOutput):+ sample: torch.FloatTensor+++if is_xformers_available():+ import xformers+ import xformers.ops+else:+ xformers = None+++def get_motion_module(in_channels, motion_module_type: str, motion_module_kwargs: dict):+ if motion_module_type == "Vanilla":+ return VanillaTemporalModule(+ in_channels=in_channels,+ **motion_module_kwargs,+ )+ elif motion_module_type == "RefImage_Vanilla":+ return VanillaTemporalModule(+ in_channels=in_channels,+ skip_ref_image=True,+ **motion_module_kwargs,+ )+ elif motion_module_type == "RefImageCond_Vanilla":+ return VanillaTemporalModule(+ in_channels=in_channels,+ cond_ref_image=True,+ **motion_module_kwargs,+ )+ else:+ raise ValueError+++class VanillaTemporalModule(nn.Module):++ def __init__(+ self,+ in_channels,+ num_attention_heads=8,+ num_transformer_block=2,+ attention_block_types=("Temporal_Self", "Temporal_Self"),+ cross_attention_dim=768,+ cross_frame_attention_mode=None,+ temporal_position_encoding=False,+ temporal_position_encoding_max_len=24,+ temporal_attention_dim_div=1,+ zero_initialize=True,+ skip_ref_image=False,+ cond_ref_image=False,+ ):+ super().__init__()+ self.skip_ref_image = skip_ref_image+ self.cond_ref_image = cond_ref_image++ self.temporal_transformer = TemporalTransformer3DModel(+ in_channels=in_channels,+ num_attention_heads=num_attention_heads,+ attention_head_dim=in_channels+ // num_attention_heads+ // temporal_attention_dim_div,+ num_layers=num_transformer_block,+ attention_block_types=attention_block_types,+ cross_attention_dim=cross_attention_dim,+ cross_frame_attention_mode=cross_frame_attention_mode,+ temporal_position_encoding=temporal_position_encoding,+ temporal_position_encoding_max_len=temporal_position_encoding_max_len,+ )++ if zero_initialize:+ self.temporal_transformer.proj_out = zero_module(+ self.temporal_transformer.proj_out+ )++ def set_use_cross_frame_attention(self, value):+ self.skip_ref_image = value++ def forward(+ self,+ input_tensor,+ temb,+ encoder_hidden_states,+ attention_mask=None,+ anchor_frame_idx=None,+ debug=False+ ):+ hidden_states = input_tensor+ if self.skip_ref_image:+ # if input_tensor.shape[2] > 1:+ hidden_states, ref_hidden_states = input_tensor[:, :, :-1], input_tensor[:, :, -1:]++ hidden_states = self.temporal_transformer(+ hidden_states, encoder_hidden_states, attention_mask, debug=debug+ )++ output = hidden_states+ if self.skip_ref_image:+ # if input_tensor.shape[2] > 1:+ output = torch.cat([output, ref_hidden_states], dim=2)+ elif self.cond_ref_image:+ output = torch.cat([output[:, :, :-1], input_tensor[:, :, -1:]], dim=2)+ return output+++class TemporalTransformer3DModel(nn.Module):+ def __init__(+ self,+ in_channels,+ num_attention_heads,+ attention_head_dim,+ num_layers,+ attention_block_types=(+ "Temporal_Self",+ "Temporal_Self",+ ),+ dropout=0.0,+ norm_num_groups=32,+ cross_attention_dim=768,+ activation_fn="geglu",+ attention_bias=False,+ upcast_attention=False,+ cross_frame_attention_mode=None,+ temporal_position_encoding=False,+ temporal_position_encoding_max_len=24,+ ):+ super().__init__()++ inner_dim = num_attention_heads * attention_head_dim++ self.norm = torch.nn.GroupNorm(+ num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True+ )+ self.proj_in = nn.Linear(in_channels, inner_dim)++ self.transformer_blocks = nn.ModuleList(+ [+ TemporalTransformerBlock(+ dim=inner_dim,+ num_attention_heads=num_attention_heads,+ attention_head_dim=attention_head_dim,+ attention_block_types=attention_block_types,+ dropout=dropout,+ norm_num_groups=norm_num_groups,+ cross_attention_dim=cross_attention_dim,+ activation_fn=activation_fn,+ attention_bias=attention_bias,+ upcast_attention=upcast_attention,+ cross_frame_attention_mode=cross_frame_attention_mode,+ temporal_position_encoding=temporal_position_encoding,+ temporal_position_encoding_max_len=temporal_position_encoding_max_len,+ )+ for d in range(num_layers)+ ]+ )+ self.proj_out = nn.Linear(inner_dim, in_channels)++ def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, debug=False):+ assert (+ hidden_states.dim() == 5+ ), f"Expected hidden_states to have ndim=5, but got ndim={hidden_states.dim()}."+ video_length = hidden_states.shape[2]+ hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w")++ if encoder_hidden_states is not None and encoder_hidden_states.ndim == 4:+ assert encoder_hidden_states.shape[1] == video_length, (video_length, encoder_hidden_states.shape)+ encoder_hidden_states = rearrange(encoder_hidden_states, "b d n c -> (b d) n c",)++ batch, channel, height, weight = hidden_states.shape+ residual = hidden_states++ hidden_states = self.norm(hidden_states)+ inner_dim = hidden_states.shape[1]+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(+ batch, height * weight, inner_dim+ )+ hidden_states = self.proj_in(hidden_states)++ # Transformer Blocks+ for block in self.transformer_blocks:+ hidden_states = block(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ video_length=video_length,+ )++ # output+ hidden_states = self.proj_out(hidden_states)+ hidden_states = (+ hidden_states.reshape(batch, height, weight, inner_dim)+ .permute(0, 3, 1, 2)+ .contiguous()+ )+ if False:+ print(+ 'TemporalModule',+ hidden_states.shape,+ # round(torch.abs(residual).mean().item(), 6),+ # round(torch.abs(residual).max().item(), 6),+ # round(torch.abs(hidden_states).mean().item(), 6),+ # round(torch.abs(hidden_states).max().item(), 6),+ )+ # hidden_states *= 0+ output = hidden_states + residual+ output = rearrange(output, "(b f) c h w -> b c f h w", f=video_length)++ return output+++class TemporalTransformerBlock(nn.Module):++ def __init__(+ self,+ dim,+ num_attention_heads,+ attention_head_dim,+ attention_block_types=(+ "Temporal_Self",+ "Temporal_Self",+ ),+ dropout=0.0,+ norm_num_groups=32,+ cross_attention_dim=768,+ activation_fn="geglu",+ attention_bias=False,+ upcast_attention=False,+ cross_frame_attention_mode=None,+ temporal_position_encoding=False,+ temporal_position_encoding_max_len=24,+ proj_out_dim=None,+ ):+ super().__init__()++ attention_blocks = []+ norms = []++ for block_name in attention_block_types:+ attention_blocks.append(+ VersatileAttention(+ attention_mode=block_name.split("_")[0],+ cross_attention_dim=cross_attention_dim+ if block_name.endswith("_Cross")+ else None,+ query_dim=dim,+ heads=num_attention_heads,+ dim_head=attention_head_dim,+ dropout=dropout,+ bias=attention_bias,+ upcast_attention=upcast_attention,+ cross_frame_attention_mode=cross_frame_attention_mode,+ temporal_position_encoding=temporal_position_encoding,+ temporal_position_encoding_max_len=temporal_position_encoding_max_len,+ )+ )+ norms.append(nn.LayerNorm(dim))++ self.attention_blocks = nn.ModuleList(attention_blocks)+ self.norms = nn.ModuleList(norms)++ self.ff = FeedForward(dim, dropout=dropout, activation_fn=activation_fn)+ self.ff_norm = nn.LayerNorm(dim)++ self.proj_out = nn.Linear(dim, proj_out_dim) if proj_out_dim is not None else None++ def forward(+ self,+ hidden_states,+ encoder_hidden_states=None,+ attention_mask=None,+ video_length=None,+ att_flag=False+ ):+ for attention_block, norm in zip(self.attention_blocks, self.norms):+ norm_hidden_states = norm(hidden_states)+ if att_flag:+ print(+ 'block',+ round(torch.abs(hidden_states).mean().item(), 6),+ round(torch.abs(norm_hidden_states).mean().item(), 6),+ )+ hidden_states = (+ attention_block(+ norm_hidden_states,+ encoder_hidden_states=encoder_hidden_states+ if attention_block.is_cross_attention+ else None,+ video_length=video_length,+ att_flag=att_flag+ )+ + hidden_states+ )++ hidden_states = self.ff(self.ff_norm(hidden_states)) + hidden_states++ output = hidden_states if self.proj_out is None else self.proj_out(hidden_states)+ return output+++class PositionalEncoding(nn.Module):+ def __init__(self, d_model, dropout=0.0, max_len=24):+ super().__init__()+ self.dropout = nn.Dropout(p=dropout)+ position = torch.arange(max_len).unsqueeze(1)+ div_term = torch.exp(+ torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model)+ )+ pe = torch.zeros(1, max_len, d_model)+ pe[0, :, 0::2] = torch.sin(position * div_term)+ pe[0, :, 1::2] = torch.cos(position * div_term)+ self.register_buffer("pe", pe)++ def forward(self, x):+ x = x + self.pe[:, : x.size(1)]+ return self.dropout(x)+++class VersatileAttention(Attention):+ def __init__(+ self,+ attention_mode=None,+ cross_frame_attention_mode=None,+ temporal_position_encoding=False,+ temporal_position_encoding_max_len=24,+ *args,+ **kwargs,+ ):+ super().__init__(*args, **kwargs)+ assert attention_mode in ["Temporal", "Spatial"], attention_mode++ self.attention_mode = attention_mode+ self.is_cross_attention = kwargs["cross_attention_dim"] is not None++ self.pos_encoder = (+ PositionalEncoding(+ kwargs["query_dim"],+ dropout=0.0,+ max_len=temporal_position_encoding_max_len,+ )+ if (temporal_position_encoding and attention_mode == "Temporal")+ else None+ )++ def extra_repr(self):+ return f"(Module Info) Attention_Mode: {self.attention_mode}, Is_Cross_Attention: {self.is_cross_attention}"++ def set_use_memory_efficient_attention_xformers(+ self,+ use_memory_efficient_attention_xformers: bool,+ attention_op: Optional[Callable] = None,+ ):+ if use_memory_efficient_attention_xformers:+ if not is_xformers_available():+ raise ModuleNotFoundError(+ (+ "Refer to https://github.com/facebookresearch/xformers for more information on how to install"+ " xformers"+ ),+ name="xformers",+ )+ elif not torch.cuda.is_available():+ raise ValueError(+ "torch.cuda.is_available() should be True but is False. xformers' memory efficient attention is"+ " only available for GPU "+ )+ else:+ try:+ # Make sure we can run the memory efficient attention+ _ = xformers.ops.memory_efficient_attention(+ torch.randn((1, 2, 40), device="cuda"),+ torch.randn((1, 2, 40), device="cuda"),+ torch.randn((1, 2, 40), device="cuda"),+ )+ except Exception as e:+ raise e++ # XFormersAttnProcessor corrupts video generation and work with Pytorch 1.13.+ # Pytorch 2.0.1 AttnProcessor works the same as XFormersAttnProcessor in Pytorch 1.13.+ # You don't need XFormersAttnProcessor here.+ # processor = XFormersAttnProcessor(+ # attention_op=attention_op,+ # )+ processor = AttnProcessor()+ else:+ processor = AttnProcessor()++ self.set_processor(processor)++ def forward(+ self,+ hidden_states,+ encoder_hidden_states=None,+ attention_mask=None,+ video_length=None,+ bank=None,+ att_flag=False,+ **cross_attention_kwargs,+ ):+ if self.attention_mode == "Temporal":+ d = hidden_states.shape[1] # d means HxW+ hidden_states = rearrange(hidden_states, "(b f) d c -> (b d) f c", f=video_length)++ if encoder_hidden_states is not None:+ if not encoder_hidden_states.shape[0] == hidden_states.shape[0]:+ encoder_hidden_states = repeat(encoder_hidden_states, "b n c -> (b d) n c", d=d)++ if bank is not None and self.attention_mode == "Temporal" and not self.is_cross_attention:+ # motion_frames作为之前的帧,引入motion module进行condition+ modify_norm_hidden_states = torch.cat(bank + [hidden_states], dim=1)++ if self.pos_encoder is not None:+ modify_norm_hidden_states = self.pos_encoder(modify_norm_hidden_states)++ hidden_states = self.processor(+ self,+ hidden_states,+ encoder_hidden_states=modify_norm_hidden_states,+ attention_mask=attention_mask,+ **cross_attention_kwargs,+ ) # 改为cross-att++ else:+ if self.pos_encoder is not None:+ hidden_states = self.pos_encoder(hidden_states)+ inp = hidden_states+ hidden_states = self.processor(+ self,+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ attention_mask=attention_mask,+ **cross_attention_kwargs,+ )+ if att_flag:+ print(+ 'ver_att',+ round(torch.abs(inp).mean().item(), 6),+ round(torch.abs(encoder_hidden_states).mean().item(), 6),+ round(torch.abs(hidden_states).mean().item(), 6),+ )++ if self.attention_mode == "Temporal":+ hidden_states = rearrange(hidden_states, "(b d) f c -> (b f) d c", d=d)++ return hidden_states\ No newline at end of filediff --git a/src/models/mutual_self_attention.py b/src/models/mutual_self_attention.pynew file mode 100644index 0000000..81781a7--- /dev/null+++ b/src/models/mutual_self_attention.py@@ -0,0 +1,465 @@+from typing import Any, Dict, Optional++import torch+from einops import rearrange++from src.models.attention import TemporalBasicTransformerBlock++from .attention import BasicTransformerBlock+++def torch_dfs(model: torch.nn.Module):+ result = [model]+ for child in model.children():+ result += torch_dfs(child)+ return result+++class ReferenceAttentionControl:+ def __init__(+ self,+ unet,+ mode="write",+ do_classifier_free_guidance=False,+ attention_auto_machine_weight=float("inf"),+ gn_auto_machine_weight=1.0,+ style_fidelity=1.0,+ reference_attn=True,+ reference_adain=False,+ fusion_blocks="midup",+ batch_size=1,+ cache_kv=False,+ ) -> None:+ # 10. Modify self attention and group norm+ self.unet = unet+ assert mode in ["read", "write"]+ assert fusion_blocks in ["midup", "full"]+ self.reference_attn = reference_attn+ self.reference_adain = reference_adain+ self.fusion_blocks = fusion_blocks+ self.cache_kv = cache_kv+ self.register_reference_hooks(+ mode,+ do_classifier_free_guidance,+ attention_auto_machine_weight,+ gn_auto_machine_weight,+ style_fidelity,+ reference_attn,+ reference_adain,+ fusion_blocks,+ batch_size=batch_size,+ cache_kv=self.cache_kv,+ )++ def register_reference_hooks(+ self,+ mode,+ do_classifier_free_guidance,+ attention_auto_machine_weight,+ gn_auto_machine_weight,+ style_fidelity,+ reference_attn,+ reference_adain,+ dtype=torch.float16,+ batch_size=1,+ num_images_per_prompt=1,+ device=torch.device("cpu"),+ fusion_blocks="midup",+ cache_kv=False,+ ):+ MODE = mode+ do_classifier_free_guidance = do_classifier_free_guidance+ attention_auto_machine_weight = attention_auto_machine_weight+ gn_auto_machine_weight = gn_auto_machine_weight+ style_fidelity = style_fidelity+ reference_attn = reference_attn+ reference_adain = reference_adain+ fusion_blocks = fusion_blocks+ num_images_per_prompt = num_images_per_prompt+ cache_kv = cache_kv+ dtype = dtype+ if do_classifier_free_guidance:+ uc_mask = (+ torch.Tensor(+ [1] * batch_size * num_images_per_prompt * 16+ + [0] * batch_size * num_images_per_prompt * 16+ )+ .to(device)+ .bool()+ )+ else:+ uc_mask = (+ torch.Tensor([0] * batch_size * num_images_per_prompt * 2)+ .to(device)+ .bool()+ )++ def hacked_basic_transformer_inner_forward(+ self,+ hidden_states: torch.FloatTensor,+ attention_mask: Optional[torch.FloatTensor] = None,+ encoder_hidden_states: Optional[torch.FloatTensor] = None,+ encoder_attention_mask: Optional[torch.FloatTensor] = None,+ timestep: Optional[torch.LongTensor] = None,+ cross_attention_kwargs: Dict[str, Any] = None,+ class_labels: Optional[torch.LongTensor] = None,+ video_length=None,+ ):+ if self.use_ada_layer_norm: # False+ norm_hidden_states = self.norm1(hidden_states, timestep)+ elif self.use_ada_layer_norm_zero:+ (+ norm_hidden_states,+ gate_msa,+ shift_mlp,+ scale_mlp,+ gate_mlp,+ ) = self.norm1(+ hidden_states,+ timestep,+ class_labels,+ hidden_dtype=hidden_states.dtype,+ )+ else:+ norm_hidden_states = self.norm1(hidden_states)++ # 1. Self-Attention+ # self.only_cross_attention = False+ cross_attention_kwargs = (+ cross_attention_kwargs if cross_attention_kwargs is not None else {}+ )+ if self.only_cross_attention:+ attn_output = self.attn1(+ norm_hidden_states,+ encoder_hidden_states=encoder_hidden_states+ if self.only_cross_attention+ else None,+ attention_mask=attention_mask,+ **cross_attention_kwargs,+ )+ else:+ if MODE == "write":+ self.bank.append(norm_hidden_states.clone())+ attn_output = self.attn1(+ norm_hidden_states,+ encoder_hidden_states=encoder_hidden_states+ if self.only_cross_attention+ else None,+ attention_mask=attention_mask,+ **cross_attention_kwargs,+ )+ if MODE == "read":+ kv_cache = norm_hidden_states.clone()+ kv_cache = rearrange(+ kv_cache, "(b t) l c -> b t l c", t=video_length+ )+ self.kv_cache = kv_cache[:,:2,:,:]++ bank_fea = [+ rearrange(+ d.unsqueeze(1).repeat(1, video_length, 1, 1),+ "b t l c -> (b t) l c",+ )+ for d in self.bank+ ]++ if self.kv_bank is not None and cache_kv:+ ahead_fea = self.kv_bank.unsqueeze(1).repeat(1, video_length, 1, 1, 1)+ ahead_fea = rearrange(ahead_fea, "b t n l c -> (b t) (n l) c")+ bank_fea.append(ahead_fea)++ modify_norm_hidden_states = torch.cat(+ [norm_hidden_states] + bank_fea, dim=1+ )++ hidden_states_uc = (+ self.attn1(+ norm_hidden_states,+ encoder_hidden_states=modify_norm_hidden_states,+ attention_mask=attention_mask,+ )+ # self.attn1(+ # modify_norm_hidden_states,+ # encoder_hidden_states=modify_norm_hidden_states,+ # attention_mask=attention_mask,+ # )[:, : hidden_states.shape[-2], :]+ + hidden_states+ )+ if do_classifier_free_guidance:+ hidden_states_c = hidden_states_uc.clone()+ _uc_mask = uc_mask.clone()+ if hidden_states.shape[0] != _uc_mask.shape[0]:+ _uc_mask = (+ torch.Tensor(+ [1] * (hidden_states.shape[0] // 2)+ + [0] * (hidden_states.shape[0] // 2)+ )+ .to(device)+ .bool()+ )+ if self.kv_bank is not None:+ # if False:+ modify_norm_hidden_states = torch.cat(+ [norm_hidden_states, ahead_fea], dim=1+ )+ else:+ modify_norm_hidden_states = norm_hidden_states+ hidden_states_c[_uc_mask] = (+ self.attn1(+ norm_hidden_states[_uc_mask],+ encoder_hidden_states=modify_norm_hidden_states[_uc_mask],+ attention_mask=attention_mask,+ )+ + hidden_states[_uc_mask]+ )+ hidden_states = hidden_states_c.clone()+ else:+ hidden_states = hidden_states_uc++ # self.bank.clear()+ if self.attn2 is not None:+ # Cross-Attention+ norm_hidden_states = (+ self.norm2(hidden_states, timestep)+ if self.use_ada_layer_norm+ else self.norm2(hidden_states)+ )+ hidden_states = (+ self.attn2(+ norm_hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ attention_mask=attention_mask,+ )+ + hidden_states+ )++ # Feed-forward+ hidden_states = self.ff(self.norm3(hidden_states)) + hidden_states++ # Temporal-Attention+ if self.unet_use_temporal_attention:+ d = hidden_states.shape[1]+ hidden_states = rearrange(+ hidden_states, "(b f) d c -> (b d) f c", f=video_length+ )+ norm_hidden_states = (+ self.norm_temp(hidden_states, timestep)+ if self.use_ada_layer_norm+ else self.norm_temp(hidden_states)+ )+ hidden_states = (+ self.attn_temp(norm_hidden_states) + hidden_states+ )+ hidden_states = rearrange(+ hidden_states, "(b d) f c -> (b f) d c", d=d+ )++ return hidden_states++ if self.use_ada_layer_norm_zero:+ attn_output = gate_msa.unsqueeze(1) * attn_output+ hidden_states = attn_output + hidden_states++ if self.attn2 is not None:+ norm_hidden_states = (+ self.norm2(hidden_states, timestep)+ if self.use_ada_layer_norm+ else self.norm2(hidden_states)+ )++ # 2. Cross-Attention+ attn_output = self.attn2(+ norm_hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ attention_mask=encoder_attention_mask,+ **cross_attention_kwargs,+ )+ hidden_states = attn_output + hidden_states++ # 3. Feed-forward+ norm_hidden_states = self.norm3(hidden_states)++ if self.use_ada_layer_norm_zero:+ norm_hidden_states = (+ norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]+ )++ ff_output = self.ff(norm_hidden_states)++ if self.use_ada_layer_norm_zero:+ ff_output = gate_mlp.unsqueeze(1) * ff_output++ hidden_states = ff_output + hidden_states++ return hidden_states++ if self.reference_attn:+ if self.fusion_blocks == "midup":+ attn_modules = [+ module+ for module in (+ torch_dfs(self.unet.mid_block) + torch_dfs(self.unet.up_blocks)+ )+ if isinstance(module, BasicTransformerBlock)+ or isinstance(module, TemporalBasicTransformerBlock)+ ]+ elif self.fusion_blocks == "full":+ attn_modules = [+ module+ for module in torch_dfs(self.unet)+ if isinstance(module, BasicTransformerBlock)+ or isinstance(module, TemporalBasicTransformerBlock)+ ]+ attn_modules = sorted(+ attn_modules, key=lambda x: -x.norm1.normalized_shape[0]+ )++ for i, module in enumerate(attn_modules):+ module._original_inner_forward = module.forward+ if isinstance(module, BasicTransformerBlock):+ module.forward = hacked_basic_transformer_inner_forward.__get__(+ module, BasicTransformerBlock+ )+ if isinstance(module, TemporalBasicTransformerBlock):+ module.forward = hacked_basic_transformer_inner_forward.__get__(+ module, TemporalBasicTransformerBlock+ )++ module.bank = []+ if(self.cache_kv):+ module.kv_bank = None+ module.kv_cache = None+ module.attn_weight = float(i) / float(len(attn_modules))++ def update(self, writer, dtype=torch.float16, drop_ratio=0.):+ if self.reference_attn:+ if self.fusion_blocks == "midup":+ reader_attn_modules = [+ module+ for module in (+ torch_dfs(self.unet.mid_block) + torch_dfs(self.unet.up_blocks)+ )+ if isinstance(module, TemporalBasicTransformerBlock)+ ]+ writer_attn_modules = [+ module+ for module in (+ torch_dfs(writer.unet.mid_block)+ + torch_dfs(writer.unet.up_blocks)+ )+ if isinstance(module, BasicTransformerBlock)+ ]+ elif self.fusion_blocks == "full":+ reader_attn_modules = [+ module+ for module in torch_dfs(self.unet)+ if isinstance(module, TemporalBasicTransformerBlock)+ ]+ writer_attn_modules = [+ module+ for module in torch_dfs(writer.unet)+ if isinstance(module, BasicTransformerBlock)+ ]++ reader_attn_modules = sorted(+ reader_attn_modules, key=lambda x: -x.norm1.normalized_shape[0]+ )+ writer_attn_modules = sorted(+ writer_attn_modules, key=lambda x: -x.norm1.normalized_shape[0]+ )+ for r, w in zip(reader_attn_modules, writer_attn_modules):+ if drop_ratio > 0:+ r.bank = []+ for v in w.bank:+ N, L, D = v.shape # batch, length, dim+ len_keep = int(L * (1 - drop_ratio))++ noise = torch.rand(N, L) # noise in [0, 1]+ ids_shuffle = torch.argsort(noise, dim=1) # ascend: small is keep, large is remove+ ids_keep = ids_shuffle[:, :len_keep].to(v.device)+ visible_tokens = torch.gather(v.clone(), dim=1, index=ids_keep.unsqueeze(-1).repeat(1, 1, D))+ r.bank.append((visible_tokens).to(dtype))++ else:+ r.bank = [v.clone().to(dtype) for v in w.bank]+ # w.bank.clear()++ def update_hkf(self, writer, dtype=torch.float16, drop_ratio=0.):+ if self.reference_attn:+ if self.fusion_blocks == "midup":+ reader_attn_modules = [+ module+ for module in (+ torch_dfs(self.unet.mid_block) + torch_dfs(self.unet.up_blocks)+ )+ if isinstance(module, TemporalBasicTransformerBlock)+ ]+ writer_attn_modules = [+ module+ for module in (+ torch_dfs(writer.unet.mid_block)+ + torch_dfs(writer.unet.up_blocks)+ )+ if isinstance(module, BasicTransformerBlock)+ ]+ elif self.fusion_blocks == "full":+ reader_attn_modules = [+ module+ for module in torch_dfs(self.unet)+ if isinstance(module, TemporalBasicTransformerBlock)+ ]+ writer_attn_modules = [+ module+ for module in torch_dfs(writer.unet)+ if isinstance(module, BasicTransformerBlock)+ ]++ reader_attn_modules = sorted(+ reader_attn_modules, key=lambda x: -x.norm1.normalized_shape[0]+ )+ writer_attn_modules = sorted(+ writer_attn_modules, key=lambda x: -x.norm1.normalized_shape[0]+ )+ for r, w in zip(reader_attn_modules, writer_attn_modules):+ if r.kv_bank is None:+ r.kv_bank = torch.cat([v.clone().unsqueeze(1).to(dtype) for v in w.bank], dim=1)+ else:+ r.kv_bank = torch.cat([r.kv_bank] + [v.clone().unsqueeze(1).to(dtype) for v in w.bank], dim=1).to(dtype)++ def output(self, dtype=torch.float16):+ res = {}+ for i in range(3):+ for j in range(2):+ res[f"d{i}{j}"] = torch.cat([v.clone().to(dtype=dtype, device=self.unet.device) for v in self.unet.down_blocks[i].attentions[j].transformer_blocks[0].bank], dim=1)+ res["m"] = torch.cat([v.clone().to(dtype=dtype, device=self.unet.device) for v in self.unet.mid_block.attentions[0].transformer_blocks[0].bank], dim=1)+ for i in range(1, 4):+ for j in range(3):+ res[f"u{i}{j}"] = torch.cat([v.clone().to(dtype=dtype, device=self.unet.device) for v in self.unet.up_blocks[i].attentions[j].transformer_blocks[0].bank], dim=1)+ return res++ def clear(self):+ if self.reference_attn:+ if self.fusion_blocks == "midup":+ reader_attn_modules = [+ module+ for module in (+ torch_dfs(self.unet.mid_block) + torch_dfs(self.unet.up_blocks)+ )+ if isinstance(module, BasicTransformerBlock)+ or isinstance(module, TemporalBasicTransformerBlock)+ ]+ elif self.fusion_blocks == "full":+ reader_attn_modules = [+ module+ for module in torch_dfs(self.unet)+ if isinstance(module, BasicTransformerBlock)+ or isinstance(module, TemporalBasicTransformerBlock)+ ]+ reader_attn_modules = sorted(+ reader_attn_modules, key=lambda x: -x.norm1.normalized_shape[0]+ )+ for r in reader_attn_modules:+ r.bank.clear()+ if self.cache_kv:+ r.kv_bank=None+ r.kv_cache=None\ No newline at end of filediff --git a/src/models/pose_guider.py b/src/models/pose_guider.pynew file mode 100644index 0000000..f079429--- /dev/null+++ b/src/models/pose_guider.py@@ -0,0 +1,57 @@+from typing import Tuple++import torch.nn as nn+import torch.nn.functional as F+import torch.nn.init as init+from diffusers.models.modeling_utils import ModelMixin++from src.models.motion_module import zero_module+from src.models.resnet import InflatedConv3d+++class PoseGuider(ModelMixin):+ def __init__(+ self,+ conditioning_embedding_channels: int = 320,+ conditioning_channels: int = 3,+ block_out_channels: Tuple[int] = (16, 32, 96, 256),+ ):+ super().__init__()+ self.conv_in = InflatedConv3d(+ conditioning_channels, block_out_channels[0], kernel_size=3, padding=1+ )++ self.blocks = nn.ModuleList([])++ for i in range(len(block_out_channels) - 1):+ channel_in = block_out_channels[i]+ channel_out = block_out_channels[i + 1]+ self.blocks.append(+ InflatedConv3d(channel_in, channel_in, kernel_size=3, padding=1)+ )+ self.blocks.append(+ InflatedConv3d(+ channel_in, channel_out, kernel_size=3, padding=1, stride=2+ )+ )++ self.conv_out_modify = zero_module(+ InflatedConv3d(+ block_out_channels[-1],+ conditioning_embedding_channels,+ kernel_size=3,+ padding=1,+ )+ )++ def forward(self, conditioning):+ embedding = self.conv_in(conditioning)+ embedding = F.silu(embedding)++ for block in self.blocks:+ embedding = block(embedding)+ embedding = F.silu(embedding)++ embedding = self.conv_out_modify(embedding)++ return embeddingdiff --git a/src/models/resnet.py b/src/models/resnet.pynew file mode 100644index 0000000..ea73144--- /dev/null+++ b/src/models/resnet.py@@ -0,0 +1,255 @@+import torch+import torch.nn as nn+import torch.nn.functional as F+from einops import rearrange+from typing import Dict, Optional+++class InflatedConv3d(nn.Conv2d):+ def forward(self, x):+ video_length = x.shape[2]++ x = rearrange(x, "b c f h w -> (b f) c h w")+ x = super().forward(x)+ x = rearrange(x, "(b f) c h w -> b c f h w", f=video_length)++ return x+++class InflatedGroupNorm(nn.GroupNorm):+ def forward(self, x):+ video_length = x.shape[2]++ x = rearrange(x, "b c f h w -> (b f) c h w")+ x = super().forward(x)+ x = rearrange(x, "(b f) c h w -> b c f h w", f=video_length)++ return x+++class Upsample3D(nn.Module):+ def __init__(+ self,+ channels,+ use_conv=False,+ use_conv_transpose=False,+ out_channels=None,+ name="conv",+ ):+ super().__init__()+ self.channels = channels+ self.out_channels = out_channels or channels+ self.use_conv = use_conv+ self.use_conv_transpose = use_conv_transpose+ self.name = name++ conv = None+ if use_conv_transpose:+ raise NotImplementedError+ elif use_conv:+ self.conv = InflatedConv3d(self.channels, self.out_channels, 3, padding=1)++ def forward(self, hidden_states, output_size=None):+ assert hidden_states.shape[1] == self.channels++ if self.use_conv_transpose:+ raise NotImplementedError++ # Cast to float32 to as 'upsample_nearest2d_out_frame' op does not support bfloat16+ dtype = hidden_states.dtype+ if dtype == torch.bfloat16:+ hidden_states = hidden_states.to(torch.float32)++ # upsample_nearest_nhwc fails with large batch sizes. see https://github.com/huggingface/diffusers/issues/984+ if hidden_states.shape[0] >= 64:+ hidden_states = hidden_states.contiguous()++ # if `output_size` is passed we force the interpolation output+ # size and do not make use of `scale_factor=2`+ if output_size is None:+ hidden_states = F.interpolate(+ hidden_states, scale_factor=[1.0, 2.0, 2.0], mode="nearest"+ )+ else:+ hidden_states = F.interpolate(+ hidden_states, size=output_size, mode="nearest"+ )++ # If the input is bfloat16, we cast back to bfloat16+ if dtype == torch.bfloat16:+ hidden_states = hidden_states.to(dtype)++ # if self.use_conv:+ # if self.name == "conv":+ # hidden_states = self.conv(hidden_states)+ # else:+ # hidden_states = self.Conv2d_0(hidden_states)+ hidden_states = self.conv(hidden_states)++ return hidden_states+++class Downsample3D(nn.Module):+ def __init__(+ self, channels, use_conv=False, out_channels=None, padding=1, name="conv"+ ):+ super().__init__()+ self.channels = channels+ self.out_channels = out_channels or channels+ self.use_conv = use_conv+ self.padding = padding+ stride = 2+ self.name = name++ if use_conv:+ self.conv = InflatedConv3d(+ self.channels, self.out_channels, 3, stride=stride, padding=padding+ )+ else:+ raise NotImplementedError++ def forward(self, hidden_states):+ assert hidden_states.shape[1] == self.channels+ if self.use_conv and self.padding == 0:+ raise NotImplementedError++ assert hidden_states.shape[1] == self.channels+ hidden_states = self.conv(hidden_states)++ return hidden_states+++class ResnetBlock3D(nn.Module):+ def __init__(+ self,+ *,+ in_channels,+ out_channels=None,+ conv_shortcut=False,+ dropout=0.0,+ temb_channels=512,+ groups=32,+ groups_out=None,+ pre_norm=True,+ eps=1e-6,+ non_linearity="swish",+ time_embedding_norm="default",+ output_scale_factor=1.0,+ use_in_shortcut=None,+ use_inflated_groupnorm=None,+ ):+ super().__init__()+ self.pre_norm = pre_norm+ self.pre_norm = True+ self.in_channels = in_channels+ out_channels = in_channels if out_channels is None else out_channels+ self.out_channels = out_channels+ self.use_conv_shortcut = conv_shortcut+ self.time_embedding_norm = time_embedding_norm+ self.output_scale_factor = output_scale_factor++ if groups_out is None:+ groups_out = groups++ assert use_inflated_groupnorm != None+ if use_inflated_groupnorm:+ self.norm1 = InflatedGroupNorm(+ num_groups=groups, num_channels=in_channels, eps=eps, affine=True+ )+ else:+ self.norm1 = torch.nn.GroupNorm(+ num_groups=groups, num_channels=in_channels, eps=eps, affine=True+ )++ self.conv1 = InflatedConv3d(+ in_channels, out_channels, kernel_size=3, stride=1, padding=1+ )++ if temb_channels is not None:+ if self.time_embedding_norm == "default":+ time_emb_proj_out_channels = out_channels+ elif self.time_embedding_norm == "scale_shift":+ time_emb_proj_out_channels = out_channels * 2+ else:+ raise ValueError(+ f"unknown time_embedding_norm : {self.time_embedding_norm} "+ )++ self.time_emb_proj = torch.nn.Linear(+ temb_channels, time_emb_proj_out_channels+ )+ else:+ self.time_emb_proj = None++ if use_inflated_groupnorm:+ self.norm2 = InflatedGroupNorm(+ num_groups=groups_out, num_channels=out_channels, eps=eps, affine=True+ )+ else:+ self.norm2 = torch.nn.GroupNorm(+ num_groups=groups_out, num_channels=out_channels, eps=eps, affine=True+ )+ self.dropout = torch.nn.Dropout(dropout)+ self.conv2 = InflatedConv3d(+ out_channels, out_channels, kernel_size=3, stride=1, padding=1+ )++ if non_linearity == "swish":+ self.nonlinearity = lambda x: F.silu(x)+ elif non_linearity == "mish":+ self.nonlinearity = Mish()+ elif non_linearity == "silu":+ self.nonlinearity = nn.SiLU()++ self.use_in_shortcut = (+ self.in_channels != self.out_channels+ if use_in_shortcut is None+ else use_in_shortcut+ )++ self.conv_shortcut = None+ if self.use_in_shortcut:+ self.conv_shortcut = InflatedConv3d(+ in_channels, out_channels, kernel_size=1, stride=1, padding=0+ )++ def forward(self, input_tensor, temb):+ hidden_states = input_tensor++ hidden_states = self.norm1(hidden_states)+ hidden_states = self.nonlinearity(hidden_states)++ hidden_states = self.conv1(hidden_states)++ if temb is not None:+ video_length = hidden_states.shape[2]+ temb = self.time_emb_proj(self.nonlinearity(temb))+ temb = rearrange(temb, "(b f) c -> b c f", f=video_length)[:, :, :, None, None]++ # if temb is not None:+ # temb = self.time_emb_proj(self.nonlinearity(temb))[:, :, None, None, None]++ if temb is not None and self.time_embedding_norm == "default":+ hidden_states = hidden_states + temb++ hidden_states = self.norm2(hidden_states)++ if temb is not None and self.time_embedding_norm == "scale_shift":+ scale, shift = torch.chunk(temb, 2, dim=1)+ hidden_states = hidden_states * (1 + scale) + shift++ hidden_states = self.nonlinearity(hidden_states)++ hidden_states = self.dropout(hidden_states)+ hidden_states = self.conv2(hidden_states)++ if self.conv_shortcut is not None:+ input_tensor = self.conv_shortcut(input_tensor)++ output_tensor = (input_tensor + hidden_states) / self.output_scale_factor++ return output_tensor++class Mish(torch.nn.Module):+ def forward(self, hidden_states):+ return hidden_states * torch.tanh(torch.nn.functional.softplus(hidden_states))diff --git a/src/models/transformer_2d.py b/src/models/transformer_2d.pynew file mode 100644index 0000000..4c0995b--- /dev/null+++ b/src/models/transformer_2d.py@@ -0,0 +1,395 @@+from dataclasses import dataclass+from typing import Any, Dict, Optional++import torch+from diffusers.configuration_utils import ConfigMixin, register_to_config+from diffusers.models.embeddings import PixArtAlphaTextProjection as CaptionProjection+from diffusers.models.lora import LoRACompatibleConv, LoRACompatibleLinear+from diffusers.models.modeling_utils import ModelMixin+from diffusers.models.normalization import AdaLayerNormSingle+from diffusers.utils import USE_PEFT_BACKEND, BaseOutput, deprecate, is_torch_version+from torch import nn++from .attention import BasicTransformerBlock+++@dataclass+class Transformer2DModelOutput(BaseOutput):+ """+ The output of [`Transformer2DModel`].++ Args:+ sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `(batch size, num_vector_embeds - 1, num_latent_pixels)` if [`Transformer2DModel`] is discrete):+ The hidden states output conditioned on the `encoder_hidden_states` input. If discrete, returns probability+ distributions for the unnoised latent pixels.+ """++ sample: torch.FloatTensor+ ref_feature: torch.FloatTensor+++class Transformer2DModel(ModelMixin, ConfigMixin):+ """+ A 2D Transformer model for image-like data.++ Parameters:+ num_attention_heads (`int`, *optional*, defaults to 16): The number of heads to use for multi-head attention.+ attention_head_dim (`int`, *optional*, defaults to 88): The number of channels in each head.+ in_channels (`int`, *optional*):+ The number of channels in the input and output (specify if the input is **continuous**).+ num_layers (`int`, *optional*, defaults to 1): The number of layers of Transformer blocks to use.+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.+ cross_attention_dim (`int`, *optional*): The number of `encoder_hidden_states` dimensions to use.+ sample_size (`int`, *optional*): The width of the latent images (specify if the input is **discrete**).+ This is fixed during training since it is used to learn a number of position embeddings.+ num_vector_embeds (`int`, *optional*):+ The number of classes of the vector embeddings of the latent pixels (specify if the input is **discrete**).+ Includes the class for the masked latent pixel.+ activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to use in feed-forward.+ num_embeds_ada_norm ( `int`, *optional*):+ The number of diffusion steps used during training. Pass if at least one of the norm_layers is+ `AdaLayerNorm`. This is fixed during training since it is used to learn a number of embeddings that are+ added to the hidden states.++ During inference, you can denoise for up to but not more steps than `num_embeds_ada_norm`.+ attention_bias (`bool`, *optional*):+ Configure if the `TransformerBlocks` attention should contain a bias parameter.+ """++ _supports_gradient_checkpointing = True++ @register_to_config+ def __init__(+ self,+ num_attention_heads: int = 16,+ attention_head_dim: int = 88,+ in_channels: Optional[int] = None,+ out_channels: Optional[int] = None,+ num_layers: int = 1,+ dropout: float = 0.0,+ norm_num_groups: int = 32,+ cross_attention_dim: Optional[int] = None,+ attention_bias: bool = False,+ sample_size: Optional[int] = None,+ num_vector_embeds: Optional[int] = None,+ patch_size: Optional[int] = None,+ activation_fn: str = "geglu",+ num_embeds_ada_norm: Optional[int] = None,+ use_linear_projection: bool = False,+ only_cross_attention: bool = False,+ double_self_attention: bool = False,+ upcast_attention: bool = False,+ norm_type: str = "layer_norm",+ norm_elementwise_affine: bool = True,+ norm_eps: float = 1e-5,+ attention_type: str = "default",+ caption_channels: int = None,+ ):+ super().__init__()+ self.use_linear_projection = use_linear_projection+ self.num_attention_heads = num_attention_heads+ self.attention_head_dim = attention_head_dim+ inner_dim = num_attention_heads * attention_head_dim++ conv_cls = nn.Conv2d if USE_PEFT_BACKEND else LoRACompatibleConv+ linear_cls = nn.Linear if USE_PEFT_BACKEND else LoRACompatibleLinear++ # 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`+ # Define whether input is continuous or discrete depending on configuration+ self.is_input_continuous = (in_channels is not None) and (patch_size is None)+ self.is_input_vectorized = num_vector_embeds is not None+ self.is_input_patches = in_channels is not None and patch_size is not None++ if norm_type == "layer_norm" and num_embeds_ada_norm is not None:+ deprecation_message = (+ f"The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or"+ " incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config."+ " Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect"+ " results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it"+ " would be very nice if you could open a Pull request for the `transformer/config.json` file"+ )+ deprecate(+ "norm_type!=num_embeds_ada_norm",+ "1.0.0",+ deprecation_message,+ standard_warn=False,+ )+ norm_type = "ada_norm"++ if self.is_input_continuous and self.is_input_vectorized:+ raise ValueError(+ f"Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make"+ " sure that either `in_channels` or `num_vector_embeds` is None."+ )+ elif self.is_input_vectorized and self.is_input_patches:+ raise ValueError(+ f"Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make"+ " sure that either `num_vector_embeds` or `num_patches` is None."+ )+ elif (+ not self.is_input_continuous+ and not self.is_input_vectorized+ and not self.is_input_patches+ ):+ raise ValueError(+ f"Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:"+ f" {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None."+ )++ # 2. Define input layers+ self.in_channels = in_channels++ self.norm = torch.nn.GroupNorm(+ num_groups=norm_num_groups,+ num_channels=in_channels,+ eps=1e-6,+ affine=True,+ )+ if use_linear_projection:+ self.proj_in = linear_cls(in_channels, inner_dim)+ else:+ self.proj_in = conv_cls(+ in_channels, inner_dim, kernel_size=1, stride=1, padding=0+ )++ # 3. Define transformers blocks+ self.transformer_blocks = nn.ModuleList(+ [+ BasicTransformerBlock(+ inner_dim,+ num_attention_heads,+ attention_head_dim,+ dropout=dropout,+ cross_attention_dim=cross_attention_dim,+ activation_fn=activation_fn,+ num_embeds_ada_norm=num_embeds_ada_norm,+ attention_bias=attention_bias,+ only_cross_attention=only_cross_attention,+ double_self_attention=double_self_attention,+ upcast_attention=upcast_attention,+ norm_type=norm_type,+ norm_elementwise_affine=norm_elementwise_affine,+ norm_eps=norm_eps,+ attention_type=attention_type,+ )+ for d in range(num_layers)+ ]+ )++ # 4. Define output layers+ self.out_channels = in_channels if out_channels is None else out_channels+ # TODO: should use out_channels for continuous projections+ if use_linear_projection:+ self.proj_out = linear_cls(inner_dim, in_channels)+ else:+ self.proj_out = conv_cls(+ inner_dim, in_channels, kernel_size=1, stride=1, padding=0+ )++ # 5. PixArt-Alpha blocks.+ self.adaln_single = None+ self.use_additional_conditions = False+ if norm_type == "ada_norm_single":+ self.use_additional_conditions = self.config.sample_size == 128+ # TODO(Sayak, PVP) clean this, for now we use sample size to determine whether to use+ # additional conditions until we find better name+ self.adaln_single = AdaLayerNormSingle(+ inner_dim, use_additional_conditions=self.use_additional_conditions+ )++ self.caption_projection = None+ if caption_channels is not None:+ self.caption_projection = CaptionProjection(+ in_features=caption_channels, hidden_size=inner_dim+ )++ self.gradient_checkpointing = False++ def _set_gradient_checkpointing(self, module, value=False):+ if hasattr(module, "gradient_checkpointing"):+ module.gradient_checkpointing = value++ def forward(+ self,+ hidden_states: torch.Tensor,+ encoder_hidden_states: Optional[torch.Tensor] = None,+ timestep: Optional[torch.LongTensor] = None,+ added_cond_kwargs: Dict[str, torch.Tensor] = None,+ class_labels: Optional[torch.LongTensor] = None,+ cross_attention_kwargs: Dict[str, Any] = None,+ attention_mask: Optional[torch.Tensor] = None,+ encoder_attention_mask: Optional[torch.Tensor] = None,+ return_dict: bool = True,+ ):+ """+ The [`Transformer2DModel`] forward method.++ Args:+ hidden_states (`torch.LongTensor` of shape `(batch size, num latent pixels)` if discrete, `torch.FloatTensor` of shape `(batch size, channel, height, width)` if continuous):+ Input `hidden_states`.+ encoder_hidden_states ( `torch.FloatTensor` of shape `(batch size, sequence len, embed dims)`, *optional*):+ Conditional embeddings for cross attention layer. If not given, cross-attention defaults to+ self-attention.+ timestep ( `torch.LongTensor`, *optional*):+ Used to indicate denoising step. Optional timestep to be applied as an embedding in `AdaLayerNorm`.+ class_labels ( `torch.LongTensor` of shape `(batch size, num classes)`, *optional*):+ Used to indicate class labels conditioning. Optional class labels to be applied as an embedding in+ `AdaLayerZeroNorm`.+ cross_attention_kwargs ( `Dict[str, Any]`, *optional*):+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under+ `self.processor` in+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).+ attention_mask ( `torch.Tensor`, *optional*):+ An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask+ is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large+ negative values to the attention scores corresponding to "discard" tokens.+ encoder_attention_mask ( `torch.Tensor`, *optional*):+ Cross-attention mask applied to `encoder_hidden_states`. Two formats supported:++ * Mask `(batch, sequence_length)` True = keep, False = discard.+ * Bias `(batch, 1, sequence_length)` 0 = keep, -10000 = discard.++ If `ndim == 2`: will be interpreted as a mask, then converted into a bias consistent with the format+ above. This bias will be added to the cross-attention scores.+ return_dict (`bool`, *optional*, defaults to `True`):+ Whether or not to return a [`~models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain+ tuple.++ Returns:+ If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a+ `tuple` where the first element is the sample tensor.+ """+ # ensure attention_mask is a bias, and give it a singleton query_tokens dimension.+ # we may have done this conversion already, e.g. if we came here via UNet2DConditionModel#forward.+ # we can tell by counting dims; if ndim == 2: it's a mask rather than a bias.+ # expects mask of shape:+ # [batch, key_tokens]+ # adds singleton query_tokens dimension:+ # [batch, 1, key_tokens]+ # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:+ # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn)+ # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)+ if attention_mask is not None and attention_mask.ndim == 2:+ # assume that mask is expressed as:+ # (1 = keep, 0 = discard)+ # convert mask into a bias that can be added to attention scores:+ # (keep = +0, discard = -10000.0)+ attention_mask = (1 - attention_mask.to(hidden_states.dtype)) * -10000.0+ attention_mask = attention_mask.unsqueeze(1)++ # convert encoder_attention_mask to a bias the same way we do for attention_mask+ if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:+ encoder_attention_mask = (+ 1 - encoder_attention_mask.to(hidden_states.dtype)+ ) * -10000.0+ encoder_attention_mask = encoder_attention_mask.unsqueeze(1)++ # Retrieve lora scale.+ lora_scale = (+ cross_attention_kwargs.get("scale", 1.0)+ if cross_attention_kwargs is not None+ else 1.0+ )++ # 1. Input+ batch, _, height, width = hidden_states.shape+ residual = hidden_states++ hidden_states = self.norm(hidden_states)+ if not self.use_linear_projection:+ hidden_states = (+ self.proj_in(hidden_states, scale=lora_scale)+ if not USE_PEFT_BACKEND+ else self.proj_in(hidden_states)+ )+ inner_dim = hidden_states.shape[1]+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(+ batch, height * width, inner_dim+ )+ else:+ inner_dim = hidden_states.shape[1]+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(+ batch, height * width, inner_dim+ )+ hidden_states = (+ self.proj_in(hidden_states, scale=lora_scale)+ if not USE_PEFT_BACKEND+ else self.proj_in(hidden_states)+ )++ # 2. Blocks+ if self.caption_projection is not None:+ batch_size = hidden_states.shape[0]+ encoder_hidden_states = self.caption_projection(encoder_hidden_states)+ encoder_hidden_states = encoder_hidden_states.view(+ batch_size, -1, hidden_states.shape[-1]+ )++ ref_feature = hidden_states.reshape(batch, height, width, inner_dim)+ for block in self.transformer_blocks:+ if self.training and self.gradient_checkpointing:++ def create_custom_forward(module, return_dict=None):+ def custom_forward(*inputs):+ if return_dict is not None:+ return module(*inputs, return_dict=return_dict)+ else:+ return module(*inputs)++ return custom_forward++ ckpt_kwargs: Dict[str, Any] = (+ {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {}+ )+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(block),+ hidden_states,+ attention_mask,+ encoder_hidden_states,+ encoder_attention_mask,+ timestep,+ cross_attention_kwargs,+ class_labels,+ **ckpt_kwargs,+ )+ else:+ hidden_states = block(+ hidden_states,+ attention_mask=attention_mask,+ encoder_hidden_states=encoder_hidden_states,+ encoder_attention_mask=encoder_attention_mask,+ timestep=timestep,+ cross_attention_kwargs=cross_attention_kwargs,+ class_labels=class_labels,+ )++ # 3. Output+ if self.is_input_continuous:+ if not self.use_linear_projection:+ hidden_states = (+ hidden_states.reshape(batch, height, width, inner_dim)+ .permute(0, 3, 1, 2)+ .contiguous()+ )+ hidden_states = (+ self.proj_out(hidden_states, scale=lora_scale)+ if not USE_PEFT_BACKEND+ else self.proj_out(hidden_states)+ )+ else:+ hidden_states = (+ self.proj_out(hidden_states, scale=lora_scale)+ if not USE_PEFT_BACKEND+ else self.proj_out(hidden_states)+ )+ hidden_states = (+ hidden_states.reshape(batch, height, width, inner_dim)+ .permute(0, 3, 1, 2)+ .contiguous()+ )++ output = hidden_states + residual+ if not return_dict:+ return (output, ref_feature)++ return Transformer2DModelOutput(sample=output, ref_feature=ref_feature)diff --git a/src/models/transformer_3d.py b/src/models/transformer_3d.pynew file mode 100644index 0000000..d10a3a4--- /dev/null+++ b/src/models/transformer_3d.py@@ -0,0 +1,182 @@+from dataclasses import dataclass+from typing import Optional, Dict++import torch+from diffusers.configuration_utils import ConfigMixin, register_to_config+from diffusers.models import ModelMixin+from diffusers.utils import BaseOutput+from diffusers.utils.import_utils import is_xformers_available+from einops import rearrange, repeat+from torch import nn++from .attention import TemporalBasicTransformerBlock+++@dataclass+class Transformer3DModelOutput(BaseOutput):+ sample: torch.FloatTensor+++if is_xformers_available():+ import xformers+ import xformers.ops+else:+ xformers = None+++class Transformer3DModel(ModelMixin, ConfigMixin):+ _supports_gradient_checkpointing = True++ @register_to_config+ def __init__(+ self,+ num_attention_heads: int = 16,+ attention_head_dim: int = 88,+ in_channels: Optional[int] = None,+ num_layers: int = 1,+ dropout: float = 0.0,+ norm_num_groups: int = 32,+ cross_attention_dim: Optional[int] = None,+ attention_bias: bool = False,+ activation_fn: str = "geglu",+ num_embeds_ada_norm: Optional[int] = None,+ use_linear_projection: bool = False,+ only_cross_attention: bool = False,+ upcast_attention: bool = False,+ unet_use_cross_frame_attention=None,+ unet_use_temporal_attention=None,+ ):+ super().__init__()+ self.use_linear_projection = use_linear_projection+ self.num_attention_heads = num_attention_heads+ self.attention_head_dim = attention_head_dim+ inner_dim = num_attention_heads * attention_head_dim++ # Define input layers+ self.in_channels = in_channels++ self.norm = torch.nn.GroupNorm(+ num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True+ )+ if use_linear_projection:+ self.proj_in = nn.Linear(in_channels, inner_dim)+ else:+ self.proj_in = nn.Conv2d(+ in_channels, inner_dim, kernel_size=1, stride=1, padding=0+ )++ # Define transformers blocks+ self.transformer_blocks = nn.ModuleList(+ [+ TemporalBasicTransformerBlock(+ inner_dim,+ num_attention_heads,+ attention_head_dim,+ dropout=dropout,+ cross_attention_dim=cross_attention_dim,+ activation_fn=activation_fn,+ num_embeds_ada_norm=num_embeds_ada_norm,+ attention_bias=attention_bias,+ only_cross_attention=only_cross_attention,+ upcast_attention=upcast_attention,+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,+ unet_use_temporal_attention=unet_use_temporal_attention,+ )+ for d in range(num_layers)+ ]+ )++ # 4. Define output layers+ if use_linear_projection:+ self.proj_out = nn.Linear(in_channels, inner_dim)+ else:+ self.proj_out = nn.Conv2d(+ inner_dim, in_channels, kernel_size=1, stride=1, padding=0+ )++ self.gradient_checkpointing = False++ def _set_gradient_checkpointing(self, module, value=False):+ if hasattr(module, "gradient_checkpointing"):+ module.gradient_checkpointing = value++ def forward(+ self,+ hidden_states,+ encoder_hidden_states=None,+ timestep=None,+ return_dict: bool = True,+ reference=None,+ **block_kwargs,+ ):+ # Input+ assert (+ hidden_states.dim() == 5+ ), f"Expected hidden_states to have ndim=5, but got ndim={hidden_states.dim()}."+ video_length = hidden_states.shape[2]+ hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w")+ if encoder_hidden_states.shape[0] != hidden_states.shape[0]:+ encoder_hidden_states = repeat(+ encoder_hidden_states, "b n c -> (b f) n c", f=video_length+ )++ batch, channel, height, weight = hidden_states.shape+ residual = hidden_states++ hidden_states = self.norm(hidden_states)+ if not self.use_linear_projection:+ hidden_states = self.proj_in(hidden_states)+ inner_dim = hidden_states.shape[1]+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(+ batch, height * weight, inner_dim+ )+ else:+ inner_dim = hidden_states.shape[1]+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(+ batch, height * weight, inner_dim+ )+ hidden_states = self.proj_in(hidden_states)++ # Blocks+ for i, block in enumerate(self.transformer_blocks):+ if reference is None:+ hidden_states = block(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ timestep=timestep,+ video_length=video_length,+ **block_kwargs+ )+ else:+ hidden_states = block(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ timestep=timestep,+ video_length=video_length,+ reference=reference,+ **block_kwargs+ )++ # Output+ if not self.use_linear_projection:+ hidden_states = (+ hidden_states.reshape(batch, height, weight, inner_dim)+ .permute(0, 3, 1, 2)+ .contiguous()+ )+ hidden_states = self.proj_out(hidden_states)+ else:+ hidden_states = self.proj_out(hidden_states)+ hidden_states = (+ hidden_states.reshape(batch, height, weight, inner_dim)+ .permute(0, 3, 1, 2)+ .contiguous()+ )++ output = hidden_states + residual++ output = rearrange(output, "(b f) c h w -> b c f h w", f=video_length)+ if not return_dict:+ return (output,)++ return Transformer3DModelOutput(sample=output)diff --git a/src/models/unet_2d_blocks.py b/src/models/unet_2d_blocks.pynew file mode 100644index 0000000..99c7ed0--- /dev/null+++ b/src/models/unet_2d_blocks.py@@ -0,0 +1,1073 @@+from typing import Any, Dict, Optional, Tuple, Union++import numpy as np+import torch+import torch.nn.functional as F+from diffusers.models.activations import get_activation+from diffusers.models.attention_processor import Attention+from diffusers.models import DualTransformer2DModel+from diffusers.models.resnet import Downsample2D, ResnetBlock2D, Upsample2D+from diffusers.utils import is_torch_version, logging+from diffusers.utils.torch_utils import apply_freeu+from torch import nn++from .transformer_2d import Transformer2DModel++logger = logging.get_logger(__name__) # pylint: disable=invalid-name+++def get_down_block(+ down_block_type: str,+ num_layers: int,+ in_channels: int,+ out_channels: int,+ temb_channels: int,+ add_downsample: bool,+ resnet_eps: float,+ resnet_act_fn: str,+ transformer_layers_per_block: int = 1,+ num_attention_heads: Optional[int] = None,+ resnet_groups: Optional[int] = None,+ cross_attention_dim: Optional[int] = None,+ downsample_padding: Optional[int] = None,+ dual_cross_attention: bool = False,+ use_linear_projection: bool = False,+ only_cross_attention: bool = False,+ upcast_attention: bool = False,+ resnet_time_scale_shift: str = "default",+ attention_type: str = "default",+ resnet_skip_time_act: bool = False,+ resnet_out_scale_factor: float = 1.0,+ cross_attention_norm: Optional[str] = None,+ attention_head_dim: Optional[int] = None,+ downsample_type: Optional[str] = None,+ dropout: float = 0.0,+):+ # If attn head dim is not defined, we default it to the number of heads+ if attention_head_dim is None:+ logger.warn(+ f"It is recommended to provide `attention_head_dim` when calling `get_down_block`. Defaulting `attention_head_dim` to {num_attention_heads}."+ )+ attention_head_dim = num_attention_heads++ down_block_type = (+ down_block_type[7:]+ if down_block_type.startswith("UNetRes")+ else down_block_type+ )+ if down_block_type == "DownBlock2D":+ return DownBlock2D(+ num_layers=num_layers,+ in_channels=in_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ dropout=dropout,+ add_downsample=add_downsample,+ resnet_eps=resnet_eps,+ resnet_act_fn=resnet_act_fn,+ resnet_groups=resnet_groups,+ downsample_padding=downsample_padding,+ resnet_time_scale_shift=resnet_time_scale_shift,+ )+ elif down_block_type == "CrossAttnDownBlock2D":+ if cross_attention_dim is None:+ raise ValueError(+ "cross_attention_dim must be specified for CrossAttnDownBlock2D"+ )+ return CrossAttnDownBlock2D(+ num_layers=num_layers,+ transformer_layers_per_block=transformer_layers_per_block,+ in_channels=in_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ dropout=dropout,+ add_downsample=add_downsample,+ resnet_eps=resnet_eps,+ resnet_act_fn=resnet_act_fn,+ resnet_groups=resnet_groups,+ downsample_padding=downsample_padding,+ cross_attention_dim=cross_attention_dim,+ num_attention_heads=num_attention_heads,+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention,+ upcast_attention=upcast_attention,+ resnet_time_scale_shift=resnet_time_scale_shift,+ attention_type=attention_type,+ )+ raise ValueError(f"{down_block_type} does not exist.")+++def get_up_block(+ up_block_type: str,+ num_layers: int,+ in_channels: int,+ out_channels: int,+ prev_output_channel: int,+ temb_channels: int,+ add_upsample: bool,+ resnet_eps: float,+ resnet_act_fn: str,+ resolution_idx: Optional[int] = None,+ transformer_layers_per_block: int = 1,+ num_attention_heads: Optional[int] = None,+ resnet_groups: Optional[int] = None,+ cross_attention_dim: Optional[int] = None,+ dual_cross_attention: bool = False,+ use_linear_projection: bool = False,+ only_cross_attention: bool = False,+ upcast_attention: bool = False,+ resnet_time_scale_shift: str = "default",+ attention_type: str = "default",+ resnet_skip_time_act: bool = False,+ resnet_out_scale_factor: float = 1.0,+ cross_attention_norm: Optional[str] = None,+ attention_head_dim: Optional[int] = None,+ upsample_type: Optional[str] = None,+ dropout: float = 0.0,+) -> nn.Module:+ # If attn head dim is not defined, we default it to the number of heads+ if attention_head_dim is None:+ logger.warn(+ f"It is recommended to provide `attention_head_dim` when calling `get_up_block`. Defaulting `attention_head_dim` to {num_attention_heads}."+ )+ attention_head_dim = num_attention_heads++ up_block_type = (+ up_block_type[7:] if up_block_type.startswith("UNetRes") else up_block_type+ )+ if up_block_type == "UpBlock2D":+ return UpBlock2D(+ num_layers=num_layers,+ in_channels=in_channels,+ out_channels=out_channels,+ prev_output_channel=prev_output_channel,+ temb_channels=temb_channels,+ resolution_idx=resolution_idx,+ dropout=dropout,+ add_upsample=add_upsample,+ resnet_eps=resnet_eps,+ resnet_act_fn=resnet_act_fn,+ resnet_groups=resnet_groups,+ resnet_time_scale_shift=resnet_time_scale_shift,+ )+ elif up_block_type == "CrossAttnUpBlock2D":+ if cross_attention_dim is None:+ raise ValueError(+ "cross_attention_dim must be specified for CrossAttnUpBlock2D"+ )+ return CrossAttnUpBlock2D(+ num_layers=num_layers,+ transformer_layers_per_block=transformer_layers_per_block,+ in_channels=in_channels,+ out_channels=out_channels,+ prev_output_channel=prev_output_channel,+ temb_channels=temb_channels,+ resolution_idx=resolution_idx,+ dropout=dropout,+ add_upsample=add_upsample,+ resnet_eps=resnet_eps,+ resnet_act_fn=resnet_act_fn,+ resnet_groups=resnet_groups,+ cross_attention_dim=cross_attention_dim,+ num_attention_heads=num_attention_heads,+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention,+ upcast_attention=upcast_attention,+ resnet_time_scale_shift=resnet_time_scale_shift,+ attention_type=attention_type,+ )++ raise ValueError(f"{up_block_type} does not exist.")+++class AutoencoderTinyBlock(nn.Module):+ """+ Tiny Autoencoder block used in [`AutoencoderTiny`]. It is a mini residual module consisting of plain conv + ReLU+ blocks.++ Args:+ in_channels (`int`): The number of input channels.+ out_channels (`int`): The number of output channels.+ act_fn (`str`):+ ` The activation function to use. Supported values are `"swish"`, `"mish"`, `"gelu"`, and `"relu"`.++ Returns:+ `torch.FloatTensor`: A tensor with the same shape as the input tensor, but with the number of channels equal to+ `out_channels`.+ """++ def __init__(self, in_channels: int, out_channels: int, act_fn: str):+ super().__init__()+ act_fn = get_activation(act_fn)+ self.conv = nn.Sequential(+ nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1),+ act_fn,+ nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1),+ act_fn,+ nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1),+ )+ self.skip = (+ nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False)+ if in_channels != out_channels+ else nn.Identity()+ )+ self.fuse = nn.ReLU()++ def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:+ return self.fuse(self.conv(x) + self.skip(x))+++class UNetMidBlock2D(nn.Module):+ """+ A 2D UNet mid-block [`UNetMidBlock2D`] with multiple residual blocks and optional attention blocks.++ Args:+ in_channels (`int`): The number of input channels.+ temb_channels (`int`): The number of temporal embedding channels.+ dropout (`float`, *optional*, defaults to 0.0): The dropout rate.+ num_layers (`int`, *optional*, defaults to 1): The number of residual blocks.+ resnet_eps (`float`, *optional*, 1e-6 ): The epsilon value for the resnet blocks.+ resnet_time_scale_shift (`str`, *optional*, defaults to `default`):+ The type of normalization to apply to the time embeddings. This can help to improve the performance of the+ model on tasks with long-range temporal dependencies.+ resnet_act_fn (`str`, *optional*, defaults to `swish`): The activation function for the resnet blocks.+ resnet_groups (`int`, *optional*, defaults to 32):+ The number of groups to use in the group normalization layers of the resnet blocks.+ attn_groups (`Optional[int]`, *optional*, defaults to None): The number of groups for the attention blocks.+ resnet_pre_norm (`bool`, *optional*, defaults to `True`):+ Whether to use pre-normalization for the resnet blocks.+ add_attention (`bool`, *optional*, defaults to `True`): Whether to add attention blocks.+ attention_head_dim (`int`, *optional*, defaults to 1):+ Dimension of a single attention head. The number of attention heads is determined based on this value and+ the number of input channels.+ output_scale_factor (`float`, *optional*, defaults to 1.0): The output scale factor.++ Returns:+ `torch.FloatTensor`: The output of the last residual block, which is a tensor of shape `(batch_size,+ in_channels, height, width)`.++ """++ def __init__(+ self,+ in_channels: int,+ temb_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default", # default, spatial+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ attn_groups: Optional[int] = None,+ resnet_pre_norm: bool = True,+ add_attention: bool = True,+ attention_head_dim: int = 1,+ output_scale_factor: float = 1.0,+ ):+ super().__init__()+ resnet_groups = (+ resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)+ )+ self.add_attention = add_attention++ if attn_groups is None:+ attn_groups = (+ resnet_groups if resnet_time_scale_shift == "default" else None+ )++ # there is always at least one resnet+ resnets = [+ ResnetBlock2D(+ in_channels=in_channels,+ out_channels=in_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ )+ ]+ attentions = []++ if attention_head_dim is None:+ logger.warn(+ f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: {in_channels}."+ )+ attention_head_dim = in_channels++ for _ in range(num_layers):+ if self.add_attention:+ attentions.append(+ Attention(+ in_channels,+ heads=in_channels // attention_head_dim,+ dim_head=attention_head_dim,+ rescale_output_factor=output_scale_factor,+ eps=resnet_eps,+ norm_num_groups=attn_groups,+ spatial_norm_dim=temb_channels+ if resnet_time_scale_shift == "spatial"+ else None,+ residual_connection=True,+ bias=True,+ upcast_softmax=True,+ _from_deprecated_attn_block=True,+ )+ )+ else:+ attentions.append(None)++ resnets.append(+ ResnetBlock2D(+ in_channels=in_channels,+ out_channels=in_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ )+ )++ self.attentions = nn.ModuleList(attentions)+ self.resnets = nn.ModuleList(resnets)++ def forward(+ self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None+ ) -> torch.FloatTensor:+ hidden_states = self.resnets[0](hidden_states, temb)+ for attn, resnet in zip(self.attentions, self.resnets[1:]):+ if attn is not None:+ hidden_states = attn(hidden_states, temb=temb)+ hidden_states = resnet(hidden_states, temb)++ return hidden_states+++class UNetMidBlock2DCrossAttn(nn.Module):+ def __init__(+ self,+ in_channels: int,+ temb_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ transformer_layers_per_block: Union[int, Tuple[int]] = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default",+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ num_attention_heads: int = 1,+ output_scale_factor: float = 1.0,+ cross_attention_dim: int = 1280,+ dual_cross_attention: bool = False,+ use_linear_projection: bool = False,+ upcast_attention: bool = False,+ attention_type: str = "default",+ ):+ super().__init__()++ self.has_cross_attention = True+ self.num_attention_heads = num_attention_heads+ resnet_groups = (+ resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)+ )++ # support for variable transformer layers per block+ if isinstance(transformer_layers_per_block, int):+ transformer_layers_per_block = [transformer_layers_per_block] * num_layers++ # there is always at least one resnet+ resnets = [+ ResnetBlock2D(+ in_channels=in_channels,+ out_channels=in_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ )+ ]+ attentions = []++ for i in range(num_layers):+ if not dual_cross_attention:+ attentions.append(+ Transformer2DModel(+ num_attention_heads,+ in_channels // num_attention_heads,+ in_channels=in_channels,+ num_layers=transformer_layers_per_block[i],+ cross_attention_dim=cross_attention_dim,+ norm_num_groups=resnet_groups,+ use_linear_projection=use_linear_projection,+ upcast_attention=upcast_attention,+ attention_type=attention_type,+ )+ )+ else:+ attentions.append(+ DualTransformer2DModel(+ num_attention_heads,+ in_channels // num_attention_heads,+ in_channels=in_channels,+ num_layers=1,+ cross_attention_dim=cross_attention_dim,+ norm_num_groups=resnet_groups,+ )+ )+ resnets.append(+ ResnetBlock2D(+ in_channels=in_channels,+ out_channels=in_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ )+ )++ self.attentions = nn.ModuleList(attentions)+ self.resnets = nn.ModuleList(resnets)++ self.gradient_checkpointing = False++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ temb: Optional[torch.FloatTensor] = None,+ encoder_hidden_states: Optional[torch.FloatTensor] = None,+ attention_mask: Optional[torch.FloatTensor] = None,+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,+ encoder_attention_mask: Optional[torch.FloatTensor] = None,+ ) -> torch.FloatTensor:+ lora_scale = (+ cross_attention_kwargs.get("scale", 1.0)+ if cross_attention_kwargs is not None+ else 1.0+ )+ hidden_states = self.resnets[0](hidden_states, temb, scale=lora_scale)+ for attn, resnet in zip(self.attentions, self.resnets[1:]):+ if self.training and self.gradient_checkpointing:++ def create_custom_forward(module, return_dict=None):+ def custom_forward(*inputs):+ if return_dict is not None:+ return module(*inputs, return_dict=return_dict)+ else:+ return module(*inputs)++ return custom_forward++ ckpt_kwargs: Dict[str, Any] = (+ {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {}+ )+ hidden_states, ref_feature = attn(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ cross_attention_kwargs=cross_attention_kwargs,+ attention_mask=attention_mask,+ encoder_attention_mask=encoder_attention_mask,+ return_dict=False,+ )+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet),+ hidden_states,+ temb,+ **ckpt_kwargs,+ )+ else:+ hidden_states, ref_feature = attn(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ cross_attention_kwargs=cross_attention_kwargs,+ attention_mask=attention_mask,+ encoder_attention_mask=encoder_attention_mask,+ return_dict=False,+ )+ hidden_states = resnet(hidden_states, temb, scale=lora_scale)++ return hidden_states+++class CrossAttnDownBlock2D(nn.Module):+ def __init__(+ self,+ in_channels: int,+ out_channels: int,+ temb_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ transformer_layers_per_block: Union[int, Tuple[int]] = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default",+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ num_attention_heads: int = 1,+ cross_attention_dim: int = 1280,+ output_scale_factor: float = 1.0,+ downsample_padding: int = 1,+ add_downsample: bool = True,+ dual_cross_attention: bool = False,+ use_linear_projection: bool = False,+ only_cross_attention: bool = False,+ upcast_attention: bool = False,+ attention_type: str = "default",+ ):+ super().__init__()+ resnets = []+ attentions = []++ self.has_cross_attention = True+ self.num_attention_heads = num_attention_heads+ if isinstance(transformer_layers_per_block, int):+ transformer_layers_per_block = [transformer_layers_per_block] * num_layers++ for i in range(num_layers):+ in_channels = in_channels if i == 0 else out_channels+ resnets.append(+ ResnetBlock2D(+ in_channels=in_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ )+ )+ if not dual_cross_attention:+ attentions.append(+ Transformer2DModel(+ num_attention_heads,+ out_channels // num_attention_heads,+ in_channels=out_channels,+ num_layers=transformer_layers_per_block[i],+ cross_attention_dim=cross_attention_dim,+ norm_num_groups=resnet_groups,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention,+ upcast_attention=upcast_attention,+ attention_type=attention_type,+ )+ )+ else:+ attentions.append(+ DualTransformer2DModel(+ num_attention_heads,+ out_channels // num_attention_heads,+ in_channels=out_channels,+ num_layers=1,+ cross_attention_dim=cross_attention_dim,+ norm_num_groups=resnet_groups,+ )+ )+ self.attentions = nn.ModuleList(attentions)+ self.resnets = nn.ModuleList(resnets)++ if add_downsample:+ self.downsamplers = nn.ModuleList(+ [+ Downsample2D(+ out_channels,+ use_conv=True,+ out_channels=out_channels,+ padding=downsample_padding,+ name="op",+ )+ ]+ )+ else:+ self.downsamplers = None++ self.gradient_checkpointing = False++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ temb: Optional[torch.FloatTensor] = None,+ encoder_hidden_states: Optional[torch.FloatTensor] = None,+ attention_mask: Optional[torch.FloatTensor] = None,+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,+ encoder_attention_mask: Optional[torch.FloatTensor] = None,+ additional_residuals: Optional[torch.FloatTensor] = None,+ ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]:+ output_states = ()++ lora_scale = (+ cross_attention_kwargs.get("scale", 1.0)+ if cross_attention_kwargs is not None+ else 1.0+ )++ blocks = list(zip(self.resnets, self.attentions))++ for i, (resnet, attn) in enumerate(blocks):+ if self.training and self.gradient_checkpointing:++ def create_custom_forward(module, return_dict=None):+ def custom_forward(*inputs):+ if return_dict is not None:+ return module(*inputs, return_dict=return_dict)+ else:+ return module(*inputs)++ return custom_forward++ ckpt_kwargs: Dict[str, Any] = (+ {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {}+ )+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet),+ hidden_states,+ temb,+ **ckpt_kwargs,+ )+ hidden_states, ref_feature = attn(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ cross_attention_kwargs=cross_attention_kwargs,+ attention_mask=attention_mask,+ encoder_attention_mask=encoder_attention_mask,+ return_dict=False,+ )+ else:+ hidden_states = resnet(hidden_states, temb, scale=lora_scale)+ hidden_states, ref_feature = attn(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ cross_attention_kwargs=cross_attention_kwargs,+ attention_mask=attention_mask,+ encoder_attention_mask=encoder_attention_mask,+ return_dict=False,+ )++ # apply additional residuals to the output of the last pair of resnet and attention blocks+ if i == len(blocks) - 1 and additional_residuals is not None:+ hidden_states = hidden_states + additional_residuals++ output_states = output_states + (hidden_states,)++ if self.downsamplers is not None:+ for downsampler in self.downsamplers:+ hidden_states = downsampler(hidden_states, scale=lora_scale)++ output_states = output_states + (hidden_states,)++ return hidden_states, output_states+++class DownBlock2D(nn.Module):+ def __init__(+ self,+ in_channels: int,+ out_channels: int,+ temb_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default",+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ output_scale_factor: float = 1.0,+ add_downsample: bool = True,+ downsample_padding: int = 1,+ ):+ super().__init__()+ resnets = []++ for i in range(num_layers):+ in_channels = in_channels if i == 0 else out_channels+ resnets.append(+ ResnetBlock2D(+ in_channels=in_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ )+ )++ self.resnets = nn.ModuleList(resnets)++ if add_downsample:+ self.downsamplers = nn.ModuleList(+ [+ Downsample2D(+ out_channels,+ use_conv=True,+ out_channels=out_channels,+ padding=downsample_padding,+ name="op",+ )+ ]+ )+ else:+ self.downsamplers = None++ self.gradient_checkpointing = False++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ temb: Optional[torch.FloatTensor] = None,+ scale: float = 1.0,+ ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]:+ output_states = ()++ for resnet in self.resnets:+ if self.training and self.gradient_checkpointing:++ def create_custom_forward(module):+ def custom_forward(*inputs):+ return module(*inputs)++ return custom_forward++ if is_torch_version(">=", "1.11.0"):+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet),+ hidden_states,+ temb,+ use_reentrant=False,+ )+ else:+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet), hidden_states, temb+ )+ else:+ hidden_states = resnet(hidden_states, temb, scale=scale)++ output_states = output_states + (hidden_states,)++ if self.downsamplers is not None:+ for downsampler in self.downsamplers:+ hidden_states = downsampler(hidden_states, scale=scale)++ output_states = output_states + (hidden_states,)++ return hidden_states, output_states+++class CrossAttnUpBlock2D(nn.Module):+ def __init__(+ self,+ in_channels: int,+ out_channels: int,+ prev_output_channel: int,+ temb_channels: int,+ resolution_idx: Optional[int] = None,+ dropout: float = 0.0,+ num_layers: int = 1,+ transformer_layers_per_block: Union[int, Tuple[int]] = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default",+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ num_attention_heads: int = 1,+ cross_attention_dim: int = 1280,+ output_scale_factor: float = 1.0,+ add_upsample: bool = True,+ dual_cross_attention: bool = False,+ use_linear_projection: bool = False,+ only_cross_attention: bool = False,+ upcast_attention: bool = False,+ attention_type: str = "default",+ ):+ super().__init__()+ resnets = []+ attentions = []++ self.has_cross_attention = True+ self.num_attention_heads = num_attention_heads++ if isinstance(transformer_layers_per_block, int):+ transformer_layers_per_block = [transformer_layers_per_block] * num_layers++ for i in range(num_layers):+ res_skip_channels = in_channels if (i == num_layers - 1) else out_channels+ resnet_in_channels = prev_output_channel if i == 0 else out_channels++ resnets.append(+ ResnetBlock2D(+ in_channels=resnet_in_channels + res_skip_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ )+ )+ if not dual_cross_attention:+ attentions.append(+ Transformer2DModel(+ num_attention_heads,+ out_channels // num_attention_heads,+ in_channels=out_channels,+ num_layers=transformer_layers_per_block[i],+ cross_attention_dim=cross_attention_dim,+ norm_num_groups=resnet_groups,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention,+ upcast_attention=upcast_attention,+ attention_type=attention_type,+ )+ )+ else:+ attentions.append(+ DualTransformer2DModel(+ num_attention_heads,+ out_channels // num_attention_heads,+ in_channels=out_channels,+ num_layers=1,+ cross_attention_dim=cross_attention_dim,+ norm_num_groups=resnet_groups,+ )+ )+ self.attentions = nn.ModuleList(attentions)+ self.resnets = nn.ModuleList(resnets)++ if add_upsample:+ self.upsamplers = nn.ModuleList(+ [Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]+ )+ else:+ self.upsamplers = None++ self.gradient_checkpointing = False+ self.resolution_idx = resolution_idx++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ res_hidden_states_tuple: Tuple[torch.FloatTensor, ...],+ temb: Optional[torch.FloatTensor] = None,+ encoder_hidden_states: Optional[torch.FloatTensor] = None,+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,+ upsample_size: Optional[int] = None,+ attention_mask: Optional[torch.FloatTensor] = None,+ encoder_attention_mask: Optional[torch.FloatTensor] = None,+ ) -> torch.FloatTensor:+ lora_scale = (+ cross_attention_kwargs.get("scale", 1.0)+ if cross_attention_kwargs is not None+ else 1.0+ )+ is_freeu_enabled = (+ getattr(self, "s1", None)+ and getattr(self, "s2", None)+ and getattr(self, "b1", None)+ and getattr(self, "b2", None)+ )++ for resnet, attn in zip(self.resnets, self.attentions):+ # pop res hidden states+ res_hidden_states = res_hidden_states_tuple[-1]+ res_hidden_states_tuple = res_hidden_states_tuple[:-1]++ # FreeU: Only operate on the first two stages+ if is_freeu_enabled:+ hidden_states, res_hidden_states = apply_freeu(+ self.resolution_idx,+ hidden_states,+ res_hidden_states,+ s1=self.s1,+ s2=self.s2,+ b1=self.b1,+ b2=self.b2,+ )++ hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)++ if self.training and self.gradient_checkpointing:++ def create_custom_forward(module, return_dict=None):+ def custom_forward(*inputs):+ if return_dict is not None:+ return module(*inputs, return_dict=return_dict)+ else:+ return module(*inputs)++ return custom_forward++ ckpt_kwargs: Dict[str, Any] = (+ {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {}+ )+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet),+ hidden_states,+ temb,+ **ckpt_kwargs,+ )+ hidden_states, ref_feature = attn(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ cross_attention_kwargs=cross_attention_kwargs,+ attention_mask=attention_mask,+ encoder_attention_mask=encoder_attention_mask,+ return_dict=False,+ )+ else:+ hidden_states = resnet(hidden_states, temb, scale=lora_scale)+ hidden_states, ref_feature = attn(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ cross_attention_kwargs=cross_attention_kwargs,+ attention_mask=attention_mask,+ encoder_attention_mask=encoder_attention_mask,+ return_dict=False,+ )++ if self.upsamplers is not None:+ for upsampler in self.upsamplers:+ hidden_states = upsampler(+ hidden_states, upsample_size, scale=lora_scale+ )++ return hidden_states+++class UpBlock2D(nn.Module):+ def __init__(+ self,+ in_channels: int,+ prev_output_channel: int,+ out_channels: int,+ temb_channels: int,+ resolution_idx: Optional[int] = None,+ dropout: float = 0.0,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default",+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ output_scale_factor: float = 1.0,+ add_upsample: bool = True,+ ):+ super().__init__()+ resnets = []++ for i in range(num_layers):+ res_skip_channels = in_channels if (i == num_layers - 1) else out_channels+ resnet_in_channels = prev_output_channel if i == 0 else out_channels++ resnets.append(+ ResnetBlock2D(+ in_channels=resnet_in_channels + res_skip_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ )+ )++ self.resnets = nn.ModuleList(resnets)++ if add_upsample:+ self.upsamplers = nn.ModuleList(+ [Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]+ )+ else:+ self.upsamplers = None++ self.gradient_checkpointing = False+ self.resolution_idx = resolution_idx++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ res_hidden_states_tuple: Tuple[torch.FloatTensor, ...],+ temb: Optional[torch.FloatTensor] = None,+ upsample_size: Optional[int] = None,+ scale: float = 1.0,+ ) -> torch.FloatTensor:+ is_freeu_enabled = (+ getattr(self, "s1", None)+ and getattr(self, "s2", None)+ and getattr(self, "b1", None)+ and getattr(self, "b2", None)+ )++ for resnet in self.resnets:+ # pop res hidden states+ res_hidden_states = res_hidden_states_tuple[-1]+ res_hidden_states_tuple = res_hidden_states_tuple[:-1]++ # FreeU: Only operate on the first two stages+ if is_freeu_enabled:+ hidden_states, res_hidden_states = apply_freeu(+ self.resolution_idx,+ hidden_states,+ res_hidden_states,+ s1=self.s1,+ s2=self.s2,+ b1=self.b1,+ b2=self.b2,+ )++ hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)++ if self.training and self.gradient_checkpointing:++ def create_custom_forward(module):+ def custom_forward(*inputs):+ return module(*inputs)++ return custom_forward++ if is_torch_version(">=", "1.11.0"):+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet),+ hidden_states,+ temb,+ use_reentrant=False,+ )+ else:+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet), hidden_states, temb+ )+ else:+ hidden_states = resnet(hidden_states, temb, scale=scale)++ if self.upsamplers is not None:+ for upsampler in self.upsamplers:+ hidden_states = upsampler(hidden_states, upsample_size, scale=scale)++ return hidden_statesdiff --git a/src/models/unet_2d_condition.py b/src/models/unet_2d_condition.pynew file mode 100644index 0000000..38969ba--- /dev/null+++ b/src/models/unet_2d_condition.py@@ -0,0 +1,1311 @@+from dataclasses import dataclass+from typing import Any, Dict, List, Optional, Tuple, Union++import torch+import torch.nn as nn+import torch.utils.checkpoint+from diffusers.configuration_utils import ConfigMixin, register_to_config+from diffusers.loaders import UNet2DConditionLoadersMixin+from diffusers.models.activations import get_activation+from diffusers.models.attention_processor import (+ ADDED_KV_ATTENTION_PROCESSORS,+ CROSS_ATTENTION_PROCESSORS,+ AttentionProcessor,+ AttnAddedKVProcessor,+ AttnProcessor,+)+from diffusers.models.embeddings import (+ GaussianFourierProjection,+ ImageHintTimeEmbedding,+ ImageProjection,+ ImageTimeEmbedding,+ TextImageProjection,+ TextImageTimeEmbedding,+ TextTimeEmbedding,+ TimestepEmbedding,+ Timesteps,+)+from diffusers.models.embeddings import GLIGENTextBoundingboxProjection as PositionNet+from diffusers.models.modeling_utils import ModelMixin+from diffusers.utils import (+ USE_PEFT_BACKEND,+ BaseOutput,+ deprecate,+ logging,+ scale_lora_layers,+ unscale_lora_layers,+)++from .unet_2d_blocks import (+ UNetMidBlock2D,+ UNetMidBlock2DCrossAttn,+ get_down_block,+ get_up_block,+)++logger = logging.get_logger(__name__) # pylint: disable=invalid-name+++@dataclass+class UNet2DConditionOutput(BaseOutput):+ """+ The output of [`UNet2DConditionModel`].++ Args:+ sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):+ The hidden states output conditioned on `encoder_hidden_states` input. Output of last layer of model.+ """++ sample: torch.FloatTensor = None+ ref_features: Tuple[torch.FloatTensor] = None+++class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin):+ r"""+ A conditional 2D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample+ shaped output.++ This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented+ for all models (such as downloading or saving).++ Parameters:+ sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`):+ Height and width of input/output sample.+ in_channels (`int`, *optional*, defaults to 4): Number of channels in the input sample.+ out_channels (`int`, *optional*, defaults to 4): Number of channels in the output.+ center_input_sample (`bool`, *optional*, defaults to `False`): Whether to center the input sample.+ flip_sin_to_cos (`bool`, *optional*, defaults to `False`):+ Whether to flip the sin to cos in the time embedding.+ freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding.+ down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`):+ The tuple of downsample blocks to use.+ mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2DCrossAttn"`):+ Block type for middle of UNet, it can be one of `UNetMidBlock2DCrossAttn`, `UNetMidBlock2D`, or+ `UNetMidBlock2DSimpleCrossAttn`. If `None`, the mid block layer is skipped.+ up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`):+ The tuple of upsample blocks to use.+ only_cross_attention(`bool` or `Tuple[bool]`, *optional*, default to `False`):+ Whether to include self-attention in the basic transformer blocks, see+ [`~models.attention.BasicTransformerBlock`].+ block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`):+ The tuple of output channels for each block.+ layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block.+ downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution.+ mid_block_scale_factor (`float`, *optional*, defaults to 1.0): The scale factor to use for the mid block.+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.+ act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use.+ norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization.+ If `None`, normalization and activation layers is skipped in post-processing.+ norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization.+ cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280):+ The dimension of the cross attention features.+ transformer_layers_per_block (`int`, `Tuple[int]`, or `Tuple[Tuple]` , *optional*, defaults to 1):+ The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for+ [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`],+ [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`].+ reverse_transformer_layers_per_block : (`Tuple[Tuple]`, *optional*, defaults to None):+ The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`], in the upsampling+ blocks of the U-Net. Only relevant if `transformer_layers_per_block` is of type `Tuple[Tuple]` and for+ [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`],+ [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`].+ encoder_hid_dim (`int`, *optional*, defaults to None):+ If `encoder_hid_dim_type` is defined, `encoder_hidden_states` will be projected from `encoder_hid_dim`+ dimension to `cross_attention_dim`.+ encoder_hid_dim_type (`str`, *optional*, defaults to `None`):+ If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text+ embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`.+ attention_head_dim (`int`, *optional*, defaults to 8): The dimension of the attention heads.+ num_attention_heads (`int`, *optional*):+ The number of attention heads. If not defined, defaults to `attention_head_dim`+ resnet_time_scale_shift (`str`, *optional*, defaults to `"default"`): Time scale shift config+ for ResNet blocks (see [`~models.resnet.ResnetBlock2D`]). Choose from `default` or `scale_shift`.+ class_embed_type (`str`, *optional*, defaults to `None`):+ The type of class embedding to use which is ultimately summed with the time embeddings. Choose from `None`,+ `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`.+ addition_embed_type (`str`, *optional*, defaults to `None`):+ Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or+ "text". "text" will use the `TextTimeEmbedding` layer.+ addition_time_embed_dim: (`int`, *optional*, defaults to `None`):+ Dimension for the timestep embeddings.+ num_class_embeds (`int`, *optional*, defaults to `None`):+ Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing+ class conditioning with `class_embed_type` equal to `None`.+ time_embedding_type (`str`, *optional*, defaults to `positional`):+ The type of position embedding to use for timesteps. Choose from `positional` or `fourier`.+ time_embedding_dim (`int`, *optional*, defaults to `None`):+ An optional override for the dimension of the projected time embedding.+ time_embedding_act_fn (`str`, *optional*, defaults to `None`):+ Optional activation function to use only once on the time embeddings before they are passed to the rest of+ the UNet. Choose from `silu`, `mish`, `gelu`, and `swish`.+ timestep_post_act (`str`, *optional*, defaults to `None`):+ The second activation function to use in timestep embedding. Choose from `silu`, `mish` and `gelu`.+ time_cond_proj_dim (`int`, *optional*, defaults to `None`):+ The dimension of `cond_proj` layer in the timestep embedding.+ conv_in_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_in` layer. conv_out_kernel (`int`,+ *optional*, default to `3`): The kernel size of `conv_out` layer. projection_class_embeddings_input_dim (`int`,+ *optional*): The dimension of the `class_labels` input when+ `class_embed_type="projection"`. Required when `class_embed_type="projection"`.+ class_embeddings_concat (`bool`, *optional*, defaults to `False`): Whether to concatenate the time+ embeddings with the class embeddings.+ mid_block_only_cross_attention (`bool`, *optional*, defaults to `None`):+ Whether to use cross attention with the mid block when using the `UNetMidBlock2DSimpleCrossAttn`. If+ `only_cross_attention` is given as a single boolean and `mid_block_only_cross_attention` is `None`, the+ `only_cross_attention` value is used as the value for `mid_block_only_cross_attention`. Default to `False`+ otherwise.+ """++ _supports_gradient_checkpointing = True++ @register_to_config+ def __init__(+ self,+ sample_size: Optional[int] = None,+ in_channels: int = 4,+ out_channels: int = 4,+ center_input_sample: bool = False,+ flip_sin_to_cos: bool = True,+ freq_shift: int = 0,+ down_block_types: Tuple[str] = (+ "CrossAttnDownBlock2D",+ "CrossAttnDownBlock2D",+ "CrossAttnDownBlock2D",+ "DownBlock2D",+ ),+ mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn",+ up_block_types: Tuple[str] = (+ "UpBlock2D",+ "CrossAttnUpBlock2D",+ "CrossAttnUpBlock2D",+ "CrossAttnUpBlock2D",+ ),+ only_cross_attention: Union[bool, Tuple[bool]] = False,+ block_out_channels: Tuple[int] = (320, 640, 1280, 1280),+ layers_per_block: Union[int, Tuple[int]] = 2,+ downsample_padding: int = 1,+ mid_block_scale_factor: float = 1,+ dropout: float = 0.0,+ act_fn: str = "silu",+ norm_num_groups: Optional[int] = 32,+ norm_eps: float = 1e-5,+ cross_attention_dim: Union[int, Tuple[int]] = 1280,+ transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple]] = 1,+ reverse_transformer_layers_per_block: Optional[Tuple[Tuple[int]]] = None,+ encoder_hid_dim: Optional[int] = None,+ encoder_hid_dim_type: Optional[str] = None,+ attention_head_dim: Union[int, Tuple[int]] = 8,+ num_attention_heads: Optional[Union[int, Tuple[int]]] = None,+ dual_cross_attention: bool = False,+ use_linear_projection: bool = False,+ class_embed_type: Optional[str] = None,+ addition_embed_type: Optional[str] = None,+ addition_time_embed_dim: Optional[int] = None,+ num_class_embeds: Optional[int] = None,+ upcast_attention: bool = False,+ resnet_time_scale_shift: str = "default",+ resnet_skip_time_act: bool = False,+ resnet_out_scale_factor: int = 1.0,+ time_embedding_type: str = "positional",+ time_embedding_dim: Optional[int] = None,+ time_embedding_act_fn: Optional[str] = None,+ timestep_post_act: Optional[str] = None,+ time_cond_proj_dim: Optional[int] = None,+ conv_in_kernel: int = 3,+ conv_out_kernel: int = 3,+ projection_class_embeddings_input_dim: Optional[int] = None,+ attention_type: str = "default",+ class_embeddings_concat: bool = False,+ mid_block_only_cross_attention: Optional[bool] = None,+ cross_attention_norm: Optional[str] = None,+ addition_embed_type_num_heads=64,+ ):+ super().__init__()++ self.sample_size = sample_size++ if num_attention_heads is not None:+ raise ValueError(+ "At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19."+ )++ # If `num_attention_heads` is not defined (which is the case for most models)+ # it will default to `attention_head_dim`. This looks weird upon first reading it and it is.+ # The reason for this behavior is to correct for incorrectly named variables that were introduced+ # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131+ # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking+ # which is why we correct for the naming here.+ num_attention_heads = num_attention_heads or attention_head_dim++ # Check inputs+ if len(down_block_types) != len(up_block_types):+ raise ValueError(+ f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}."+ )++ if len(block_out_channels) != len(down_block_types):+ raise ValueError(+ f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}."+ )++ if not isinstance(only_cross_attention, bool) and len(+ only_cross_attention+ ) != len(down_block_types):+ raise ValueError(+ f"Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}."+ )++ if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(+ down_block_types+ ):+ raise ValueError(+ f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}."+ )++ if not isinstance(attention_head_dim, int) and len(attention_head_dim) != len(+ down_block_types+ ):+ raise ValueError(+ f"Must provide the same number of `attention_head_dim` as `down_block_types`. `attention_head_dim`: {attention_head_dim}. `down_block_types`: {down_block_types}."+ )++ if isinstance(cross_attention_dim, list) and len(cross_attention_dim) != len(+ down_block_types+ ):+ raise ValueError(+ f"Must provide the same number of `cross_attention_dim` as `down_block_types`. `cross_attention_dim`: {cross_attention_dim}. `down_block_types`: {down_block_types}."+ )++ if not isinstance(layers_per_block, int) and len(layers_per_block) != len(+ down_block_types+ ):+ raise ValueError(+ f"Must provide the same number of `layers_per_block` as `down_block_types`. `layers_per_block`: {layers_per_block}. `down_block_types`: {down_block_types}."+ )+ if (+ isinstance(transformer_layers_per_block, list)+ and reverse_transformer_layers_per_block is None+ ):+ for layer_number_per_block in transformer_layers_per_block:+ if isinstance(layer_number_per_block, list):+ raise ValueError(+ "Must provide 'reverse_transformer_layers_per_block` if using asymmetrical UNet."+ )++ # input+ conv_in_padding = (conv_in_kernel - 1) // 2+ self.conv_in = nn.Conv2d(+ in_channels,+ block_out_channels[0],+ kernel_size=conv_in_kernel,+ padding=conv_in_padding,+ )++ # time+ if time_embedding_type == "fourier":+ time_embed_dim = time_embedding_dim or block_out_channels[0] * 2+ if time_embed_dim % 2 != 0:+ raise ValueError(+ f"`time_embed_dim` should be divisible by 2, but is {time_embed_dim}."+ )+ self.time_proj = GaussianFourierProjection(+ time_embed_dim // 2,+ set_W_to_weight=False,+ log=False,+ flip_sin_to_cos=flip_sin_to_cos,+ )+ timestep_input_dim = time_embed_dim+ elif time_embedding_type == "positional":+ time_embed_dim = time_embedding_dim or block_out_channels[0] * 4++ self.time_proj = Timesteps(+ block_out_channels[0], flip_sin_to_cos, freq_shift+ )+ timestep_input_dim = block_out_channels[0]+ else:+ raise ValueError(+ f"{time_embedding_type} does not exist. Please make sure to use one of `fourier` or `positional`."+ )++ self.time_embedding = TimestepEmbedding(+ timestep_input_dim,+ time_embed_dim,+ act_fn=act_fn,+ post_act_fn=timestep_post_act,+ cond_proj_dim=time_cond_proj_dim,+ )++ if encoder_hid_dim_type is None and encoder_hid_dim is not None:+ encoder_hid_dim_type = "text_proj"+ self.register_to_config(encoder_hid_dim_type=encoder_hid_dim_type)+ logger.info(+ "encoder_hid_dim_type defaults to 'text_proj' as `encoder_hid_dim` is defined."+ )++ if encoder_hid_dim is None and encoder_hid_dim_type is not None:+ raise ValueError(+ f"`encoder_hid_dim` has to be defined when `encoder_hid_dim_type` is set to {encoder_hid_dim_type}."+ )++ if encoder_hid_dim_type == "text_proj":+ self.encoder_hid_proj = nn.Linear(encoder_hid_dim, cross_attention_dim)+ elif encoder_hid_dim_type == "text_image_proj":+ # image_embed_dim DOESN'T have to be `cross_attention_dim`. To not clutter the __init__ too much+ # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use+ # case when `addition_embed_type == "text_image_proj"` (Kadinsky 2.1)`+ self.encoder_hid_proj = TextImageProjection(+ text_embed_dim=encoder_hid_dim,+ image_embed_dim=cross_attention_dim,+ cross_attention_dim=cross_attention_dim,+ )+ elif encoder_hid_dim_type == "image_proj":+ # Kandinsky 2.2+ self.encoder_hid_proj = ImageProjection(+ image_embed_dim=encoder_hid_dim,+ cross_attention_dim=cross_attention_dim,+ )+ elif encoder_hid_dim_type is not None:+ raise ValueError(+ f"encoder_hid_dim_type: {encoder_hid_dim_type} must be None, 'text_proj' or 'text_image_proj'."+ )+ else:+ self.encoder_hid_proj = None++ # class embedding+ if class_embed_type is None and num_class_embeds is not None:+ self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim)+ elif class_embed_type == "timestep":+ self.class_embedding = TimestepEmbedding(+ timestep_input_dim, time_embed_dim, act_fn=act_fn+ )+ elif class_embed_type == "identity":+ self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim)+ elif class_embed_type == "projection":+ if projection_class_embeddings_input_dim is None:+ raise ValueError(+ "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set"+ )+ # The projection `class_embed_type` is the same as the timestep `class_embed_type` except+ # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings+ # 2. it projects from an arbitrary input dimension.+ #+ # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations.+ # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings.+ # As a result, `TimestepEmbedding` can be passed arbitrary vectors.+ self.class_embedding = TimestepEmbedding(+ projection_class_embeddings_input_dim, time_embed_dim+ )+ elif class_embed_type == "simple_projection":+ if projection_class_embeddings_input_dim is None:+ raise ValueError(+ "`class_embed_type`: 'simple_projection' requires `projection_class_embeddings_input_dim` be set"+ )+ self.class_embedding = nn.Linear(+ projection_class_embeddings_input_dim, time_embed_dim+ )+ else:+ self.class_embedding = None++ if addition_embed_type == "text":+ if encoder_hid_dim is not None:+ text_time_embedding_from_dim = encoder_hid_dim+ else:+ text_time_embedding_from_dim = cross_attention_dim++ self.add_embedding = TextTimeEmbedding(+ text_time_embedding_from_dim,+ time_embed_dim,+ num_heads=addition_embed_type_num_heads,+ )+ elif addition_embed_type == "text_image":+ # text_embed_dim and image_embed_dim DON'T have to be `cross_attention_dim`. To not clutter the __init__ too much+ # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use+ # case when `addition_embed_type == "text_image"` (Kadinsky 2.1)`+ self.add_embedding = TextImageTimeEmbedding(+ text_embed_dim=cross_attention_dim,+ image_embed_dim=cross_attention_dim,+ time_embed_dim=time_embed_dim,+ )+ elif addition_embed_type == "text_time":+ self.add_time_proj = Timesteps(+ addition_time_embed_dim, flip_sin_to_cos, freq_shift+ )+ self.add_embedding = TimestepEmbedding(+ projection_class_embeddings_input_dim, time_embed_dim+ )+ elif addition_embed_type == "image":+ # Kandinsky 2.2+ self.add_embedding = ImageTimeEmbedding(+ image_embed_dim=encoder_hid_dim, time_embed_dim=time_embed_dim+ )+ elif addition_embed_type == "image_hint":+ # Kandinsky 2.2 ControlNet+ self.add_embedding = ImageHintTimeEmbedding(+ image_embed_dim=encoder_hid_dim, time_embed_dim=time_embed_dim+ )+ elif addition_embed_type is not None:+ raise ValueError(+ f"addition_embed_type: {addition_embed_type} must be None, 'text' or 'text_image'."+ )++ if time_embedding_act_fn is None:+ self.time_embed_act = None+ else:+ self.time_embed_act = get_activation(time_embedding_act_fn)++ self.down_blocks = nn.ModuleList([])+ self.up_blocks = nn.ModuleList([])++ if isinstance(only_cross_attention, bool):+ if mid_block_only_cross_attention is None:+ mid_block_only_cross_attention = only_cross_attention++ only_cross_attention = [only_cross_attention] * len(down_block_types)++ if mid_block_only_cross_attention is None:+ mid_block_only_cross_attention = False++ if isinstance(num_attention_heads, int):+ num_attention_heads = (num_attention_heads,) * len(down_block_types)++ if isinstance(attention_head_dim, int):+ attention_head_dim = (attention_head_dim,) * len(down_block_types)++ if isinstance(cross_attention_dim, int):+ cross_attention_dim = (cross_attention_dim,) * len(down_block_types)++ if isinstance(layers_per_block, int):+ layers_per_block = [layers_per_block] * len(down_block_types)++ if isinstance(transformer_layers_per_block, int):+ transformer_layers_per_block = [transformer_layers_per_block] * len(+ down_block_types+ )++ if class_embeddings_concat:+ # The time embeddings are concatenated with the class embeddings. The dimension of the+ # time embeddings passed to the down, middle, and up blocks is twice the dimension of the+ # regular time embeddings+ blocks_time_embed_dim = time_embed_dim * 2+ else:+ blocks_time_embed_dim = time_embed_dim++ # down+ output_channel = block_out_channels[0]+ for i, down_block_type in enumerate(down_block_types):+ input_channel = output_channel+ output_channel = block_out_channels[i]+ is_final_block = i == len(block_out_channels) - 1++ down_block = get_down_block(+ down_block_type,+ num_layers=layers_per_block[i],+ transformer_layers_per_block=transformer_layers_per_block[i],+ in_channels=input_channel,+ out_channels=output_channel,+ temb_channels=blocks_time_embed_dim,+ add_downsample=not is_final_block,+ resnet_eps=norm_eps,+ resnet_act_fn=act_fn,+ resnet_groups=norm_num_groups,+ cross_attention_dim=cross_attention_dim[i],+ num_attention_heads=num_attention_heads[i],+ downsample_padding=downsample_padding,+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention[i],+ upcast_attention=upcast_attention,+ resnet_time_scale_shift=resnet_time_scale_shift,+ attention_type=attention_type,+ resnet_skip_time_act=resnet_skip_time_act,+ resnet_out_scale_factor=resnet_out_scale_factor,+ cross_attention_norm=cross_attention_norm,+ attention_head_dim=attention_head_dim[i]+ if attention_head_dim[i] is not None+ else output_channel,+ dropout=dropout,+ )+ self.down_blocks.append(down_block)++ # mid+ if mid_block_type == "UNetMidBlock2DCrossAttn":+ self.mid_block = UNetMidBlock2DCrossAttn(+ transformer_layers_per_block=transformer_layers_per_block[-1],+ in_channels=block_out_channels[-1],+ temb_channels=blocks_time_embed_dim,+ dropout=dropout,+ resnet_eps=norm_eps,+ resnet_act_fn=act_fn,+ output_scale_factor=mid_block_scale_factor,+ resnet_time_scale_shift=resnet_time_scale_shift,+ cross_attention_dim=cross_attention_dim[-1],+ num_attention_heads=num_attention_heads[-1],+ resnet_groups=norm_num_groups,+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ upcast_attention=upcast_attention,+ attention_type=attention_type,+ )+ elif mid_block_type == "UNetMidBlock2DSimpleCrossAttn":+ raise NotImplementedError(f"Unsupport mid_block_type: {mid_block_type}")+ elif mid_block_type == "UNetMidBlock2D":+ self.mid_block = UNetMidBlock2D(+ in_channels=block_out_channels[-1],+ temb_channels=blocks_time_embed_dim,+ dropout=dropout,+ num_layers=0,+ resnet_eps=norm_eps,+ resnet_act_fn=act_fn,+ output_scale_factor=mid_block_scale_factor,+ resnet_groups=norm_num_groups,+ resnet_time_scale_shift=resnet_time_scale_shift,+ add_attention=False,+ )+ elif mid_block_type is None:+ self.mid_block = None+ else:+ raise ValueError(f"unknown mid_block_type : {mid_block_type}")++ # count how many layers upsample the images+ self.num_upsamplers = 0++ # up+ reversed_block_out_channels = list(reversed(block_out_channels))+ reversed_num_attention_heads = list(reversed(num_attention_heads))+ reversed_layers_per_block = list(reversed(layers_per_block))+ reversed_cross_attention_dim = list(reversed(cross_attention_dim))+ reversed_transformer_layers_per_block = (+ list(reversed(transformer_layers_per_block))+ if reverse_transformer_layers_per_block is None+ else reverse_transformer_layers_per_block+ )+ only_cross_attention = list(reversed(only_cross_attention))++ output_channel = reversed_block_out_channels[0]+ for i, up_block_type in enumerate(up_block_types):+ is_final_block = i == len(block_out_channels) - 1++ prev_output_channel = output_channel+ output_channel = reversed_block_out_channels[i]+ input_channel = reversed_block_out_channels[+ min(i + 1, len(block_out_channels) - 1)+ ]++ # add upsample block for all BUT final layer+ if not is_final_block:+ add_upsample = True+ self.num_upsamplers += 1+ else:+ add_upsample = False++ up_block = get_up_block(+ up_block_type,+ num_layers=reversed_layers_per_block[i] + 1,+ transformer_layers_per_block=reversed_transformer_layers_per_block[i],+ in_channels=input_channel,+ out_channels=output_channel,+ prev_output_channel=prev_output_channel,+ temb_channels=blocks_time_embed_dim,+ add_upsample=add_upsample,+ resnet_eps=norm_eps,+ resnet_act_fn=act_fn,+ resolution_idx=i,+ resnet_groups=norm_num_groups,+ cross_attention_dim=reversed_cross_attention_dim[i],+ num_attention_heads=reversed_num_attention_heads[i],+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention[i],+ upcast_attention=upcast_attention,+ resnet_time_scale_shift=resnet_time_scale_shift,+ attention_type=attention_type,+ resnet_skip_time_act=resnet_skip_time_act,+ resnet_out_scale_factor=resnet_out_scale_factor,+ cross_attention_norm=cross_attention_norm,+ attention_head_dim=attention_head_dim[i]+ if attention_head_dim[i] is not None+ else output_channel,+ dropout=dropout,+ )+ self.up_blocks.append(up_block)+ prev_output_channel = output_channel++ # out+ if norm_num_groups is not None:+ self.conv_norm_out = nn.GroupNorm(+ num_channels=block_out_channels[0],+ num_groups=norm_num_groups,+ eps=norm_eps,+ )++ self.conv_act = get_activation(act_fn)++ else:+ self.conv_norm_out = None+ self.conv_act = None+ self.conv_norm_out = None++ conv_out_padding = (conv_out_kernel - 1) // 2+ # self.conv_out = nn.Conv2d(+ # block_out_channels[0],+ # out_channels,+ # kernel_size=conv_out_kernel,+ # padding=conv_out_padding,+ # )++ if attention_type in ["gated", "gated-text-image"]:+ positive_len = 768+ if isinstance(cross_attention_dim, int):+ positive_len = cross_attention_dim+ elif isinstance(cross_attention_dim, tuple) or isinstance(+ cross_attention_dim, list+ ):+ positive_len = cross_attention_dim[0]++ feature_type = "text-only" if attention_type == "gated" else "text-image"+ self.position_net = PositionNet(+ positive_len=positive_len,+ out_dim=cross_attention_dim,+ feature_type=feature_type,+ )++ @property+ def attn_processors(self) -> Dict[str, AttentionProcessor]:+ r"""+ Returns:+ `dict` of attention processors: A dictionary containing all attention processors used in the model with+ indexed by its weight name.+ """+ # set recursively+ processors = {}++ def fn_recursive_add_processors(+ name: str,+ module: torch.nn.Module,+ processors: Dict[str, AttentionProcessor],+ ):+ if hasattr(module, "get_processor"):+ processors[f"{name}.processor"] = module.get_processor(+ return_deprecated_lora=True+ )++ for sub_name, child in module.named_children():+ fn_recursive_add_processors(f"{name}.{sub_name}", child, processors)++ return processors++ for name, module in self.named_children():+ fn_recursive_add_processors(name, module, processors)++ return processors++ def set_attn_processor(+ self,+ processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]],+ _remove_lora=False,+ ):+ r"""+ Sets the attention processor to use to compute attention.++ Parameters:+ processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):+ The instantiated processor class or a dictionary of processor classes that will be set as the processor+ for **all** `Attention` layers.++ If `processor` is a dict, the key needs to define the path to the corresponding cross attention+ processor. This is strongly recommended when setting trainable attention processors.++ """+ count = len(self.attn_processors.keys())++ if isinstance(processor, dict) and len(processor) != count:+ raise ValueError(+ f"A dict of processors was passed, but the number of processors {len(processor)} does not match the"+ f" number of attention layers: {count}. Please make sure to pass {count} processor classes."+ )++ def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor):+ if hasattr(module, "set_processor"):+ if not isinstance(processor, dict):+ module.set_processor(processor, _remove_lora=_remove_lora)+ else:+ module.set_processor(+ processor.pop(f"{name}.processor"), _remove_lora=_remove_lora+ )++ for sub_name, child in module.named_children():+ fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor)++ for name, module in self.named_children():+ fn_recursive_attn_processor(name, module, processor)++ def set_default_attn_processor(self):+ """+ Disables custom attention processors and sets the default attention implementation.+ """+ if all(+ proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS+ for proc in self.attn_processors.values()+ ):+ processor = AttnAddedKVProcessor()+ elif all(+ proc.__class__ in CROSS_ATTENTION_PROCESSORS+ for proc in self.attn_processors.values()+ ):+ processor = AttnProcessor()+ else:+ raise ValueError(+ f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}"+ )++ self.set_attn_processor(processor, _remove_lora=True)++ def set_attention_slice(self, slice_size):+ r"""+ Enable sliced attention computation.++ When this option is enabled, the attention module splits the input tensor in slices to compute attention in+ several steps. This is useful for saving some memory in exchange for a small decrease in speed.++ Args:+ slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`):+ When `"auto"`, input to the attention heads is halved, so attention is computed in two steps. If+ `"max"`, maximum amount of memory is saved by running only one slice at a time. If a number is+ provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim`+ must be a multiple of `slice_size`.+ """+ sliceable_head_dims = []++ def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module):+ if hasattr(module, "set_attention_slice"):+ sliceable_head_dims.append(module.sliceable_head_dim)++ for child in module.children():+ fn_recursive_retrieve_sliceable_dims(child)++ # retrieve number of attention layers+ for module in self.children():+ fn_recursive_retrieve_sliceable_dims(module)++ num_sliceable_layers = len(sliceable_head_dims)++ if slice_size == "auto":+ # half the attention head size is usually a good trade-off between+ # speed and memory+ slice_size = [dim // 2 for dim in sliceable_head_dims]+ elif slice_size == "max":+ # make smallest slice possible+ slice_size = num_sliceable_layers * [1]++ slice_size = (+ num_sliceable_layers * [slice_size]+ if not isinstance(slice_size, list)+ else slice_size+ )++ if len(slice_size) != len(sliceable_head_dims):+ raise ValueError(+ f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different"+ f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}."+ )++ for i in range(len(slice_size)):+ size = slice_size[i]+ dim = sliceable_head_dims[i]+ if size is not None and size > dim:+ raise ValueError(f"size {size} has to be smaller or equal to {dim}.")++ # Recursively walk through all the children.+ # Any children which exposes the set_attention_slice method+ # gets the message+ def fn_recursive_set_attention_slice(+ module: torch.nn.Module, slice_size: List[int]+ ):+ if hasattr(module, "set_attention_slice"):+ module.set_attention_slice(slice_size.pop())++ for child in module.children():+ fn_recursive_set_attention_slice(child, slice_size)++ reversed_slice_size = list(reversed(slice_size))+ for module in self.children():+ fn_recursive_set_attention_slice(module, reversed_slice_size)++ def _set_gradient_checkpointing(self, module, value=False):+ if hasattr(module, "gradient_checkpointing"):+ module.gradient_checkpointing = value++ def enable_freeu(self, s1, s2, b1, b2):+ r"""Enables the FreeU mechanism from https://arxiv.org/abs/2309.11497.++ The suffixes after the scaling factors represent the stage blocks where they are being applied.++ Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of values that+ are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL.++ Args:+ s1 (`float`):+ Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to+ mitigate the "oversmoothing effect" in the enhanced denoising process.+ s2 (`float`):+ Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to+ mitigate the "oversmoothing effect" in the enhanced denoising process.+ b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features.+ b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features.+ """+ for i, upsample_block in enumerate(self.up_blocks):+ setattr(upsample_block, "s1", s1)+ setattr(upsample_block, "s2", s2)+ setattr(upsample_block, "b1", b1)+ setattr(upsample_block, "b2", b2)++ def disable_freeu(self):+ """Disables the FreeU mechanism."""+ freeu_keys = {"s1", "s2", "b1", "b2"}+ for i, upsample_block in enumerate(self.up_blocks):+ for k in freeu_keys:+ if (+ hasattr(upsample_block, k)+ or getattr(upsample_block, k, None) is not None+ ):+ setattr(upsample_block, k, None)++ def forward(+ self,+ sample: torch.FloatTensor,+ timestep: Union[torch.Tensor, float, int],+ encoder_hidden_states: torch.Tensor,+ class_labels: Optional[torch.Tensor] = None,+ timestep_cond: Optional[torch.Tensor] = None,+ attention_mask: Optional[torch.Tensor] = None,+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,+ added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,+ down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None,+ mid_block_additional_residual: Optional[torch.Tensor] = None,+ down_intrablock_additional_residuals: Optional[Tuple[torch.Tensor]] = None,+ encoder_attention_mask: Optional[torch.Tensor] = None,+ return_dict: bool = True,+ only_return_middle_fea: bool = False,+ ) -> Union[UNet2DConditionOutput, Tuple]:+ r"""+ The [`UNet2DConditionModel`] forward method.++ Args:+ sample (`torch.FloatTensor`):+ The noisy input tensor with the following shape `(batch, channel, height, width)`.+ timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input.+ encoder_hidden_states (`torch.FloatTensor`):+ The encoder hidden states with shape `(batch, sequence_length, feature_dim)`.+ class_labels (`torch.Tensor`, *optional*, defaults to `None`):+ Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings.+ timestep_cond: (`torch.Tensor`, *optional*, defaults to `None`):+ Conditional embeddings for timestep. If provided, the embeddings will be summed with the samples passed+ through the `self.time_embedding` layer to obtain the timestep embeddings.+ attention_mask (`torch.Tensor`, *optional*, defaults to `None`):+ An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask+ is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large+ negative values to the attention scores corresponding to "discard" tokens.+ cross_attention_kwargs (`dict`, *optional*):+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under+ `self.processor` in+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).+ added_cond_kwargs: (`dict`, *optional*):+ A kwargs dictionary containing additional embeddings that if specified are added to the embeddings that+ are passed along to the UNet blocks.+ down_block_additional_residuals: (`tuple` of `torch.Tensor`, *optional*):+ A tuple of tensors that if specified are added to the residuals of down unet blocks.+ mid_block_additional_residual: (`torch.Tensor`, *optional*):+ A tensor that if specified is added to the residual of the middle unet block.+ encoder_attention_mask (`torch.Tensor`):+ A cross-attention mask of shape `(batch, sequence_length)` is applied to `encoder_hidden_states`. If+ `True` the mask is kept, otherwise if `False` it is discarded. Mask will be converted into a bias,+ which adds large negative values to the attention scores corresponding to "discard" tokens.+ return_dict (`bool`, *optional*, defaults to `True`):+ Whether or not to return a [`~models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain+ tuple.+ cross_attention_kwargs (`dict`, *optional*):+ A kwargs dictionary that if specified is passed along to the [`AttnProcessor`].+ added_cond_kwargs: (`dict`, *optional*):+ A kwargs dictionary containin additional embeddings that if specified are added to the embeddings that+ are passed along to the UNet blocks.+ down_block_additional_residuals (`tuple` of `torch.Tensor`, *optional*):+ additional residuals to be added to UNet long skip connections from down blocks to up blocks for+ example from ControlNet side model(s)+ mid_block_additional_residual (`torch.Tensor`, *optional*):+ additional residual to be added to UNet mid block output, for example from ControlNet side model+ down_intrablock_additional_residuals (`tuple` of `torch.Tensor`, *optional*):+ additional residuals to be added within UNet down blocks, for example from T2I-Adapter side model(s)++ Returns:+ [`~models.unet_2d_condition.UNet2DConditionOutput`] or `tuple`:+ If `return_dict` is True, an [`~models.unet_2d_condition.UNet2DConditionOutput`] is returned, otherwise+ a `tuple` is returned where the first element is the sample tensor.+ """+ # By default samples have to be AT least a multiple of the overall upsampling factor.+ # The overall upsampling factor is equal to 2 ** (# num of upsampling layers).+ # However, the upsampling interpolation output size can be forced to fit any upsampling size+ # on the fly if necessary.+ default_overall_up_factor = 2**self.num_upsamplers++ # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`+ forward_upsample_size = False+ upsample_size = None++ for dim in sample.shape[-2:]:+ if dim % default_overall_up_factor != 0:+ # Forward upsample size to force interpolation output size.+ forward_upsample_size = True+ break++ # ensure attention_mask is a bias, and give it a singleton query_tokens dimension+ # expects mask of shape:+ # [batch, key_tokens]+ # adds singleton query_tokens dimension:+ # [batch, 1, key_tokens]+ # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:+ # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn)+ # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)+ if attention_mask is not None:+ # assume that mask is expressed as:+ # (1 = keep, 0 = discard)+ # convert mask into a bias that can be added to attention scores:+ # (keep = +0, discard = -10000.0)+ attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0+ attention_mask = attention_mask.unsqueeze(1)++ # convert encoder_attention_mask to a bias the same way we do for attention_mask+ if encoder_attention_mask is not None:+ encoder_attention_mask = (+ 1 - encoder_attention_mask.to(sample.dtype)+ ) * -10000.0+ encoder_attention_mask = encoder_attention_mask.unsqueeze(1)++ # 0. center input if necessary+ if self.config.center_input_sample:+ sample = 2 * sample - 1.0++ # 1. time+ timesteps = timestep+ if not torch.is_tensor(timesteps):+ # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can+ # This would be a good case for the `match` statement (Python 3.10+)+ is_mps = sample.device.type == "mps"+ if isinstance(timestep, float):+ dtype = torch.float32 if is_mps else torch.float64+ else:+ dtype = torch.int32 if is_mps else torch.int64+ timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)+ elif len(timesteps.shape) == 0:+ timesteps = timesteps[None].to(sample.device)++ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML+ timesteps = timesteps.expand(sample.shape[0])++ t_emb = self.time_proj(timesteps)++ # `Timesteps` does not contain any weights and will always return f32 tensors+ # but time_embedding might actually be running in fp16. so we need to cast here.+ # there might be better ways to encapsulate this.+ t_emb = t_emb.to(dtype=sample.dtype)++ emb = self.time_embedding(t_emb, timestep_cond)+ aug_emb = None++ if self.class_embedding is not None:+ if class_labels is None:+ raise ValueError(+ "class_labels should be provided when num_class_embeds > 0"+ )++ if self.config.class_embed_type == "timestep":+ class_labels = self.time_proj(class_labels)++ # `Timesteps` does not contain any weights and will always return f32 tensors+ # there might be better ways to encapsulate this.+ class_labels = class_labels.to(dtype=sample.dtype)++ class_emb = self.class_embedding(class_labels).to(dtype=sample.dtype)++ if self.config.class_embeddings_concat:+ emb = torch.cat([emb, class_emb], dim=-1)+ else:+ emb = emb + class_emb++ if self.config.addition_embed_type == "text":+ aug_emb = self.add_embedding(encoder_hidden_states)+ elif self.config.addition_embed_type == "text_image":+ # Kandinsky 2.1 - style+ if "image_embeds" not in added_cond_kwargs:+ raise ValueError(+ f"{self.__class__} has the config param `addition_embed_type` set to 'text_image' which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`"+ )++ image_embs = added_cond_kwargs.get("image_embeds")+ text_embs = added_cond_kwargs.get("text_embeds", encoder_hidden_states)+ aug_emb = self.add_embedding(text_embs, image_embs)+ elif self.config.addition_embed_type == "text_time":+ # SDXL - style+ if "text_embeds" not in added_cond_kwargs:+ raise ValueError(+ f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`"+ )+ text_embeds = added_cond_kwargs.get("text_embeds")+ if "time_ids" not in added_cond_kwargs:+ raise ValueError(+ f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`"+ )+ time_ids = added_cond_kwargs.get("time_ids")+ time_embeds = self.add_time_proj(time_ids.flatten())+ time_embeds = time_embeds.reshape((text_embeds.shape[0], -1))+ add_embeds = torch.concat([text_embeds, time_embeds], dim=-1)+ add_embeds = add_embeds.to(emb.dtype)+ aug_emb = self.add_embedding(add_embeds)+ elif self.config.addition_embed_type == "image":+ # Kandinsky 2.2 - style+ if "image_embeds" not in added_cond_kwargs:+ raise ValueError(+ f"{self.__class__} has the config param `addition_embed_type` set to 'image' which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`"+ )+ image_embs = added_cond_kwargs.get("image_embeds")+ aug_emb = self.add_embedding(image_embs)+ elif self.config.addition_embed_type == "image_hint":+ # Kandinsky 2.2 - style+ if (+ "image_embeds" not in added_cond_kwargs+ or "hint" not in added_cond_kwargs+ ):+ raise ValueError(+ f"{self.__class__} has the config param `addition_embed_type` set to 'image_hint' which requires the keyword arguments `image_embeds` and `hint` to be passed in `added_cond_kwargs`"+ )+ image_embs = added_cond_kwargs.get("image_embeds")+ hint = added_cond_kwargs.get("hint")+ aug_emb, hint = self.add_embedding(image_embs, hint)+ sample = torch.cat([sample, hint], dim=1)++ emb = emb + aug_emb if aug_emb is not None else emb++ if self.time_embed_act is not None:+ emb = self.time_embed_act(emb)++ if (+ self.encoder_hid_proj is not None+ and self.config.encoder_hid_dim_type == "text_proj"+ ):+ encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states)+ elif (+ self.encoder_hid_proj is not None+ and self.config.encoder_hid_dim_type == "text_image_proj"+ ):+ # Kadinsky 2.1 - style+ if "image_embeds" not in added_cond_kwargs:+ raise ValueError(+ f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'text_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`"+ )++ image_embeds = added_cond_kwargs.get("image_embeds")+ encoder_hidden_states = self.encoder_hid_proj(+ encoder_hidden_states, image_embeds+ )+ elif (+ self.encoder_hid_proj is not None+ and self.config.encoder_hid_dim_type == "image_proj"+ ):+ # Kandinsky 2.2 - style+ if "image_embeds" not in added_cond_kwargs:+ raise ValueError(+ f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`"+ )+ image_embeds = added_cond_kwargs.get("image_embeds")+ encoder_hidden_states = self.encoder_hid_proj(image_embeds)+ elif (+ self.encoder_hid_proj is not None+ and self.config.encoder_hid_dim_type == "ip_image_proj"+ ):+ if "image_embeds" not in added_cond_kwargs:+ raise ValueError(+ f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'ip_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`"+ )+ image_embeds = added_cond_kwargs.get("image_embeds")+ image_embeds = self.encoder_hid_proj(image_embeds).to(+ encoder_hidden_states.dtype+ )+ encoder_hidden_states = torch.cat(+ [encoder_hidden_states, image_embeds], dim=1+ )++ # 2. pre-process+ sample = self.conv_in(sample)++ # 2.5 GLIGEN position net+ if (+ cross_attention_kwargs is not None+ and cross_attention_kwargs.get("gligen", None) is not None+ ):+ cross_attention_kwargs = cross_attention_kwargs.copy()+ gligen_args = cross_attention_kwargs.pop("gligen")+ cross_attention_kwargs["gligen"] = {+ "objs": self.position_net(**gligen_args)+ }++ # 3. down+ lora_scale = (+ cross_attention_kwargs.get("scale", 1.0)+ if cross_attention_kwargs is not None+ else 1.0+ )+ if USE_PEFT_BACKEND:+ # weight the lora layers by setting `lora_scale` for each PEFT layer+ scale_lora_layers(self, lora_scale)++ is_controlnet = (+ mid_block_additional_residual is not None+ and down_block_additional_residuals is not None+ )+ # using new arg down_intrablock_additional_residuals for T2I-Adapters, to distinguish from controlnets+ is_adapter = down_intrablock_additional_residuals is not None+ # maintain backward compatibility for legacy usage, where+ # T2I-Adapter and ControlNet both use down_block_additional_residuals arg+ # but can only use one or the other+ if (+ not is_adapter+ and mid_block_additional_residual is None+ and down_block_additional_residuals is not None+ ):+ deprecate(+ "T2I should not use down_block_additional_residuals",+ "1.3.0",+ "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated \+ and will be removed in diffusers 1.3.0. `down_block_additional_residuals` should only be used \+ for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ",+ standard_warn=False,+ )+ down_intrablock_additional_residuals = down_block_additional_residuals+ is_adapter = True++ down_block_res_samples = (sample,)+ tot_referece_features = ()+ for downsample_block in self.down_blocks:+ if (+ hasattr(downsample_block, "has_cross_attention")+ and downsample_block.has_cross_attention+ ):+ # For t2i-adapter CrossAttnDownBlock2D+ additional_residuals = {}+ if is_adapter and len(down_intrablock_additional_residuals) > 0:+ additional_residuals[+ "additional_residuals"+ ] = down_intrablock_additional_residuals.pop(0)++ sample, res_samples = downsample_block(+ hidden_states=sample,+ temb=emb,+ encoder_hidden_states=encoder_hidden_states,+ attention_mask=attention_mask,+ cross_attention_kwargs=cross_attention_kwargs,+ encoder_attention_mask=encoder_attention_mask,+ **additional_residuals,+ )+ else:+ sample, res_samples = downsample_block(+ hidden_states=sample, temb=emb, scale=lora_scale+ )+ if is_adapter and len(down_intrablock_additional_residuals) > 0:+ sample += down_intrablock_additional_residuals.pop(0)++ down_block_res_samples += res_samples++ if only_return_middle_fea:+ return sample++ if is_controlnet:+ new_down_block_res_samples = ()++ for down_block_res_sample, down_block_additional_residual in zip(+ down_block_res_samples, down_block_additional_residuals+ ):+ down_block_res_sample = (+ down_block_res_sample + down_block_additional_residual+ )+ new_down_block_res_samples = new_down_block_res_samples + (+ down_block_res_sample,+ )++ down_block_res_samples = new_down_block_res_samples++ # 4. mid+ if self.mid_block is not None:+ if (+ hasattr(self.mid_block, "has_cross_attention")+ and self.mid_block.has_cross_attention+ ):+ sample = self.mid_block(+ sample,+ emb,+ encoder_hidden_states=encoder_hidden_states,+ attention_mask=attention_mask,+ cross_attention_kwargs=cross_attention_kwargs,+ encoder_attention_mask=encoder_attention_mask,+ )+ else:+ sample = self.mid_block(sample, emb)++ # To support T2I-Adapter-XL+ if (+ is_adapter+ and len(down_intrablock_additional_residuals) > 0+ and sample.shape == down_intrablock_additional_residuals[0].shape+ ):+ sample += down_intrablock_additional_residuals.pop(0)++ if is_controlnet:+ sample = sample + mid_block_additional_residual++ # 5. up+ for i, upsample_block in enumerate(self.up_blocks):+ is_final_block = i == len(self.up_blocks) - 1++ res_samples = down_block_res_samples[-len(upsample_block.resnets) :]+ down_block_res_samples = down_block_res_samples[+ : -len(upsample_block.resnets)+ ]++ # if we have not reached the final block and need to forward the+ # upsample size, we do it here+ if not is_final_block and forward_upsample_size:+ upsample_size = down_block_res_samples[-1].shape[2:]++ if (+ hasattr(upsample_block, "has_cross_attention")+ and upsample_block.has_cross_attention+ ):+ sample = upsample_block(+ hidden_states=sample,+ temb=emb,+ res_hidden_states_tuple=res_samples,+ encoder_hidden_states=encoder_hidden_states,+ cross_attention_kwargs=cross_attention_kwargs,+ upsample_size=upsample_size,+ attention_mask=attention_mask,+ encoder_attention_mask=encoder_attention_mask,+ )+ else:+ sample = upsample_block(+ hidden_states=sample,+ temb=emb,+ res_hidden_states_tuple=res_samples,+ upsample_size=upsample_size,+ scale=lora_scale,+ )++ # 6. post-process+ # if self.conv_norm_out:+ # sample = self.conv_norm_out(sample)+ # sample = self.conv_act(sample)+ # sample = self.conv_out(sample)++ if USE_PEFT_BACKEND:+ # remove `lora_scale` from each PEFT layer+ unscale_lora_layers(self, lora_scale)++ if not return_dict:+ return (sample,)++ return UNet2DConditionOutput(sample=sample)diff --git a/src/models/unet_2d_decoder.py b/src/models/unet_2d_decoder.pynew file mode 100644index 0000000..e0a1933--- /dev/null+++ b/src/models/unet_2d_decoder.py@@ -0,0 +1,1011 @@+from dataclasses import dataclass+from typing import Any, Dict, List, Optional, Tuple, Union++import torch+from torch import nn+from torch.nn import functional as F++from diffusers.configuration_utils import ConfigMixin, register_to_config+from diffusers.loaders import FromOriginalControlNetMixin+from diffusers.utils import BaseOutput, logging+from diffusers.models.embeddings import (+ TextImageProjection,+ TextImageTimeEmbedding,+ TextTimeEmbedding,+ TimestepEmbedding,+ Timesteps,+)+from diffusers.models.modeling_utils import ModelMixin+from diffusers.models.unet_2d_blocks import (+ CrossAttnDownBlock2D,+ DownBlock2D,+ UNetMidBlock2DCrossAttn,+ get_down_block,+)+from diffusers.models.unet_2d_condition import UNet2DConditionModel+from diffusers.models.controlnet import (+ ControlNetConditioningEmbedding,+ ControlNetOutput,+ ControlNetModel,+)++logger = logging.get_logger(__name__) # pylint: disable=invalid-name+++class UNetDec_ControlNetModel(ModelMixin, ConfigMixin, FromOriginalControlNetMixin):+ """+ A ControlNet model.++ Args:+ in_channels (`int`, defaults to 4):+ The number of channels in the input sample.+ flip_sin_to_cos (`bool`, defaults to `True`):+ Whether to flip the sin to cos in the time embedding.+ freq_shift (`int`, defaults to 0):+ The frequency shift to apply to the time embedding.+ down_block_types (`tuple[str]`, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`):+ The tuple of downsample blocks to use.+ only_cross_attention (`Union[bool, Tuple[bool]]`, defaults to `False`):+ block_out_channels (`tuple[int]`, defaults to `(320, 640, 1280, 1280)`):+ The tuple of output channels for each block.+ layers_per_block (`int`, defaults to 2):+ The number of layers per block.+ downsample_padding (`int`, defaults to 1):+ The padding to use for the downsampling convolution.+ mid_block_scale_factor (`float`, defaults to 1):+ The scale factor to use for the mid block.+ act_fn (`str`, defaults to "silu"):+ The activation function to use.+ norm_num_groups (`int`, *optional*, defaults to 32):+ The number of groups to use for the normalization. If None, normalization and activation layers is skipped+ in post-processing.+ norm_eps (`float`, defaults to 1e-5):+ The epsilon to use for the normalization.+ cross_attention_dim (`int`, defaults to 1280):+ The dimension of the cross attention features.+ transformer_layers_per_block (`int` or `Tuple[int]`, *optional*, defaults to 1):+ The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for+ [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`],+ [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`].+ encoder_hid_dim (`int`, *optional*, defaults to None):+ If `encoder_hid_dim_type` is defined, `encoder_hidden_states` will be projected from `encoder_hid_dim`+ dimension to `cross_attention_dim`.+ encoder_hid_dim_type (`str`, *optional*, defaults to `None`):+ If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text+ embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`.+ attention_head_dim (`Union[int, Tuple[int]]`, defaults to 8):+ The dimension of the attention heads.+ use_linear_projection (`bool`, defaults to `False`):+ class_embed_type (`str`, *optional*, defaults to `None`):+ The type of class embedding to use which is ultimately summed with the time embeddings. Choose from None,+ `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`.+ addition_embed_type (`str`, *optional*, defaults to `None`):+ Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or+ "text". "text" will use the `TextTimeEmbedding` layer.+ num_class_embeds (`int`, *optional*, defaults to 0):+ Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing+ class conditioning with `class_embed_type` equal to `None`.+ upcast_attention (`bool`, defaults to `False`):+ resnet_time_scale_shift (`str`, defaults to `"default"`):+ Time scale shift config for ResNet blocks (see `ResnetBlock2D`). Choose from `default` or `scale_shift`.+ projection_class_embeddings_input_dim (`int`, *optional*, defaults to `None`):+ The dimension of the `class_labels` input when `class_embed_type="projection"`. Required when+ `class_embed_type="projection"`.+ controlnet_conditioning_channel_order (`str`, defaults to `"rgb"`):+ The channel order of conditional image. Will convert to `rgb` if it's `bgr`.+ conditioning_embedding_out_channels (`tuple[int]`, *optional*, defaults to `(16, 32, 96, 256)`):+ The tuple of output channel for each block in the `conditioning_embedding` layer.+ global_pool_conditions (`bool`, defaults to `False`):+ """++ _supports_gradient_checkpointing = True++ @register_to_config+ def __init__(+ self,+ in_channels: int = 4,+ conditioning_channels: int = 3,+ flip_sin_to_cos: bool = True,+ freq_shift: int = 0,+ down_block_types: Tuple[str] = (+ "CrossAttnDownBlock2D",+ "CrossAttnDownBlock2D",+ "CrossAttnDownBlock2D",+ "DownBlock2D",+ ),+ up_block_types: Tuple[str] = (+ "UpBlock2D",+ "CrossAttnUpBlock2D",+ "CrossAttnUpBlock2D",+ "CrossAttnUpBlock2D",+ ),+ only_cross_attention: Union[bool, Tuple[bool]] = False,+ block_out_channels: Tuple[int] = (320, 640, 1280, 1280),+ layers_per_block: int = 2,+ downsample_padding: int = 1,+ mid_block_scale_factor: float = 1,+ act_fn: str = "silu",+ norm_num_groups: Optional[int] = 32,+ norm_eps: float = 1e-5,+ cross_attention_dim: int = 1280,+ transformer_layers_per_block: Union[int, Tuple[int]] = 1,+ encoder_hid_dim: Optional[int] = None,+ encoder_hid_dim_type: Optional[str] = None,+ attention_head_dim: Union[int, Tuple[int]] = 8,+ num_attention_heads: Optional[Union[int, Tuple[int]]] = None,+ use_linear_projection: bool = False,+ class_embed_type: Optional[str] = None,+ addition_embed_type: Optional[str] = None,+ addition_time_embed_dim: Optional[int] = None,+ num_class_embeds: Optional[int] = None,+ upcast_attention: bool = False,+ resnet_time_scale_shift: str = "default",+ projection_class_embeddings_input_dim: Optional[int] = None,+ controlnet_conditioning_channel_order: str = "rgb",+ conditioning_embedding_out_channels: Optional[Tuple[int]] = (16, 32, 96, 256),+ global_pool_conditions: bool = False,+ addition_embed_type_num_heads=64,+ ):+ super().__init__()++ # If `num_attention_heads` is not defined (which is the case for most models)+ # it will default to `attention_head_dim`. This looks weird upon first reading it and it is.+ # The reason for this behavior is to correct for incorrectly named variables that were introduced+ # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131+ # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking+ # which is why we correct for the naming here.+ num_attention_heads = num_attention_heads or attention_head_dim++ # Check inputs+ if len(block_out_channels) != len(down_block_types):+ raise ValueError(+ f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}."+ )++ if not isinstance(only_cross_attention, bool) and len(+ only_cross_attention+ ) != len(down_block_types):+ raise ValueError(+ f"Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}."+ )++ if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(down_block_types):+ raise ValueError(+ f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}."+ )++ if isinstance(transformer_layers_per_block, int):+ transformer_layers_per_block = [transformer_layers_per_block] * len(down_block_types)++ # # input+ # conv_in_kernel = 3+ # conv_in_padding = (conv_in_kernel - 1) // 2+ # self.conv_in = nn.Conv2d(+ # in_channels,+ # block_out_channels[0],+ # kernel_size=conv_in_kernel,+ # padding=conv_in_padding,+ # )+ self.conv_in = None++ # time+ time_embed_dim = block_out_channels[0] * 4+ self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift)+ timestep_input_dim = block_out_channels[0]+ self.time_embedding = TimestepEmbedding(+ timestep_input_dim,+ time_embed_dim,+ act_fn=act_fn,+ )++ if encoder_hid_dim_type is None and encoder_hid_dim is not None:+ encoder_hid_dim_type = "text_proj"+ self.register_to_config(encoder_hid_dim_type=encoder_hid_dim_type)+ logger.info(+ "encoder_hid_dim_type defaults to 'text_proj' as `encoder_hid_dim` is defined."+ )++ if encoder_hid_dim is None and encoder_hid_dim_type is not None:+ raise ValueError(+ f"`encoder_hid_dim` has to be defined when `encoder_hid_dim_type` is set to {encoder_hid_dim_type}."+ )++ if encoder_hid_dim_type == "text_proj":+ self.encoder_hid_proj = nn.Linear(encoder_hid_dim, cross_attention_dim)+ elif encoder_hid_dim_type == "text_image_proj":+ # image_embed_dim DOESN'T have to be `cross_attention_dim`. To not clutter the __init__ too much+ # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use+ # case when `addition_embed_type == "text_image_proj"` (Kadinsky 2.1)`+ self.encoder_hid_proj = TextImageProjection(+ text_embed_dim=encoder_hid_dim,+ image_embed_dim=cross_attention_dim,+ cross_attention_dim=cross_attention_dim,+ )++ elif encoder_hid_dim_type is not None:+ raise ValueError(+ f"encoder_hid_dim_type: {encoder_hid_dim_type} must be None, 'text_proj' or 'text_image_proj'."+ )+ else:+ self.encoder_hid_proj = None++ # class embedding+ if class_embed_type is None and num_class_embeds is not None:+ self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim)+ elif class_embed_type == "timestep":+ self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim)+ elif class_embed_type == "identity":+ self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim)+ elif class_embed_type == "projection":+ if projection_class_embeddings_input_dim is None:+ raise ValueError(+ "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set"+ )+ # The projection `class_embed_type` is the same as the timestep `class_embed_type` except+ # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings+ # 2. it projects from an arbitrary input dimension.+ #+ # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations.+ # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings.+ # As a result, `TimestepEmbedding` can be passed arbitrary vectors.+ self.class_embedding = TimestepEmbedding(+ projection_class_embeddings_input_dim, time_embed_dim+ )+ else:+ self.class_embedding = None++ if addition_embed_type == "text":+ if encoder_hid_dim is not None:+ text_time_embedding_from_dim = encoder_hid_dim+ else:+ text_time_embedding_from_dim = cross_attention_dim++ self.add_embedding = TextTimeEmbedding(+ text_time_embedding_from_dim,+ time_embed_dim,+ num_heads=addition_embed_type_num_heads,+ )+ elif addition_embed_type == "text_image":+ # text_embed_dim and image_embed_dim DON'T have to be `cross_attention_dim`. To not clutter the __init__ too much+ # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use+ # case when `addition_embed_type == "text_image"` (Kadinsky 2.1)`+ self.add_embedding = TextImageTimeEmbedding(+ text_embed_dim=cross_attention_dim,+ image_embed_dim=cross_attention_dim,+ time_embed_dim=time_embed_dim,+ )+ elif addition_embed_type == "text_time":+ self.add_time_proj = Timesteps(addition_time_embed_dim, flip_sin_to_cos, freq_shift)+ self.add_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim)+ elif addition_embed_type is not None:+ raise ValueError(+ f"addition_embed_type: {addition_embed_type} must be None, 'text' or 'text_image'."+ )++ # control net conditioning embedding+ self.controlnet_cond_embedding = ControlNetConditioningEmbedding(+ conditioning_embedding_channels=block_out_channels[0],+ block_out_channels=conditioning_embedding_out_channels,+ conditioning_channels=conditioning_channels,+ )++ self.down_blocks = nn.ModuleList([])++ if isinstance(only_cross_attention, bool):+ only_cross_attention = [only_cross_attention] * len(down_block_types)++ if isinstance(attention_head_dim, int):+ attention_head_dim = (attention_head_dim,) * len(down_block_types)++ if isinstance(num_attention_heads, int):+ num_attention_heads = (num_attention_heads,) * len(down_block_types)++ # # down+ # output_channel = block_out_channels[0]++ # controlnet_block = nn.Conv2d(output_channel, output_channel, kernel_size=1)+ # controlnet_block = zero_module(controlnet_block)+ # self.controlnet_down_blocks.append(controlnet_block)++ # for i, down_block_type in enumerate(down_block_types):+ # input_channel = output_channel+ # output_channel = block_out_channels[i]+ # is_final_block = i == len(block_out_channels) - 1++ # for _ in range(layers_per_block):+ # controlnet_block = nn.Conv2d(output_channel, output_channel, kernel_size=1)+ # controlnet_block = zero_module(controlnet_block)+ # self.controlnet_down_blocks.append(controlnet_block)++ # if not is_final_block:+ # controlnet_block = nn.Conv2d(output_channel, output_channel, kernel_size=1)+ # controlnet_block = zero_module(controlnet_block)+ # self.controlnet_down_blocks.append(controlnet_block)++ # mid+ mid_block_channel = block_out_channels[-1]++ controlnet_block = nn.Conv2d(mid_block_channel, mid_block_channel, kernel_size=1)+ controlnet_block = zero_module(controlnet_block)+ self.controlnet_mid_block = controlnet_block++ self.mid_block = UNetMidBlock2DCrossAttn(+ transformer_layers_per_block=transformer_layers_per_block[-1],+ in_channels=mid_block_channel,+ temb_channels=time_embed_dim,+ resnet_eps=norm_eps,+ resnet_act_fn=act_fn,+ output_scale_factor=mid_block_scale_factor,+ resnet_time_scale_shift=resnet_time_scale_shift,+ cross_attention_dim=cross_attention_dim,+ num_attention_heads=num_attention_heads[-1],+ resnet_groups=norm_num_groups,+ use_linear_projection=use_linear_projection,+ upcast_attention=upcast_attention,+ )++ # up+ self.controlnet_up_blocks = nn.ModuleList([])+ self.num_upsamplers = 0+ self.up_blocks = nn.ModuleList([])+ reversed_block_out_channels = list(reversed(block_out_channels))+ reversed_num_attention_heads = list(reversed(num_attention_heads))+ reversed_transformer_layers_per_block = list(reversed(transformer_layers_per_block))+ only_cross_attention = list(reversed(only_cross_attention))++ output_channel = reversed_block_out_channels[0]+ for i, up_block_type in enumerate(up_block_types):+ is_final_block = i == len(block_out_channels) - 1++ prev_output_channel = output_channel+ output_channel = reversed_block_out_channels[i]+ input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)]++ # add upsample block for all BUT final layer+ if not is_final_block:+ add_upsample = True+ self.num_upsamplers += 1+ else:+ add_upsample = False++ up_block = get_up_block(+ up_block_type,+ num_layers=layers_per_block + 1,+ transformer_layers_per_block=reversed_transformer_layers_per_block[i],+ in_channels=input_channel,+ out_channels=output_channel,+ prev_output_channel=prev_output_channel,+ temb_channels=time_embed_dim,+ add_upsample=add_upsample,+ resnet_eps=norm_eps,+ resnet_act_fn=act_fn,+ resnet_groups=norm_num_groups,+ cross_attention_dim=cross_attention_dim,+ num_attention_heads=reversed_num_attention_heads[i],+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention[i],+ upcast_attention=upcast_attention,+ resnet_time_scale_shift=resnet_time_scale_shift,+ attention_head_dim=(+ attention_head_dim[i]+ if attention_head_dim[i] is not None+ else output_channel+ ),+ )+ self.up_blocks.append(up_block)+ prev_output_channel = output_channel++ # if i>0: # 因为我们只输出transformer相关的,而第一级的upblock是纯conv+ for _ in range(layers_per_block + 1):+ controlnet_block = nn.Conv2d(output_channel, output_channel, kernel_size=1)+ controlnet_block = zero_module(controlnet_block)+ self.controlnet_up_blocks.append(controlnet_block)++ @classmethod+ def from_unet(+ cls,+ unet: UNet2DConditionModel,+ controlnet_conditioning_channel_order: str = "rgb",+ conditioning_embedding_out_channels: Optional[Tuple[int]] = (16, 32, 96, 256),+ load_weights_from_unet: bool = True,+ ):+ r"""+ Instantiate a [`ControlNetModel`] from [`UNet2DConditionModel`].++ Parameters:+ unet (`UNet2DConditionModel`):+ The UNet model weights to copy to the [`ControlNetModel`]. All configuration options are also copied+ where applicable.+ """+ transformer_layers_per_block = (+ unet.config.transformer_layers_per_block+ if "transformer_layers_per_block" in unet.config+ else 1+ )+ encoder_hid_dim = (+ unet.config.encoder_hid_dim if "encoder_hid_dim" in unet.config else None+ )+ encoder_hid_dim_type = (+ unet.config.encoder_hid_dim_type+ if "encoder_hid_dim_type" in unet.config+ else None+ )+ addition_embed_type = (+ unet.config.addition_embed_type+ if "addition_embed_type" in unet.config+ else None+ )+ addition_time_embed_dim = (+ unet.config.addition_time_embed_dim+ if "addition_time_embed_dim" in unet.config+ else None+ )++ controlnet = cls(+ encoder_hid_dim=encoder_hid_dim,+ encoder_hid_dim_type=encoder_hid_dim_type,+ addition_embed_type=addition_embed_type,+ addition_time_embed_dim=addition_time_embed_dim,+ transformer_layers_per_block=transformer_layers_per_block,+ in_channels=unet.config.in_channels,+ flip_sin_to_cos=unet.config.flip_sin_to_cos,+ freq_shift=unet.config.freq_shift,+ down_block_types=unet.config.down_block_types,+ only_cross_attention=unet.config.only_cross_attention,+ block_out_channels=unet.config.block_out_channels,+ layers_per_block=unet.config.layers_per_block,+ downsample_padding=unet.config.downsample_padding,+ mid_block_scale_factor=unet.config.mid_block_scale_factor,+ act_fn=unet.config.act_fn,+ norm_num_groups=unet.config.norm_num_groups,+ norm_eps=unet.config.norm_eps,+ cross_attention_dim=unet.config.cross_attention_dim,+ attention_head_dim=unet.config.attention_head_dim,+ num_attention_heads=unet.config.num_attention_heads,+ use_linear_projection=unet.config.use_linear_projection,+ class_embed_type=unet.config.class_embed_type,+ num_class_embeds=unet.config.num_class_embeds,+ upcast_attention=unet.config.upcast_attention,+ resnet_time_scale_shift=unet.config.resnet_time_scale_shift,+ projection_class_embeddings_input_dim=unet.config.projection_class_embeddings_input_dim,+ controlnet_conditioning_channel_order=controlnet_conditioning_channel_order,+ conditioning_embedding_out_channels=conditioning_embedding_out_channels,+ ).requires_grad_(False)++ if load_weights_from_unet:+ # controlnet.conv_in.load_state_dict(unet.conv_in.state_dict())+ controlnet.time_proj.load_state_dict(unet.time_proj.state_dict())+ controlnet.time_embedding.load_state_dict(unet.time_embedding.state_dict())++ if controlnet.class_embedding:+ controlnet.class_embedding.load_state_dict(unet.class_embedding.state_dict())++ # controlnet.down_blocks.load_state_dict(unet.down_blocks.state_dict())+ controlnet.mid_block.load_state_dict(unet.mid_block.state_dict())+ # controlnet.up_blocks.load_state_dict(unet.up_blocks.state_dict(), strict=False) # TODO:将各个upblock第一层融合层使用weight进行部分初始化++ src_tensors = dict(named_params_and_buffers(unet.up_blocks))+ for name, tensor in named_params_and_buffers(controlnet.up_blocks):+ assert (name in src_tensors), name+ try:+ # print('Successfully initializing ControlNet!', name, tensor.shape, src_tensors[name].shape)+ tensor.copy_(src_tensors[name].detach())+ except:+ # print('Mismatch occured in initializing ControlNet!', name, tensor.shape, src_tensors[name].shape)+ # TODO: 确保所有upblock参数有初始化+ if tensor.dim() == 1:+ tensor.copy_(src_tensors[name].detach()[:tensor.shape[0]])+ else:+ tensor.copy_(src_tensors[name].detach()[:, :tensor.shape[1]])++ return controlnet++ def _set_gradient_checkpointing(self, module, value=False):+ if hasattr(module, "gradient_checkpointing"):+ module.gradient_checkpointing = value++ def forward(+ self,+ sample: torch.FloatTensor,+ timestep: Union[torch.Tensor, float, int],+ encoder_hidden_states: torch.Tensor,+ # controlnet_cond: torch.FloatTensor,+ conditioning_scale: float = 1.0,+ class_labels: Optional[torch.Tensor] = None,+ timestep_cond: Optional[torch.Tensor] = None,+ attention_mask: Optional[torch.Tensor] = None,+ added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,+ guess_mode: bool = False,+ return_dict: bool = True,+ only_return_transformer_layers_out: bool = False,+ ) -> Union[ControlNetOutput, Tuple]:+ """+ The [`ControlNetModel`] forward method.++ Args:+ sample (`torch.FloatTensor`):+ The noisy input tensor.+ timestep (`Union[torch.Tensor, float, int]`):+ The number of timesteps to denoise an input.+ encoder_hidden_states (`torch.Tensor`):+ The encoder hidden states.+ controlnet_cond (`torch.FloatTensor`):+ The conditional input tensor of shape `(batch_size, sequence_length, hidden_size)`.+ conditioning_scale (`float`, defaults to `1.0`):+ The scale factor for ControlNet outputs.+ class_labels (`torch.Tensor`, *optional*, defaults to `None`):+ Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings.+ timestep_cond (`torch.Tensor`, *optional*, defaults to `None`):+ attention_mask (`torch.Tensor`, *optional*, defaults to `None`):+ added_cond_kwargs (`dict`):+ Additional conditions for the Stable Diffusion XL UNet.+ cross_attention_kwargs (`dict[str]`, *optional*, defaults to `None`):+ A kwargs dictionary that if specified is passed along to the `AttnProcessor`.+ guess_mode (`bool`, defaults to `False`):+ In this mode, the ControlNet encoder tries its best to recognize the input content of the input even if+ you remove all prompts. A `guidance_scale` between 3.0 and 5.0 is recommended.+ return_dict (`bool`, defaults to `True`):+ Whether or not to return a [`~models.controlnet.ControlNetOutput`] instead of a plain tuple.++ Returns:+ [`~models.controlnet.ControlNetOutput`] **or** `tuple`:+ If `return_dict` is `True`, a [`~models.controlnet.ControlNetOutput`] is returned, otherwise a tuple is+ returned where the first element is the sample tensor.+ """++ # prepare attention_mask+ if attention_mask is not None:+ attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0+ attention_mask = attention_mask.unsqueeze(1)++ # 1. time+ timesteps = timestep+ if not torch.is_tensor(timesteps):+ # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can+ # This would be a good case for the `match` statement (Python 3.10+)+ is_mps = sample.device.type == "mps"+ if isinstance(timestep, float):+ dtype = torch.float32 if is_mps else torch.float64+ else:+ dtype = torch.int32 if is_mps else torch.int64+ timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)+ elif len(timesteps.shape) == 0:+ timesteps = timesteps[None].to(sample.device)++ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML+ timesteps = timesteps.expand(sample.shape[0])++ t_emb = self.time_proj(timesteps)++ # timesteps does not contain any weights and will always return f32 tensors+ # but time_embedding might actually be running in fp16. so we need to cast here.+ # there might be better ways to encapsulate this.+ t_emb = t_emb.to(dtype=sample.dtype)++ emb = self.time_embedding(t_emb, timestep_cond)+ aug_emb = None++ if self.class_embedding is not None:+ if class_labels is None:+ raise ValueError(+ "class_labels should be provided when num_class_embeds > 0"+ )++ if self.config.class_embed_type == "timestep":+ class_labels = self.time_proj(class_labels)++ class_emb = self.class_embedding(class_labels).to(dtype=self.dtype)+ emb = emb + class_emb++ if self.config.addition_embed_type is not None:+ if self.config.addition_embed_type == "text":+ aug_emb = self.add_embedding(encoder_hidden_states)++ elif self.config.addition_embed_type == "text_time":+ if "text_embeds" not in added_cond_kwargs:+ raise ValueError(+ f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`"+ )+ text_embeds = added_cond_kwargs.get("text_embeds")+ if "time_ids" not in added_cond_kwargs:+ raise ValueError(+ f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`"+ )+ time_ids = added_cond_kwargs.get("time_ids")+ time_embeds = self.add_time_proj(time_ids.flatten())+ time_embeds = time_embeds.reshape((text_embeds.shape[0], -1))++ add_embeds = torch.concat([text_embeds, time_embeds], dim=-1)+ add_embeds = add_embeds.to(emb.dtype)+ aug_emb = self.add_embedding(add_embeds)++ emb = emb + aug_emb if aug_emb is not None else emb++ if self.conv_in is not None:+ sample = self.conv_in(sample)++ # 4. mid+ if self.mid_block is not None:+ sample = self.mid_block(+ sample,+ emb,+ encoder_hidden_states=encoder_hidden_states,+ attention_mask=attention_mask,+ cross_attention_kwargs=cross_attention_kwargs,+ )+ mid_block_res_sample = self.controlnet_mid_block(sample)++ # 5. up++ out_block_res_samples = []+ for i, upsample_block in enumerate(self.up_blocks):+ if (hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention):+ sample, res_samples = upsample_block(+ hidden_states=sample,+ temb=emb,+ # res_hidden_states_tuple=res_samples,+ encoder_hidden_states=encoder_hidden_states,+ cross_attention_kwargs=cross_attention_kwargs,+ attention_mask=attention_mask,+ )+ else:+ sample, res_samples = upsample_block(+ hidden_states=sample,+ temb=emb,+ # res_hidden_states_tuple=res_samples,+ )+ # print('Out', i, [res_sample.shape for res_sample in res_samples])+ # if i in range(1, len(self.up_blocks)-1):+ # res_samples[-1] = res_samples[-1][:, :sample.shape[1]//2] # 为了适配controlNet输出的shape++ out_block_res_samples += res_samples++ # 5. Control net blocks++ controlnet_up_block_res_samples = ()+ assert len(out_block_res_samples) == len(self.controlnet_up_blocks), (len(out_block_res_samples), len(self.controlnet_up_blocks))+ for i, controlnet_block in enumerate(self.controlnet_up_blocks): # zero proj out+ # if only_return_transformer_layers_out and i < 3: # 第一个upblock是纯conv的+ # continue+ up_block_res_sample = controlnet_block(out_block_res_samples[i])+ controlnet_up_block_res_samples = controlnet_up_block_res_samples + (up_block_res_sample,)++ up_block_res_samples = controlnet_up_block_res_samples++ # 6. scaling+ if guess_mode and not self.config.global_pool_conditions:+ raise NotImplementedError+ scales = torch.logspace(-1, 0, len(down_block_res_samples) + 1, device=sample.device) # 0.1 to 1.0++ scales = scales * conditioning_scale+ down_block_res_samples = [sample * scale for sample, scale in zip(down_block_res_samples, scales)]+ mid_block_res_sample = mid_block_res_sample * scales[-1] # last one+ else:+ up_block_res_samples = [sample * conditioning_scale for sample in up_block_res_samples]+ mid_block_res_sample = mid_block_res_sample * conditioning_scale++ if self.config.global_pool_conditions:+ up_block_res_samples = [torch.mean(sample, dim=(2, 3), keepdim=True) for sample in up_block_res_samples]+ mid_block_res_sample = torch.mean(mid_block_res_sample, dim=(2, 3), keepdim=True)++ if only_return_transformer_layers_out:+ down_block_res_samples = [sample for i, sample in enumerate(up_block_res_samples) if not i % 3== 2][::-1]+ else:+ down_block_res_samples = list(reversed(up_block_res_samples))++ if not return_dict:+ return (down_block_res_samples, mid_block_res_sample, up_block_res_samples)++ return ControlNetOutput(+ down_block_res_samples=down_block_res_samples,+ mid_block_res_sample=mid_block_res_sample,+ up_block_res_samples=up_block_res_samples,+ )+++def zero_module(module):+ for p in module.parameters():+ nn.init.zeros_(p)+ return module+++def named_params_and_buffers(module):+ assert isinstance(module, torch.nn.Module)+ return list(module.named_parameters()) + list(module.named_buffers())+++from src.models.unet_2d_blocks import UpBlock2D, CrossAttnUpBlock2D+from diffusers.models.resnet import ResnetBlock2D++from diffusers.utils import is_torch_version+class UpBlock2D_woskip(UpBlock2D):+ def __init__(+ self,+ prev_output_channel: int,+ out_channels: int,+ temb_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default",+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ output_scale_factor=1.0,+ add_upsample=True,+ ):+ super(UpBlock2D_woskip, self).__init__(+ in_channels=0,+ prev_output_channel=prev_output_channel,+ out_channels=out_channels,+ temb_channels=temb_channels,+ dropout=dropout,+ num_layers=num_layers,+ resnet_eps=resnet_eps,+ resnet_time_scale_shift=resnet_time_scale_shift,+ resnet_act_fn=resnet_act_fn,+ resnet_groups=resnet_groups,+ resnet_pre_norm=resnet_pre_norm,+ output_scale_factor=output_scale_factor,+ add_upsample=add_upsample,+ )+ resnets = []++ for i in range(num_layers):+ resnet_in_channels = prev_output_channel if i == 0 else out_channels++ resnets.append(+ ResnetBlock2D(+ in_channels=resnet_in_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ )+ )++ self.resnets = nn.ModuleList(resnets)++ def forward(self, hidden_states, temb=None, upsample_size=None, scale: float = 1.0):+ output_states = []+ for resnet in self.resnets:+ if self.training and self.gradient_checkpointing:++ def create_custom_forward(module):+ def custom_forward(*inputs):+ return module(*inputs)++ return custom_forward++ if is_torch_version(">=", "1.11.0"):+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet), hidden_states, temb, use_reentrant=False+ )+ else:+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet), hidden_states, temb+ )+ else:+ hidden_states = resnet(hidden_states, temb, scale=scale)+ output_states = output_states + [hidden_states,]++ if self.upsamplers is not None:+ for upsampler in self.upsamplers:+ hidden_states = upsampler(hidden_states, upsample_size, scale=scale)++ return hidden_states, output_states+++class CrossAttnUpBlock2D_woskip(CrossAttnUpBlock2D):+ def __init__(+ self,+ out_channels: int,+ prev_output_channel: int,+ temb_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ transformer_layers_per_block: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default",+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ num_attention_heads=1,+ cross_attention_dim=1280,+ output_scale_factor=1.0,+ add_upsample=True,+ dual_cross_attention=False,+ use_linear_projection=False,+ only_cross_attention=False,+ upcast_attention=False,+ attention_type="default",+ ):+ super(CrossAttnUpBlock2D_woskip, self).__init__(+ in_channels=0,+ out_channels=out_channels,+ prev_output_channel=prev_output_channel,+ temb_channels=temb_channels,+ dropout=dropout,+ num_layers=num_layers,+ transformer_layers_per_block=transformer_layers_per_block,+ resnet_eps=resnet_eps,+ resnet_time_scale_shift=resnet_time_scale_shift,+ resnet_act_fn=resnet_act_fn,+ resnet_groups=resnet_groups,+ resnet_pre_norm=resnet_pre_norm,+ num_attention_heads=num_attention_heads,+ cross_attention_dim=cross_attention_dim,+ output_scale_factor=output_scale_factor,+ add_upsample=add_upsample,+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention,+ upcast_attention=upcast_attention,+ attention_type=attention_type,+ )+ resnets = []++ for i in range(num_layers):+ resnet_in_channels = prev_output_channel if i == 0 else out_channels+ resnets.append(+ ResnetBlock2D(+ in_channels=resnet_in_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ )+ )+ self.resnets = nn.ModuleList(resnets)++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ temb: Optional[torch.FloatTensor] = None,+ encoder_hidden_states: Optional[torch.FloatTensor] = None,+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,+ upsample_size: Optional[int] = None,+ attention_mask: Optional[torch.FloatTensor] = None,+ encoder_attention_mask: Optional[torch.FloatTensor] = None,+ ):+ lora_scale = (+ cross_attention_kwargs.get("scale", 1.0)+ if cross_attention_kwargs is not None+ else 1.0+ )+ output_states = []+ for resnet, attn in zip(self.resnets, self.attentions):++ if self.training and self.gradient_checkpointing:++ def create_custom_forward(module, return_dict=None):+ def custom_forward(*inputs):+ if return_dict is not None:+ return module(*inputs, return_dict=return_dict)+ else:+ return module(*inputs)++ return custom_forward++ ckpt_kwargs: Dict[str, Any] = (+ {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {}+ )+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet),+ hidden_states,+ temb,+ **ckpt_kwargs,+ )+ hidden_states = attn(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ cross_attention_kwargs=cross_attention_kwargs,+ attention_mask=attention_mask,+ encoder_attention_mask=encoder_attention_mask,+ return_dict=False,+ )[0]+ else:+ hidden_states = resnet(hidden_states, temb, scale=lora_scale)+ hidden_states = attn(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ cross_attention_kwargs=cross_attention_kwargs,+ attention_mask=attention_mask,+ encoder_attention_mask=encoder_attention_mask,+ return_dict=False,+ )[0]+ output_states = output_states + [hidden_states,]+ if self.upsamplers is not None:+ for upsampler in self.upsamplers:+ hidden_states = upsampler(hidden_states, upsample_size, scale=lora_scale)++ return hidden_states, output_states+++def get_up_block(+ up_block_type,+ num_layers,+ in_channels,+ out_channels,+ prev_output_channel,+ temb_channels,+ add_upsample,+ resnet_eps,+ resnet_act_fn,+ transformer_layers_per_block=1,+ num_attention_heads=None,+ resnet_groups=None,+ cross_attention_dim=None,+ dual_cross_attention=False,+ use_linear_projection=False,+ only_cross_attention=False,+ upcast_attention=False,+ resnet_time_scale_shift="default",+ attention_type="default",+ resnet_skip_time_act=False,+ resnet_out_scale_factor=1.0,+ cross_attention_norm=None,+ attention_head_dim=None,+ upsample_type=None,+ dropout=0.0,+):+ # If attn head dim is not defined, we default it to the number of heads+ if attention_head_dim is None:+ logger.warn(+ f"It is recommended to provide `attention_head_dim` when calling `get_up_block`. Defaulting `attention_head_dim` to {num_attention_heads}."+ )+ attention_head_dim = num_attention_heads++ up_block_type = up_block_type[7:] if up_block_type.startswith("UNetRes") else up_block_type++ if up_block_type == "UpBlock2D":+ return UpBlock2D_woskip(+ num_layers=num_layers,+ # in_channels=in_channels,+ out_channels=out_channels,+ prev_output_channel=prev_output_channel,+ temb_channels=temb_channels,+ dropout=dropout,+ add_upsample=add_upsample,+ resnet_eps=resnet_eps,+ resnet_act_fn=resnet_act_fn,+ resnet_groups=resnet_groups,+ resnet_time_scale_shift=resnet_time_scale_shift,+ )+ elif up_block_type == "CrossAttnUpBlock2D":+ if cross_attention_dim is None:+ raise ValueError("cross_attention_dim must be specified for CrossAttnUpBlock2D")+ return CrossAttnUpBlock2D_woskip(+ num_layers=num_layers,+ transformer_layers_per_block=transformer_layers_per_block,+ # in_channels=in_channels,+ out_channels=out_channels,+ prev_output_channel=prev_output_channel,+ temb_channels=temb_channels,+ dropout=dropout,+ add_upsample=add_upsample,+ resnet_eps=resnet_eps,+ resnet_act_fn=resnet_act_fn,+ resnet_groups=resnet_groups,+ cross_attention_dim=cross_attention_dim,+ num_attention_heads=num_attention_heads,+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention,+ upcast_attention=upcast_attention,+ resnet_time_scale_shift=resnet_time_scale_shift,+ attention_type=attention_type,+ )++ raise ValueError(f"{up_block_type} does not exist.")diff --git a/src/models/unet_3d.py b/src/models/unet_3d.pynew file mode 100644index 0000000..cec8390--- /dev/null+++ b/src/models/unet_3d.py@@ -0,0 +1,716 @@+from collections import OrderedDict+from dataclasses import dataclass+import pdb+from os import PathLike+from pathlib import Path+from typing import Dict, List, Optional, Tuple, Union++import torch+import torch.nn as nn+import torch.utils.checkpoint+import torch.nn.functional as F+from diffusers.configuration_utils import ConfigMixin, register_to_config+from diffusers.models.attention_processor import AttentionProcessor+from diffusers.models.embeddings import TimestepEmbedding, Timesteps+from diffusers.models.modeling_utils import ModelMixin+from diffusers.utils import SAFETENSORS_WEIGHTS_NAME, WEIGHTS_NAME, BaseOutput, logging+from safetensors.torch import load_file++from .resnet import InflatedConv3d, InflatedGroupNorm+from .unet_3d_blocks import UNetMidBlock3DCrossAttn, get_down_block, get_up_block++logger = logging.get_logger(__name__) # pylint: disable=invalid-name+++@dataclass+class UNet3DConditionOutput(BaseOutput):+ sample: torch.FloatTensor+++class UNet3DConditionModel(ModelMixin, ConfigMixin):+ _supports_gradient_checkpointing = True++ @register_to_config+ def __init__(+ self,+ sample_size: Optional[int] = None,+ in_channels: int = 4,+ out_channels: int = 4,+ center_input_sample: bool = False,+ flip_sin_to_cos: bool = True,+ freq_shift: int = 0,+ down_block_types: Tuple[str] = (+ "CrossAttnDownBlock3D",+ "CrossAttnDownBlock3D",+ "CrossAttnDownBlock3D",+ "DownBlock3D",+ ),+ mid_block_type: str = "UNetMidBlock3DCrossAttn",+ up_block_types: Tuple[str] = (+ "UpBlock3D",+ "CrossAttnUpBlock3D",+ "CrossAttnUpBlock3D",+ "CrossAttnUpBlock3D",+ ),+ only_cross_attention: Union[bool, Tuple[bool]] = False,+ block_out_channels: Tuple[int] = (320, 640, 1280, 1280),+ layers_per_block: int = 2,+ downsample_padding: int = 1,+ mid_block_scale_factor: float = 1,+ act_fn: str = "silu",+ norm_num_groups: int = 32,+ norm_eps: float = 1e-5,+ cross_attention_dim: int = 1280,+ attention_head_dim: Union[int, Tuple[int]] = 8,+ dual_cross_attention: bool = False,+ use_linear_projection: bool = False,+ class_embed_type: Optional[str] = None,+ num_class_embeds: Optional[int] = None,+ upcast_attention: bool = False,+ resnet_time_scale_shift: str = "default",+ use_inflated_groupnorm=False,+ # Additional+ use_motion_module=False,+ use_temporal_module=False,+ motion_module_resolutions=(1, 2, 4, 8),+ motion_module_mid_block=False,+ motion_module_decoder_only=False,+ motion_module_type=None,+ temporal_module_type=None,+ motion_module_kwargs={},+ temporal_module_kwargs={},+ unet_use_cross_frame_attention=None,+ unet_use_temporal_attention=None,+ ):+ super().__init__()++ self.sample_size = sample_size+ time_embed_dim = block_out_channels[0] * 4++ # input+ self.conv_in = InflatedConv3d(+ in_channels, block_out_channels[0], kernel_size=3, padding=(1, 1)+ )++ # time+ self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift)+ timestep_input_dim = block_out_channels[0]++ self.time_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim)++ # class embedding+ if class_embed_type is None and num_class_embeds is not None:+ self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim)+ elif class_embed_type == "timestep":+ self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim)+ elif class_embed_type == "identity":+ self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim)+ else:+ self.class_embedding = None++ self.down_blocks = nn.ModuleList([])+ self.mid_block = None+ self.up_blocks = nn.ModuleList([])++ if isinstance(only_cross_attention, bool):+ only_cross_attention = [only_cross_attention] * len(down_block_types)++ if isinstance(attention_head_dim, int):+ attention_head_dim = (attention_head_dim,) * len(down_block_types)++ # down+ output_channel = block_out_channels[0]+ for i, down_block_type in enumerate(down_block_types):+ res = 2**i+ input_channel = output_channel+ output_channel = block_out_channels[i]+ is_final_block = i == len(block_out_channels) - 1++ down_block = get_down_block(+ down_block_type,+ num_layers=layers_per_block,+ in_channels=input_channel,+ out_channels=output_channel,+ temb_channels=time_embed_dim,+ add_downsample=not is_final_block,+ resnet_eps=norm_eps,+ resnet_act_fn=act_fn,+ resnet_groups=norm_num_groups,+ cross_attention_dim=cross_attention_dim,+ attn_num_head_channels=attention_head_dim[i],+ downsample_padding=downsample_padding,+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention[i],+ upcast_attention=upcast_attention,+ resnet_time_scale_shift=resnet_time_scale_shift,+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,+ unet_use_temporal_attention=unet_use_temporal_attention,+ use_inflated_groupnorm=use_inflated_groupnorm,+ use_motion_module=use_motion_module+ and (res in motion_module_resolutions)+ and (not motion_module_decoder_only),+ use_temporal_module=use_temporal_module+ and (res in motion_module_resolutions)+ and (not motion_module_decoder_only),+ motion_module_type=motion_module_type,+ temporal_module_type=temporal_module_type,+ motion_module_kwargs=motion_module_kwargs,+ temporal_module_kwargs=temporal_module_kwargs+ )+ self.down_blocks.append(down_block)++ # mid+ if mid_block_type == "UNetMidBlock3DCrossAttn":+ self.mid_block = UNetMidBlock3DCrossAttn(+ in_channels=block_out_channels[-1],+ temb_channels=time_embed_dim,+ resnet_eps=norm_eps,+ resnet_act_fn=act_fn,+ output_scale_factor=mid_block_scale_factor,+ resnet_time_scale_shift=resnet_time_scale_shift,+ cross_attention_dim=cross_attention_dim,+ attn_num_head_channels=attention_head_dim[-1],+ resnet_groups=norm_num_groups,+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ upcast_attention=upcast_attention,+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,+ unet_use_temporal_attention=unet_use_temporal_attention,+ use_inflated_groupnorm=use_inflated_groupnorm,+ use_motion_module=use_motion_module and motion_module_mid_block,+ use_temporal_module=use_temporal_module and motion_module_mid_block,+ motion_module_type=motion_module_type,+ temporal_module_type=temporal_module_type,+ motion_module_kwargs=motion_module_kwargs,+ temporal_module_kwargs=temporal_module_kwargs,+ )+ else:+ raise ValueError(f"unknown mid_block_type : {mid_block_type}")++ # count how many layers upsample the videos+ self.num_upsamplers = 0++ # up+ reversed_block_out_channels = list(reversed(block_out_channels))+ reversed_attention_head_dim = list(reversed(attention_head_dim))+ only_cross_attention = list(reversed(only_cross_attention))+ output_channel = reversed_block_out_channels[0]+ for i, up_block_type in enumerate(up_block_types):+ res = 2 ** (3 - i)+ is_final_block = i == len(block_out_channels) - 1++ prev_output_channel = output_channel+ output_channel = reversed_block_out_channels[i]+ input_channel = reversed_block_out_channels[+ min(i + 1, len(block_out_channels) - 1)+ ]++ # add upsample block for all BUT final layer+ if not is_final_block:+ add_upsample = True+ self.num_upsamplers += 1+ else:+ add_upsample = False++ up_block = get_up_block(+ up_block_type,+ num_layers=layers_per_block + 1,+ in_channels=input_channel,+ out_channels=output_channel,+ prev_output_channel=prev_output_channel,+ temb_channels=time_embed_dim,+ add_upsample=add_upsample,+ resnet_eps=norm_eps,+ resnet_act_fn=act_fn,+ resnet_groups=norm_num_groups,+ cross_attention_dim=cross_attention_dim,+ attn_num_head_channels=reversed_attention_head_dim[i],+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention[i],+ upcast_attention=upcast_attention,+ resnet_time_scale_shift=resnet_time_scale_shift,+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,+ unet_use_temporal_attention=unet_use_temporal_attention,+ use_inflated_groupnorm=use_inflated_groupnorm,+ use_motion_module=use_motion_module+ and (res in motion_module_resolutions),+ use_temporal_module=use_temporal_module+ and (res in motion_module_resolutions),+ motion_module_type=motion_module_type,+ temporal_module_type=temporal_module_type,+ motion_module_kwargs=motion_module_kwargs,+ temporal_module_kwargs=temporal_module_kwargs,+ )+ self.up_blocks.append(up_block)+ prev_output_channel = output_channel++ # out+ if use_inflated_groupnorm:+ self.conv_norm_out = InflatedGroupNorm(+ num_channels=block_out_channels[0],+ num_groups=norm_num_groups,+ eps=norm_eps,+ )+ else:+ self.conv_norm_out = nn.GroupNorm(+ num_channels=block_out_channels[0],+ num_groups=norm_num_groups,+ eps=norm_eps,+ )+ self.conv_act = nn.SiLU()+ self.conv_out = InflatedConv3d(+ block_out_channels[0], out_channels, kernel_size=3, padding=1+ )++ @property+ # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors+ def attn_processors(self) -> Dict[str, AttentionProcessor]:+ r"""+ Returns:+ `dict` of attention processors: A dictionary containing all attention processors used in the model with+ indexed by its weight name.+ """+ # set recursively+ processors = {}++ def fn_recursive_add_processors(+ name: str,+ module: torch.nn.Module,+ processors: Dict[str, AttentionProcessor],+ ):+ if hasattr(module, "set_processor"):+ processors[f"{name}.processor"] = module.processor++ for sub_name, child in module.named_children():+ if "temporal_transformer" not in sub_name:+ fn_recursive_add_processors(f"{name}.{sub_name}", child, processors)++ return processors++ for name, module in self.named_children():+ if "temporal_transformer" not in name:+ fn_recursive_add_processors(name, module, processors)++ return processors++ def set_attention_slice(self, slice_size):+ r"""+ Enable sliced attention computation.++ When this option is enabled, the attention module will split the input tensor in slices, to compute attention+ in several steps. This is useful to save some memory in exchange for a small speed decrease.++ Args:+ slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`):+ When `"auto"`, halves the input to the attention heads, so attention will be computed in two steps. If+ `"max"`, maxium amount of memory will be saved by running only one slice at a time. If a number is+ provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim`+ must be a multiple of `slice_size`.+ """+ sliceable_head_dims = []++ def fn_recursive_retrieve_slicable_dims(module: torch.nn.Module):+ if hasattr(module, "set_attention_slice"):+ sliceable_head_dims.append(module.sliceable_head_dim)++ for child in module.children():+ fn_recursive_retrieve_slicable_dims(child)++ # retrieve number of attention layers+ for module in self.children():+ fn_recursive_retrieve_slicable_dims(module)++ num_slicable_layers = len(sliceable_head_dims)++ if slice_size == "auto":+ # half the attention head size is usually a good trade-off between+ # speed and memory+ slice_size = [dim // 2 for dim in sliceable_head_dims]+ elif slice_size == "max":+ # make smallest slice possible+ slice_size = num_slicable_layers * [1]++ slice_size = (+ num_slicable_layers * [slice_size]+ if not isinstance(slice_size, list)+ else slice_size+ )++ if len(slice_size) != len(sliceable_head_dims):+ raise ValueError(+ f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different"+ f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}."+ )++ for i in range(len(slice_size)):+ size = slice_size[i]+ dim = sliceable_head_dims[i]+ if size is not None and size > dim:+ raise ValueError(f"size {size} has to be smaller or equal to {dim}.")++ # Recursively walk through all the children.+ # Any children which exposes the set_attention_slice method+ # gets the message+ def fn_recursive_set_attention_slice(+ module: torch.nn.Module, slice_size: List[int]+ ):+ if hasattr(module, "set_attention_slice"):+ module.set_attention_slice(slice_size.pop())++ for child in module.children():+ fn_recursive_set_attention_slice(child, slice_size)++ reversed_slice_size = list(reversed(slice_size))+ for module in self.children():+ fn_recursive_set_attention_slice(module, reversed_slice_size)++ def set_use_cross_frame_attention(self, value):++ def fn_recursive_set_use_cf_att(module: torch.nn.Module, value):+ if hasattr(module, "set_use_cross_frame_attention"):+ module.set_use_cross_frame_attention(value)++ for child in module.children():+ fn_recursive_set_use_cf_att(child, value)++ for module in self.children():+ fn_recursive_set_use_cf_att(module, value)++ def _set_gradient_checkpointing(self, module, value=False):+ if hasattr(module, "gradient_checkpointing"):+ module.gradient_checkpointing = value++ # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.set_attn_processor+ def set_attn_processor(+ self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]+ ):+ r"""+ Sets the attention processor to use to compute attention.++ Parameters:+ processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):+ The instantiated processor class or a dictionary of processor classes that will be set as the processor+ for **all** `Attention` layers.++ If `processor` is a dict, the key needs to define the path to the corresponding cross attention+ processor. This is strongly recommended when setting trainable attention processors.++ """+ count = len(self.attn_processors.keys())++ if isinstance(processor, dict) and len(processor) != count:+ raise ValueError(+ f"A dict of processors was passed, but the number of processors {len(processor)} does not match the"+ f" number of attention layers: {count}. Please make sure to pass {count} processor classes."+ )++ def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor):+ if hasattr(module, "set_processor"):+ if not isinstance(processor, dict):+ module.set_processor(processor)+ else:+ module.set_processor(processor.pop(f"{name}.processor"))++ for sub_name, child in module.named_children():+ fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor)++ for name, module in self.named_children():+ fn_recursive_attn_processor(name, module, processor)++ def forward(+ self,+ sample: torch.FloatTensor,+ timestep: Union[torch.Tensor, float, int],+ encoder_hidden_states: torch.Tensor,+ class_labels: Optional[torch.Tensor] = None,+ pose_cond_fea = None,+ attention_mask: Optional[torch.Tensor] = None,+ down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None,+ mid_block_additional_residual: Optional[torch.Tensor] = None,+ return_dict: bool = True,+ skip_mm: bool = False,+ ) -> Union[UNet3DConditionOutput, Tuple]:+ r"""+ Args:+ sample (`torch.FloatTensor`): (batch, channel, height, width) noisy inputs tensor+ timestep (`torch.FloatTensor` or `float` or `int`): (batch) timesteps+ encoder_hidden_states (`torch.FloatTensor`): (batch, sequence_length, feature_dim) encoder hidden states+ return_dict (`bool`, *optional*, defaults to `True`):+ Whether or not to return a [`models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain tuple.++ Returns:+ [`~models.unet_2d_condition.UNet2DConditionOutput`] or `tuple`:+ [`~models.unet_2d_condition.UNet2DConditionOutput`] if `return_dict` is True, otherwise a `tuple`. When+ returning a tuple, the first element is the sample tensor.+ """+ # By default samples have to be AT least a multiple of the overall upsampling factor.+ # The overall upsampling factor is equal to 2 ** (# num of upsampling layears).+ # However, the upsampling interpolation output size can be forced to fit any upsampling size+ # on the fly if necessary.+ default_overall_up_factor = 2**self.num_upsamplers++ # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`+ forward_upsample_size = False+ upsample_size = None++ if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]):+ logger.info("Forward upsample size to force interpolation output size.")+ forward_upsample_size = True++ # prepare attention_mask+ if attention_mask is not None:+ attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0+ attention_mask = attention_mask.unsqueeze(1)++ # center input if necessary+ if self.config.center_input_sample:+ sample = 2 * sample - 1.0++ # time+ timesteps = timestep+ if not torch.is_tensor(timesteps):+ # This would be a good case for the `match` statement (Python 3.10+)+ is_mps = sample.device.type == "mps"+ if isinstance(timestep, float):+ dtype = torch.float32 if is_mps else torch.float64+ else:+ dtype = torch.int32 if is_mps else torch.int64+ timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)+ elif len(timesteps.shape) == 0:+ timesteps = timesteps[None].to(sample.device)++ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML+ # try:+ # timesteps = timesteps.expand(sample.shape[0])+ # except Exception:+ # timesteps = timesteps.expand(sample.shape[0] * sample.shape[2])++ t_emb = self.time_proj(timesteps)++ # timesteps does not contain any weights and will always return f32 tensors+ # but time_embedding might actually be running in fp16. so we need to cast here.+ # there might be better ways to encapsulate this.+ t_emb = t_emb.to(dtype=self.dtype)+ emb = self.time_embedding(t_emb)+ if self.class_embedding is not None:+ if class_labels is None:+ raise ValueError(+ "class_labels should be provided when num_class_embeds > 0"+ )++ if self.config.class_embed_type == "timestep":+ class_labels = self.time_proj(class_labels)++ class_emb = self.class_embedding(class_labels).to(dtype=self.dtype)+ emb = emb + class_emb++ # pre-process+ sample = self.conv_in(sample)+ if pose_cond_fea is not None:+ sample = sample + pose_cond_fea++ # down+ down_block_res_samples = (sample,)+ block_count = 1+ for downsample_block in self.down_blocks:+ if (+ hasattr(downsample_block, "has_cross_attention")+ and downsample_block.has_cross_attention+ ):+ sample, res_samples = downsample_block(+ hidden_states=sample,+ temb=emb,+ encoder_hidden_states=encoder_hidden_states,+ attention_mask=attention_mask,+ skip_mm=skip_mm,+ )+ else:+ sample, res_samples = downsample_block(+ hidden_states=sample,+ temb=emb,+ encoder_hidden_states=encoder_hidden_states,+ skip_mm=skip_mm,+ )+ # if pose_cond_fea is not None:+ # sample = sample + pose_cond_fea[block_count]+ # block_count += 1+ down_block_res_samples += res_samples++ if down_block_additional_residuals is not None:+ new_down_block_res_samples = ()++ for down_block_res_sample, down_block_additional_residual in zip(+ down_block_res_samples, down_block_additional_residuals+ ):+ down_block_res_sample = (+ down_block_res_sample + down_block_additional_residual+ )+ new_down_block_res_samples += (down_block_res_sample,)++ down_block_res_samples = new_down_block_res_samples++ # mid+ sample = self.mid_block(+ sample,+ emb,+ encoder_hidden_states=encoder_hidden_states,+ attention_mask=attention_mask,+ skip_mm=skip_mm,+ )++ if mid_block_additional_residual is not None:+ sample = sample + mid_block_additional_residual++ # up+ for i, upsample_block in enumerate(self.up_blocks):+ is_final_block = i == len(self.up_blocks) - 1++ res_samples = down_block_res_samples[-len(upsample_block.resnets) :]+ down_block_res_samples = down_block_res_samples[+ : -len(upsample_block.resnets)+ ]++ # if we have not reached the final block and need to forward the+ # upsample size, we do it here+ if not is_final_block and forward_upsample_size:+ upsample_size = down_block_res_samples[-1].shape[2:]++ if (+ hasattr(upsample_block, "has_cross_attention")+ and upsample_block.has_cross_attention+ ):+ sample = upsample_block(+ hidden_states=sample,+ temb=emb,+ res_hidden_states_tuple=res_samples,+ encoder_hidden_states=encoder_hidden_states,+ upsample_size=upsample_size,+ attention_mask=attention_mask,+ skip_mm=skip_mm,+ )+ else:+ sample = upsample_block(+ hidden_states=sample,+ temb=emb,+ res_hidden_states_tuple=res_samples,+ upsample_size=upsample_size,+ encoder_hidden_states=encoder_hidden_states,+ skip_mm=skip_mm,+ )++ # post-process+ sample = self.conv_norm_out(sample)+ sample = self.conv_act(sample)+ sample = self.conv_out(sample)++ if not return_dict:+ return (sample,)++ return UNet3DConditionOutput(sample=sample)++ @classmethod+ def from_pretrained_2d(+ cls,+ pretrained_model_path: PathLike,+ motion_module_path: PathLike,+ subfolder=None,+ unet_additional_kwargs=None,+ mm_zero_proj_out=False,+ ):+ pretrained_model_path = Path(pretrained_model_path)+ motion_module_path = Path(motion_module_path)+ if subfolder is not None:+ pretrained_model_path = pretrained_model_path.joinpath(subfolder)+ logger.info(+ f"loaded temporal unet's pretrained weights from {pretrained_model_path} ..."+ )++ config_file = pretrained_model_path / "config.json"+ if not (config_file.exists() and config_file.is_file()):+ raise RuntimeError(f"{config_file} does not exist or is not a file")++ unet_config = cls.load_config(config_file)+ unet_config["_class_name"] = cls.__name__+ unet_config["down_block_types"] = [+ "CrossAttnDownBlock3D",+ "CrossAttnDownBlock3D",+ "CrossAttnDownBlock3D",+ "DownBlock3D",+ ]+ unet_config["up_block_types"] = [+ "UpBlock3D",+ "CrossAttnUpBlock3D",+ "CrossAttnUpBlock3D",+ "CrossAttnUpBlock3D",+ ]+ unet_config["mid_block_type"] = "UNetMidBlock3DCrossAttn"++ model = cls.from_config(unet_config, **unet_additional_kwargs)+ # load the vanilla weights+ if pretrained_model_path.joinpath(SAFETENSORS_WEIGHTS_NAME).exists():+ logger.debug(+ f"loading safeTensors weights from {pretrained_model_path} ..."+ )+ state_dict = load_file(+ pretrained_model_path.joinpath(SAFETENSORS_WEIGHTS_NAME), device="cpu"+ )++ elif pretrained_model_path.joinpath(WEIGHTS_NAME).exists():+ logger.debug(f"loading weights from {pretrained_model_path} ...")+ state_dict = torch.load(+ pretrained_model_path.joinpath(WEIGHTS_NAME),+ map_location="cpu",+ weights_only=True,+ )+ else:+ raise FileNotFoundError(f"no weights file found in {pretrained_model_path}")++ # load the motion module weights+ if motion_module_path.exists() and motion_module_path.is_file():+ if motion_module_path.suffix.lower() in [".pth", ".pt", ".ckpt"]:+ logger.info(f"Load motion module params from {motion_module_path}")+ motion_state_dict = torch.load(+ motion_module_path, map_location="cpu", weights_only=True+ )+ elif motion_module_path.suffix.lower() == ".safetensors":+ motion_state_dict = load_file(motion_module_path, device="cpu")+ else:+ raise RuntimeError(+ f"unknown file format for motion module weights: {motion_module_path.suffix}"+ )++ motion_state_dict = {+ k.replace('motion_modules.', 'temporal_modules.'): v for k, v in motion_state_dict.items() if not "pos_encoder" in k+ }++ if mm_zero_proj_out:+ logger.info(f"Zero initialize proj_out layers in motion module...")+ new_motion_state_dict = OrderedDict()+ for k in motion_state_dict:+ if "proj_out" in k:+ continue+ new_motion_state_dict[k] = motion_state_dict[k]+ motion_state_dict = new_motion_state_dict++ # merge the state dicts+ state_dict.update(motion_state_dict)++ # load the weights into the model+ m, u = model.load_state_dict(state_dict, strict=False)+ logger.debug(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};")++ params = [+ p.numel() if "temporal_modules" in n else 0+ for n, p in model.named_parameters()+ ]+ mm_params = [+ p.numel() if "motion_modules" in n else 0+ for n, p in model.named_parameters()+ ]+ logger.info(+ f"Loaded {sum(mm_params) / 1e6}M-parameter motion module, Loaded {sum(params) / 1e6}M-parameter temporal module"+ )++ return modeldiff --git a/src/models/unet_3d_blocks.py b/src/models/unet_3d_blocks.pynew file mode 100644index 0000000..bdec28f--- /dev/null+++ b/src/models/unet_3d_blocks.py@@ -0,0 +1,1121 @@+import pdb+from typing import Dict, Optional+import torch+from torch import nn++from src.models.motion_module import get_motion_module++# from .motion_module import get_motion_module+from src.models.resnet import Downsample3D, ResnetBlock3D, Upsample3D+from .transformer_3d import Transformer3DModel+++def get_down_block(+ down_block_type,+ num_layers,+ in_channels,+ out_channels,+ temb_channels,+ add_downsample,+ resnet_eps,+ resnet_act_fn,+ attn_num_head_channels,+ resnet_groups=None,+ cross_attention_dim=None,+ downsample_padding=None,+ dual_cross_attention=False,+ use_linear_projection=False,+ only_cross_attention=False,+ upcast_attention=False,+ resnet_time_scale_shift="default",+ unet_use_cross_frame_attention=None,+ unet_use_temporal_attention=None,+ use_inflated_groupnorm=None,+ use_motion_module=None,+ motion_module_type=None,+ motion_module_kwargs=None,+ use_temporal_module=None,+ temporal_module_type=None,+ temporal_module_kwargs=None,+):+ down_block_type = (+ down_block_type[7:]+ if down_block_type.startswith("UNetRes")+ else down_block_type+ )+ if down_block_type == "DownBlock3D":+ return DownBlock3D(+ num_layers=num_layers,+ in_channels=in_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ add_downsample=add_downsample,+ resnet_eps=resnet_eps,+ resnet_act_fn=resnet_act_fn,+ resnet_groups=resnet_groups,+ downsample_padding=downsample_padding,+ resnet_time_scale_shift=resnet_time_scale_shift,+ use_inflated_groupnorm=use_inflated_groupnorm,+ use_motion_module=use_motion_module,+ motion_module_type=motion_module_type,+ motion_module_kwargs=motion_module_kwargs,+ use_temporal_module=use_temporal_module,+ temporal_module_type=temporal_module_type,+ temporal_module_kwargs=temporal_module_kwargs,+ )+ elif down_block_type == "CrossAttnDownBlock3D":+ if cross_attention_dim is None:+ raise ValueError(+ "cross_attention_dim must be specified for CrossAttnDownBlock3D"+ )+ return CrossAttnDownBlock3D(+ num_layers=num_layers,+ in_channels=in_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ add_downsample=add_downsample,+ resnet_eps=resnet_eps,+ resnet_act_fn=resnet_act_fn,+ resnet_groups=resnet_groups,+ downsample_padding=downsample_padding,+ cross_attention_dim=cross_attention_dim,+ attn_num_head_channels=attn_num_head_channels,+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention,+ upcast_attention=upcast_attention,+ resnet_time_scale_shift=resnet_time_scale_shift,+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,+ unet_use_temporal_attention=unet_use_temporal_attention,+ use_inflated_groupnorm=use_inflated_groupnorm,+ use_motion_module=use_motion_module,+ motion_module_type=motion_module_type,+ motion_module_kwargs=motion_module_kwargs,+ use_temporal_module=use_temporal_module,+ temporal_module_type=temporal_module_type,+ temporal_module_kwargs=temporal_module_kwargs,+ )+ raise ValueError(f"{down_block_type} does not exist.")+++def get_up_block(+ up_block_type,+ num_layers,+ in_channels,+ out_channels,+ prev_output_channel,+ temb_channels,+ add_upsample,+ resnet_eps,+ resnet_act_fn,+ attn_num_head_channels,+ resnet_groups=None,+ cross_attention_dim=None,+ dual_cross_attention=False,+ use_linear_projection=False,+ only_cross_attention=False,+ upcast_attention=False,+ resnet_time_scale_shift="default",+ unet_use_cross_frame_attention=None,+ unet_use_temporal_attention=None,+ use_inflated_groupnorm=None,+ use_motion_module=None,+ motion_module_type=None,+ motion_module_kwargs=None,+ use_temporal_module=None,+ temporal_module_type=None,+ temporal_module_kwargs=None,+):+ up_block_type = (+ up_block_type[7:] if up_block_type.startswith("UNetRes") else up_block_type+ )+ if up_block_type == "UpBlock3D":+ return UpBlock3D(+ num_layers=num_layers,+ in_channels=in_channels,+ out_channels=out_channels,+ prev_output_channel=prev_output_channel,+ temb_channels=temb_channels,+ add_upsample=add_upsample,+ resnet_eps=resnet_eps,+ resnet_act_fn=resnet_act_fn,+ resnet_groups=resnet_groups,+ resnet_time_scale_shift=resnet_time_scale_shift,+ use_inflated_groupnorm=use_inflated_groupnorm,+ use_motion_module=use_motion_module,+ motion_module_type=motion_module_type,+ motion_module_kwargs=motion_module_kwargs,+ use_temporal_module=use_temporal_module,+ temporal_module_type=temporal_module_type,+ temporal_module_kwargs=temporal_module_kwargs,+ )+ elif up_block_type == "CrossAttnUpBlock3D":+ if cross_attention_dim is None:+ raise ValueError(+ "cross_attention_dim must be specified for CrossAttnUpBlock3D"+ )+ return CrossAttnUpBlock3D(+ num_layers=num_layers,+ in_channels=in_channels,+ out_channels=out_channels,+ prev_output_channel=prev_output_channel,+ temb_channels=temb_channels,+ add_upsample=add_upsample,+ resnet_eps=resnet_eps,+ resnet_act_fn=resnet_act_fn,+ resnet_groups=resnet_groups,+ cross_attention_dim=cross_attention_dim,+ attn_num_head_channels=attn_num_head_channels,+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention,+ upcast_attention=upcast_attention,+ resnet_time_scale_shift=resnet_time_scale_shift,+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,+ unet_use_temporal_attention=unet_use_temporal_attention,+ use_inflated_groupnorm=use_inflated_groupnorm,+ use_motion_module=use_motion_module,+ motion_module_type=motion_module_type,+ motion_module_kwargs=motion_module_kwargs,+ use_temporal_module=use_temporal_module,+ temporal_module_type=temporal_module_type,+ temporal_module_kwargs=temporal_module_kwargs,+ )+ raise ValueError(f"{up_block_type} does not exist.")+++class UNetMidBlock3DCrossAttn(nn.Module):++ def __init__(+ self,+ in_channels: int,+ temb_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default",+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ attn_num_head_channels=1,+ output_scale_factor=1.0,+ cross_attention_dim=1280,+ dual_cross_attention=False,+ use_linear_projection=False,+ upcast_attention=False,+ unet_use_cross_frame_attention=None,+ unet_use_temporal_attention=None,+ use_inflated_groupnorm=None,+ use_motion_module=None,+ motion_module_type=None,+ motion_module_kwargs=None,+ use_temporal_module=None,+ temporal_module_type=None,+ temporal_module_kwargs=None,+ **transformer_kwargs,+ ):+ super().__init__()++ self.has_cross_attention = True+ self.attn_num_head_channels = attn_num_head_channels+ resnet_groups = (+ resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)+ )++ # there is always at least one resnet+ resnets = [+ ResnetBlock3D(+ in_channels=in_channels,+ out_channels=in_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ use_inflated_groupnorm=use_inflated_groupnorm,+ )+ ]+ attentions = []+ motion_modules = []++ for _ in range(num_layers):+ if dual_cross_attention:+ raise NotImplementedError+ attentions.append(+ Transformer3DModel(+ attn_num_head_channels,+ in_channels // attn_num_head_channels,+ in_channels=in_channels,+ num_layers=1,+ cross_attention_dim=cross_attention_dim,+ norm_num_groups=resnet_groups,+ use_linear_projection=use_linear_projection,+ upcast_attention=upcast_attention,+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,+ unet_use_temporal_attention=unet_use_temporal_attention,+ **transformer_kwargs+ )+ )+ motion_modules.append(+ get_motion_module(+ in_channels=in_channels,+ motion_module_type=motion_module_type,+ motion_module_kwargs=motion_module_kwargs,+ )+ if use_motion_module+ else None+ )+ resnets.append(+ ResnetBlock3D(+ in_channels=in_channels,+ out_channels=in_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ use_inflated_groupnorm=use_inflated_groupnorm,+ )+ )++ self.attentions = nn.ModuleList(attentions)+ self.resnets = nn.ModuleList(resnets)+ self.motion_modules = nn.ModuleList(motion_modules)+ self.temporal_modules = nn.ModuleList(+ [+ (+ get_motion_module(+ in_channels=in_channels,+ motion_module_type=temporal_module_type,+ motion_module_kwargs=temporal_module_kwargs,+ )+ if use_temporal_module+ else None+ )+ for _ in range(num_layers)+ ]+ )+ self.gradient_checkpointing = False++ def forward(+ self,+ hidden_states,+ temb=None,+ encoder_hidden_states=None,+ attention_mask=None,+ skip_mm=False,+ mid_reference=None,+ ):+ if isinstance(encoder_hidden_states, list):+ encoder_hidden_states, motion_hidden_states = encoder_hidden_states+ else:+ motion_hidden_states = encoder_hidden_states++ hidden_states = self.resnets[0](hidden_states, temb)+ for attn, resnet, motion_module, temporal_module in zip(+ self.attentions, self.resnets[1:], self.motion_modules, self.temporal_modules+ ):+ if self.training and self.gradient_checkpointing:+ def create_custom_forward(module, return_dict=None):+ def custom_forward(*inputs):+ if return_dict is not None:+ return module(*inputs, return_dict=return_dict)+ else:+ return module(*inputs)++ return custom_forward+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(attn, return_dict=False),+ hidden_states,+ encoder_hidden_states,+ )[0]+ if (motion_module is not None) and not skip_mm:+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(motion_module),+ hidden_states,+ temb,+ motion_hidden_states,+ )+ if (temporal_module is not None) and not skip_mm:+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(temporal_module),+ hidden_states.requires_grad_(),+ temb,+ None,+ )+ # hidden_states = (+ # temporal_module(hidden_states, temb, encoder_hidden_states=None)+ # if (temporal_module is not None) and not skip_mm+ # else hidden_states+ # )+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet), hidden_states, temb+ )+ else:+ hidden_states = attn(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ reference=mid_reference,+ ).sample+ hidden_states = (+ motion_module(+ hidden_states, temb, encoder_hidden_states=motion_hidden_states+ )+ if (motion_module is not None) and not skip_mm+ else hidden_states+ )+ hidden_states = (+ temporal_module(hidden_states, temb, encoder_hidden_states=None, debug=True)+ if (temporal_module is not None) and not skip_mm+ else hidden_states+ )++ hidden_states = resnet(hidden_states, temb)++ return hidden_states+++class CrossAttnDownBlock3D(nn.Module):++ def __init__(+ self,+ in_channels: int,+ out_channels: int,+ temb_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default",+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ attn_num_head_channels=1,+ cross_attention_dim=1280,+ output_scale_factor=1.0,+ downsample_padding=1,+ add_downsample=True,+ dual_cross_attention=False,+ use_linear_projection=False,+ only_cross_attention=False,+ upcast_attention=False,+ unet_use_cross_frame_attention=None,+ unet_use_temporal_attention=None,+ use_inflated_groupnorm=None,+ use_motion_module=None,+ motion_module_type=None,+ motion_module_kwargs=None,+ use_temporal_module=None,+ temporal_module_type=None,+ temporal_module_kwargs=None,+ **transformer_kwargs,+ ):+ super().__init__()+ resnets = []+ attentions = []+ motion_modules = []++ self.has_cross_attention = True+ self.attn_num_head_channels = attn_num_head_channels++ for i in range(num_layers):+ in_channels = in_channels if i == 0 else out_channels+ resnets.append(+ ResnetBlock3D(+ in_channels=in_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ use_inflated_groupnorm=use_inflated_groupnorm,+ )+ )+ if dual_cross_attention:+ raise NotImplementedError+ attentions.append(+ Transformer3DModel(+ attn_num_head_channels,+ out_channels // attn_num_head_channels,+ in_channels=out_channels,+ num_layers=1,+ cross_attention_dim=cross_attention_dim,+ norm_num_groups=resnet_groups,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention,+ upcast_attention=upcast_attention,+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,+ unet_use_temporal_attention=unet_use_temporal_attention,+ **transformer_kwargs,+ )+ )+ motion_modules.append(+ get_motion_module(+ in_channels=out_channels,+ motion_module_type=motion_module_type,+ motion_module_kwargs=motion_module_kwargs,+ )+ if use_motion_module+ else None+ )++ self.attentions = nn.ModuleList(attentions)+ self.resnets = nn.ModuleList(resnets)+ self.motion_modules = nn.ModuleList(motion_modules)+ self.temporal_modules = nn.ModuleList(+ [+ (+ get_motion_module(+ in_channels=out_channels,+ motion_module_type=temporal_module_type,+ motion_module_kwargs=temporal_module_kwargs,+ )+ if use_temporal_module+ else None+ )+ for _ in range(num_layers)+ ]+ )++ if add_downsample:+ self.downsamplers = nn.ModuleList(+ [+ Downsample3D(+ out_channels,+ use_conv=True,+ out_channels=out_channels,+ padding=downsample_padding,+ name="op",+ )+ ]+ )+ else:+ self.downsamplers = None++ self.gradient_checkpointing = False++ def forward(+ self,+ hidden_states,+ temb=None,+ encoder_hidden_states=None,+ attention_mask=None,+ skip_mm=False,+ down_reference=None,+ ):+ if isinstance(encoder_hidden_states, list):+ encoder_hidden_states, motion_hidden_states = encoder_hidden_states+ else:+ motion_hidden_states = encoder_hidden_states++ output_states = ()++ for i, (resnet, attn, motion_module, temporal_module) in enumerate(+ zip(self.resnets, self.attentions, self.motion_modules, self.temporal_modules)+ ):+ reference = None+ if not(down_reference is None):+ reference = down_reference[i]+ # self.gradient_checkpointing = False+ if self.training and self.gradient_checkpointing:++ def create_custom_forward(module, return_dict=None):+ def custom_forward(*inputs):+ if return_dict is not None:+ return module(*inputs, return_dict=return_dict)+ else:+ return module(*inputs)++ return custom_forward++ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet), hidden_states, temb+ )+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(attn, return_dict=False),+ hidden_states,+ encoder_hidden_states,+ )[0]++ # add motion module+ if (motion_module is not None) and not skip_mm:+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(motion_module),+ hidden_states,+ temb,+ motion_hidden_states,+ )+ if (temporal_module is not None) and not skip_mm:+ # hidden_states = torch.utils.checkpoint.checkpoint(+ # create_custom_forward(temporal_module),+ # hidden_states.requires_grad_(),+ # temb,+ # None,+ # )+ hidden_states = (+ temporal_module(hidden_states, temb, encoder_hidden_states=None)+ if (temporal_module is not None) and not skip_mm+ else hidden_states+ )++ else:+ hidden_states = resnet(hidden_states, temb)+ hidden_states = attn(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ reference=reference,+ ).sample++ # add motion module+ hidden_states = (+ motion_module(+ hidden_states, temb, encoder_hidden_states=motion_hidden_states+ )+ if (motion_module is not None) and not skip_mm+ else hidden_states+ )+ hidden_states = (+ temporal_module(hidden_states, temb, encoder_hidden_states=None, debug=True)+ if (temporal_module is not None) and not skip_mm+ else hidden_states+ )++ output_states += (hidden_states,)++ if self.downsamplers is not None:+ for downsampler in self.downsamplers:+ hidden_states = downsampler(hidden_states)++ output_states += (hidden_states,)++ return hidden_states, output_states+++class DownBlock3D(nn.Module):++ def __init__(+ self,+ in_channels: int,+ out_channels: int,+ temb_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default",+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ output_scale_factor=1.0,+ add_downsample=True,+ downsample_padding=1,+ use_inflated_groupnorm=None,+ use_motion_module=None,+ motion_module_type=None,+ motion_module_kwargs=None,+ use_temporal_module=None,+ temporal_module_type=None,+ temporal_module_kwargs=None,+ ):+ super().__init__()+ resnets = []+ motion_modules = []++ # use_motion_module = False+ for i in range(num_layers):+ in_channels = in_channels if i == 0 else out_channels+ resnets.append(+ ResnetBlock3D(+ in_channels=in_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ use_inflated_groupnorm=use_inflated_groupnorm,+ )+ )+ motion_modules.append(+ get_motion_module(+ in_channels=out_channels,+ motion_module_type=motion_module_type,+ motion_module_kwargs=motion_module_kwargs,+ )+ if use_motion_module+ else None+ )++ self.resnets = nn.ModuleList(resnets)+ self.motion_modules = nn.ModuleList(motion_modules)+ self.temporal_modules = nn.ModuleList(+ [+ (+ get_motion_module(+ in_channels=out_channels,+ motion_module_type=temporal_module_type,+ motion_module_kwargs=temporal_module_kwargs,+ )+ if use_temporal_module+ else None+ )+ for _ in range(num_layers)+ ]+ )++ if add_downsample:+ self.downsamplers = nn.ModuleList(+ [+ Downsample3D(+ out_channels,+ use_conv=True,+ out_channels=out_channels,+ padding=downsample_padding,+ name="op",+ )+ ]+ )+ else:+ self.downsamplers = None++ self.gradient_checkpointing = False++ def forward(self, hidden_states, temb=None, encoder_hidden_states=None, skip_mm=False):+ output_states = ()+ if isinstance(encoder_hidden_states, list):+ encoder_hidden_states, motion_hidden_states = encoder_hidden_states+ else:+ motion_hidden_states = encoder_hidden_states+ for resnet, motion_module, temporal_module in zip(+ self.resnets, self.motion_modules, self.temporal_modules+ ):+ # print(f"DownBlock3D {self.gradient_checkpointing = }")+ if self.training and self.gradient_checkpointing:++ def create_custom_forward(module):+ def custom_forward(*inputs):+ return module(*inputs)++ return custom_forward++ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet), hidden_states, temb+ )+ if (motion_module is not None) and not skip_mm:+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(motion_module),+ hidden_states,+ temb,+ motion_hidden_states,+ )++ if (temporal_module is not None) and not skip_mm:+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(temporal_module),+ hidden_states.requires_grad_(),+ temb,+ None,+ )+ else:+ hidden_states = resnet(hidden_states, temb)++ # add motion module+ hidden_states = (+ motion_module(+ hidden_states, temb, encoder_hidden_states=motion_hidden_states+ )+ if (motion_module is not None) and not skip_mm+ else hidden_states+ )+ hidden_states = (+ temporal_module(+ hidden_states, temb, encoder_hidden_states=None, debug=True+ )+ if (temporal_module is not None) and not skip_mm+ else hidden_states+ )++ output_states += (hidden_states,)++ if self.downsamplers is not None:+ for downsampler in self.downsamplers:+ hidden_states = downsampler(hidden_states)++ output_states += (hidden_states,)++ return hidden_states, output_states+++class CrossAttnUpBlock3D(nn.Module):++ def __init__(+ self,+ in_channels: int,+ out_channels: int,+ prev_output_channel: int,+ temb_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default",+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ attn_num_head_channels=1,+ cross_attention_dim=1280,+ output_scale_factor=1.0,+ add_upsample=True,+ dual_cross_attention=False,+ use_linear_projection=False,+ only_cross_attention=False,+ upcast_attention=False,+ unet_use_cross_frame_attention=None,+ unet_use_temporal_attention=None,+ use_motion_module=None,+ use_inflated_groupnorm=None,+ motion_module_type=None,+ motion_module_kwargs=None,+ use_temporal_module=None,+ temporal_module_type=None,+ temporal_module_kwargs=None,+ **transformer_kwargs,+ ):+ super().__init__()+ resnets = []+ attentions = []+ motion_modules = []++ self.has_cross_attention = True+ self.attn_num_head_channels = attn_num_head_channels++ for i in range(num_layers):+ res_skip_channels = in_channels if (i == num_layers - 1) else out_channels+ resnet_in_channels = prev_output_channel if i == 0 else out_channels++ resnets.append(+ ResnetBlock3D(+ in_channels=resnet_in_channels + res_skip_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ use_inflated_groupnorm=use_inflated_groupnorm,+ )+ )+ if dual_cross_attention:+ raise NotImplementedError+ attentions.append(+ Transformer3DModel(+ attn_num_head_channels,+ out_channels // attn_num_head_channels,+ in_channels=out_channels,+ num_layers=1,+ cross_attention_dim=cross_attention_dim,+ norm_num_groups=resnet_groups,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention,+ upcast_attention=upcast_attention,+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,+ unet_use_temporal_attention=unet_use_temporal_attention,+ **transformer_kwargs,+ )+ )+ motion_modules.append(+ get_motion_module(+ in_channels=out_channels,+ motion_module_type=motion_module_type,+ motion_module_kwargs=motion_module_kwargs,+ )+ if use_motion_module+ else None+ )++ self.attentions = nn.ModuleList(attentions)+ self.resnets = nn.ModuleList(resnets)+ self.motion_modules = nn.ModuleList(motion_modules)+ self.temporal_modules = nn.ModuleList(+ [+ (+ get_motion_module(+ in_channels=out_channels,+ motion_module_type=temporal_module_type,+ motion_module_kwargs=temporal_module_kwargs,+ )+ if use_temporal_module+ else None+ )+ for _ in range(num_layers)+ ]+ )++ if add_upsample:+ self.upsamplers = nn.ModuleList(+ [Upsample3D(out_channels, use_conv=True, out_channels=out_channels)]+ )+ else:+ self.upsamplers = None++ self.gradient_checkpointing = False++ def forward(+ self,+ hidden_states,+ res_hidden_states_tuple,+ temb=None,+ encoder_hidden_states=None,+ upsample_size=None,+ attention_mask=None,+ skip_mm=False,+ up_reference=None,+ ):+ if isinstance(encoder_hidden_states, list):+ encoder_hidden_states, motion_hidden_states = encoder_hidden_states+ else:+ motion_hidden_states = encoder_hidden_states+ for i, (resnet, attn, motion_module, temporal_module) in enumerate(+ zip(self.resnets, self.attentions, self.motion_modules, self.temporal_modules)+ ):+ reference = None+ if not(up_reference is None):+ reference = up_reference[i]+ # pop res hidden states+ res_hidden_states = res_hidden_states_tuple[-1]+ res_hidden_states_tuple = res_hidden_states_tuple[:-1]+ hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)++ if self.training and self.gradient_checkpointing:++ def create_custom_forward(module, return_dict=None):+ def custom_forward(*inputs):+ if return_dict is not None:+ return module(*inputs, return_dict=return_dict)+ else:+ return module(*inputs)++ return custom_forward++ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet), hidden_states, temb+ )+ # hidden_states = attn(+ # hidden_states,+ # encoder_hidden_states=encoder_hidden_states,+ # ).sample+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(attn, return_dict=False),+ hidden_states,+ encoder_hidden_states,+ )[0]+ if (motion_module is not None) and not skip_mm:+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(motion_module),+ hidden_states,+ temb,+ motion_hidden_states,+ )+ if (temporal_module is not None) and not skip_mm:+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(temporal_module),+ hidden_states.requires_grad_(),+ temb,+ None,+ )++ else:+ hidden_states = resnet(hidden_states, temb)+ hidden_states = attn(+ hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ reference=reference+ ).sample++ # add motion module+ hidden_states = (+ motion_module(+ hidden_states, temb, encoder_hidden_states=motion_hidden_states+ )+ if (motion_module is not None) and not skip_mm+ else hidden_states+ )++ # add temporal_module+ hidden_states = (+ temporal_module(hidden_states, temb, encoder_hidden_states=None, debug=True)+ if (temporal_module is not None) and not skip_mm+ else hidden_states+ )++ if self.upsamplers is not None:+ for upsampler in self.upsamplers:+ hidden_states = upsampler(hidden_states, upsample_size)++ return hidden_states+++class UpBlock3D(nn.Module):++ def __init__(+ self,+ in_channels: int,+ prev_output_channel: int,+ out_channels: int,+ temb_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default",+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ output_scale_factor=1.0,+ add_upsample=True,+ use_inflated_groupnorm=None,+ use_motion_module=None,+ motion_module_type=None,+ motion_module_kwargs=None,+ use_temporal_module=None,+ temporal_module_type=None,+ temporal_module_kwargs=None,+ ):+ super().__init__()+ resnets = []+ motion_modules = []++ # use_motion_module = False+ for i in range(num_layers):+ res_skip_channels = in_channels if (i == num_layers - 1) else out_channels+ resnet_in_channels = prev_output_channel if i == 0 else out_channels++ resnets.append(+ ResnetBlock3D(+ in_channels=resnet_in_channels + res_skip_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ use_inflated_groupnorm=use_inflated_groupnorm,+ )+ )+ motion_modules.append(+ get_motion_module(+ in_channels=out_channels,+ motion_module_type=motion_module_type,+ motion_module_kwargs=motion_module_kwargs,+ )+ if use_motion_module+ else None+ )++ self.resnets = nn.ModuleList(resnets)+ self.motion_modules = nn.ModuleList(motion_modules)+ self.temporal_modules = nn.ModuleList(+ [+ (+ get_motion_module(+ in_channels=out_channels,+ motion_module_type=temporal_module_type,+ motion_module_kwargs=temporal_module_kwargs,+ )+ if use_temporal_module+ else None+ )+ for _ in range(num_layers)+ ]+ )++ if add_upsample:+ self.upsamplers = nn.ModuleList(+ [Upsample3D(out_channels, use_conv=True, out_channels=out_channels)]+ )+ else:+ self.upsamplers = None++ self.gradient_checkpointing = False++ def forward(+ self,+ hidden_states,+ res_hidden_states_tuple,+ temb=None,+ upsample_size=None,+ encoder_hidden_states=None,+ skip_mm=False,+ ):+ if isinstance(encoder_hidden_states, list):+ encoder_hidden_states, motion_hidden_states = encoder_hidden_states+ else:+ motion_hidden_states = encoder_hidden_states+ for resnet, motion_module, temporal_module in zip(self.resnets, self.motion_modules, self.temporal_modules):+ # pop res hidden states+ res_hidden_states = res_hidden_states_tuple[-1]+ res_hidden_states_tuple = res_hidden_states_tuple[:-1]+ hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1)++ # print(f"UpBlock3D {self.gradient_checkpointing = }")+ if self.training and self.gradient_checkpointing:++ def create_custom_forward(module):+ def custom_forward(*inputs):+ return module(*inputs)++ return custom_forward++ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(resnet), hidden_states, temb+ )+ if (motion_module is not None) and not skip_mm:+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(motion_module),+ hidden_states,+ temb,+ motion_hidden_states,+ )+ if (temporal_module is not None) and not skip_mm:+ hidden_states = torch.utils.checkpoint.checkpoint(+ create_custom_forward(temporal_module),+ hidden_states.requires_grad_(),+ temb,+ None,+ )++ else:+ hidden_states = resnet(hidden_states, temb)+ hidden_states = (+ motion_module(+ hidden_states, temb, encoder_hidden_states=motion_hidden_states+ )+ if (motion_module is not None) and not skip_mm+ else hidden_states+ )+ hidden_states = (+ temporal_module(hidden_states, temb, encoder_hidden_states=None, debug=True)+ if (temporal_module is not None) and not skip_mm+ else hidden_states+ )++ if self.upsamplers is not None:+ for upsampler in self.upsamplers:+ hidden_states = upsampler(hidden_states, upsample_size)++ return hidden_statesdiff --git a/src/models/unet_3d_explicit_reference.py b/src/models/unet_3d_explicit_reference.pynew file mode 100644index 0000000..b51c208--- /dev/null+++ b/src/models/unet_3d_explicit_reference.py@@ -0,0 +1,744 @@+from collections import OrderedDict+from dataclasses import dataclass+import pdb+from os import PathLike+from pathlib import Path+from typing import Dict, List, Optional, Tuple, Union++import torch+import torch.nn as nn+import torch.utils.checkpoint+import torch.nn.functional as F+from diffusers.configuration_utils import ConfigMixin, register_to_config+from diffusers.models.attention_processor import AttentionProcessor+from diffusers.models.embeddings import TimestepEmbedding, Timesteps+from diffusers.models.modeling_utils import ModelMixin+from diffusers.utils import SAFETENSORS_WEIGHTS_NAME, WEIGHTS_NAME, BaseOutput, logging+from safetensors.torch import load_file++from .resnet import InflatedConv3d, InflatedGroupNorm+from .unet_3d_blocks import UNetMidBlock3DCrossAttn, get_down_block, get_up_block+from einops import rearrange++logger = logging.get_logger(__name__) # pylint: disable=invalid-name+++@dataclass+class UNet3DConditionOutput(BaseOutput):+ sample: torch.FloatTensor+++class UNet3DConditionModel(ModelMixin, ConfigMixin):+ _supports_gradient_checkpointing = True++ @register_to_config+ def __init__(+ self,+ sample_size: Optional[int] = None,+ in_channels: int = 4,+ out_channels: int = 4,+ center_input_sample: bool = False,+ flip_sin_to_cos: bool = True,+ freq_shift: int = 0,+ down_block_types: Tuple[str] = (+ "CrossAttnDownBlock3D",+ "CrossAttnDownBlock3D",+ "CrossAttnDownBlock3D",+ "DownBlock3D",+ ),+ mid_block_type: str = "UNetMidBlock3DCrossAttn",+ up_block_types: Tuple[str] = (+ "UpBlock3D",+ "CrossAttnUpBlock3D",+ "CrossAttnUpBlock3D",+ "CrossAttnUpBlock3D",+ ),+ only_cross_attention: Union[bool, Tuple[bool]] = False,+ block_out_channels: Tuple[int] = (320, 640, 1280, 1280),+ layers_per_block: int = 2,+ downsample_padding: int = 1,+ mid_block_scale_factor: float = 1,+ act_fn: str = "silu",+ norm_num_groups: int = 32,+ norm_eps: float = 1e-5,+ cross_attention_dim: int = 1280,+ attention_head_dim: Union[int, Tuple[int]] = 8,+ dual_cross_attention: bool = False,+ use_linear_projection: bool = False,+ class_embed_type: Optional[str] = None,+ num_class_embeds: Optional[int] = None,+ upcast_attention: bool = False,+ resnet_time_scale_shift: str = "default",+ use_inflated_groupnorm=False,+ # Additional+ use_motion_module=False,+ use_temporal_module=False,+ motion_module_resolutions=(1, 2, 4, 8),+ motion_module_mid_block=False,+ motion_module_decoder_only=False,+ motion_module_type=None,+ temporal_module_type=None,+ motion_module_kwargs={},+ temporal_module_kwargs={},+ unet_use_cross_frame_attention=None,+ unet_use_temporal_attention=None,+ ):+ super().__init__()++ self.sample_size = sample_size+ time_embed_dim = block_out_channels[0] * 4++ # input+ self.conv_in = InflatedConv3d(+ in_channels, block_out_channels[0], kernel_size=3, padding=(1, 1)+ )++ # time+ self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift)+ timestep_input_dim = block_out_channels[0]++ self.time_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim)++ # class embedding+ if class_embed_type is None and num_class_embeds is not None:+ self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim)+ elif class_embed_type == "timestep":+ self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim)+ elif class_embed_type == "identity":+ self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim)+ else:+ self.class_embedding = None++ self.down_blocks = nn.ModuleList([])+ self.mid_block = None+ self.up_blocks = nn.ModuleList([])++ if isinstance(only_cross_attention, bool):+ only_cross_attention = [only_cross_attention] * len(down_block_types)++ if isinstance(attention_head_dim, int):+ attention_head_dim = (attention_head_dim,) * len(down_block_types)++ # down+ output_channel = block_out_channels[0]+ for i, down_block_type in enumerate(down_block_types):+ res = 2**i+ input_channel = output_channel+ output_channel = block_out_channels[i]+ is_final_block = i == len(block_out_channels) - 1++ down_block = get_down_block(+ down_block_type,+ num_layers=layers_per_block,+ in_channels=input_channel,+ out_channels=output_channel,+ temb_channels=time_embed_dim,+ add_downsample=not is_final_block,+ resnet_eps=norm_eps,+ resnet_act_fn=act_fn,+ resnet_groups=norm_num_groups,+ cross_attention_dim=cross_attention_dim,+ attn_num_head_channels=attention_head_dim[i],+ downsample_padding=downsample_padding,+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention[i],+ upcast_attention=upcast_attention,+ resnet_time_scale_shift=resnet_time_scale_shift,+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,+ unet_use_temporal_attention=unet_use_temporal_attention,+ use_inflated_groupnorm=use_inflated_groupnorm,+ use_motion_module=use_motion_module+ and (res in motion_module_resolutions)+ and (not motion_module_decoder_only),+ use_temporal_module=use_temporal_module+ and (res in motion_module_resolutions)+ and (not motion_module_decoder_only),+ motion_module_type=motion_module_type,+ temporal_module_type=temporal_module_type,+ motion_module_kwargs=motion_module_kwargs,+ temporal_module_kwargs=temporal_module_kwargs+ )+ self.down_blocks.append(down_block)++ # mid+ if mid_block_type == "UNetMidBlock3DCrossAttn":+ self.mid_block = UNetMidBlock3DCrossAttn(+ in_channels=block_out_channels[-1],+ temb_channels=time_embed_dim,+ resnet_eps=norm_eps,+ resnet_act_fn=act_fn,+ output_scale_factor=mid_block_scale_factor,+ resnet_time_scale_shift=resnet_time_scale_shift,+ cross_attention_dim=cross_attention_dim,+ attn_num_head_channels=attention_head_dim[-1],+ resnet_groups=norm_num_groups,+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ upcast_attention=upcast_attention,+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,+ unet_use_temporal_attention=unet_use_temporal_attention,+ use_inflated_groupnorm=use_inflated_groupnorm,+ use_motion_module=use_motion_module and motion_module_mid_block,+ use_temporal_module=use_temporal_module and motion_module_mid_block,+ motion_module_type=motion_module_type,+ temporal_module_type=temporal_module_type,+ motion_module_kwargs=motion_module_kwargs,+ temporal_module_kwargs=temporal_module_kwargs,+ )+ else:+ raise ValueError(f"unknown mid_block_type : {mid_block_type}")++ # count how many layers upsample the videos+ self.num_upsamplers = 0++ # up+ reversed_block_out_channels = list(reversed(block_out_channels))+ reversed_attention_head_dim = list(reversed(attention_head_dim))+ only_cross_attention = list(reversed(only_cross_attention))+ output_channel = reversed_block_out_channels[0]+ for i, up_block_type in enumerate(up_block_types):+ res = 2 ** (3 - i)+ is_final_block = i == len(block_out_channels) - 1++ prev_output_channel = output_channel+ output_channel = reversed_block_out_channels[i]+ input_channel = reversed_block_out_channels[+ min(i + 1, len(block_out_channels) - 1)+ ]++ # add upsample block for all BUT final layer+ if not is_final_block:+ add_upsample = True+ self.num_upsamplers += 1+ else:+ add_upsample = False++ up_block = get_up_block(+ up_block_type,+ num_layers=layers_per_block + 1,+ in_channels=input_channel,+ out_channels=output_channel,+ prev_output_channel=prev_output_channel,+ temb_channels=time_embed_dim,+ add_upsample=add_upsample,+ resnet_eps=norm_eps,+ resnet_act_fn=act_fn,+ resnet_groups=norm_num_groups,+ cross_attention_dim=cross_attention_dim,+ attn_num_head_channels=reversed_attention_head_dim[i],+ dual_cross_attention=dual_cross_attention,+ use_linear_projection=use_linear_projection,+ only_cross_attention=only_cross_attention[i],+ upcast_attention=upcast_attention,+ resnet_time_scale_shift=resnet_time_scale_shift,+ unet_use_cross_frame_attention=unet_use_cross_frame_attention,+ unet_use_temporal_attention=unet_use_temporal_attention,+ use_inflated_groupnorm=use_inflated_groupnorm,+ use_motion_module=use_motion_module+ and (res in motion_module_resolutions),+ use_temporal_module=use_temporal_module+ and (res in motion_module_resolutions),+ motion_module_type=motion_module_type,+ temporal_module_type=temporal_module_type,+ motion_module_kwargs=motion_module_kwargs,+ temporal_module_kwargs=temporal_module_kwargs,+ )+ self.up_blocks.append(up_block)+ prev_output_channel = output_channel++ # out+ if use_inflated_groupnorm:+ self.conv_norm_out = InflatedGroupNorm(+ num_channels=block_out_channels[0],+ num_groups=norm_num_groups,+ eps=norm_eps,+ )+ else:+ self.conv_norm_out = nn.GroupNorm(+ num_channels=block_out_channels[0],+ num_groups=norm_num_groups,+ eps=norm_eps,+ )+ self.conv_act = nn.SiLU()+ self.conv_out = InflatedConv3d(+ block_out_channels[0], out_channels, kernel_size=3, padding=1+ )++ @property+ # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors+ def attn_processors(self) -> Dict[str, AttentionProcessor]:+ r"""+ Returns:+ `dict` of attention processors: A dictionary containing all attention processors used in the model with+ indexed by its weight name.+ """+ # set recursively+ processors = {}++ def fn_recursive_add_processors(+ name: str,+ module: torch.nn.Module,+ processors: Dict[str, AttentionProcessor],+ ):+ if hasattr(module, "set_processor"):+ processors[f"{name}.processor"] = module.processor++ for sub_name, child in module.named_children():+ if "temporal_transformer" not in sub_name:+ fn_recursive_add_processors(f"{name}.{sub_name}", child, processors)++ return processors++ for name, module in self.named_children():+ if "temporal_transformer" not in name:+ fn_recursive_add_processors(name, module, processors)++ return processors++ def set_attention_slice(self, slice_size):+ r"""+ Enable sliced attention computation.++ When this option is enabled, the attention module will split the input tensor in slices, to compute attention+ in several steps. This is useful to save some memory in exchange for a small speed decrease.++ Args:+ slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`):+ When `"auto"`, halves the input to the attention heads, so attention will be computed in two steps. If+ `"max"`, maxium amount of memory will be saved by running only one slice at a time. If a number is+ provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim`+ must be a multiple of `slice_size`.+ """+ sliceable_head_dims = []++ def fn_recursive_retrieve_slicable_dims(module: torch.nn.Module):+ if hasattr(module, "set_attention_slice"):+ sliceable_head_dims.append(module.sliceable_head_dim)++ for child in module.children():+ fn_recursive_retrieve_slicable_dims(child)++ # retrieve number of attention layers+ for module in self.children():+ fn_recursive_retrieve_slicable_dims(module)++ num_slicable_layers = len(sliceable_head_dims)++ if slice_size == "auto":+ # half the attention head size is usually a good trade-off between+ # speed and memory+ slice_size = [dim // 2 for dim in sliceable_head_dims]+ elif slice_size == "max":+ # make smallest slice possible+ slice_size = num_slicable_layers * [1]++ slice_size = (+ num_slicable_layers * [slice_size]+ if not isinstance(slice_size, list)+ else slice_size+ )++ if len(slice_size) != len(sliceable_head_dims):+ raise ValueError(+ f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different"+ f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}."+ )++ for i in range(len(slice_size)):+ size = slice_size[i]+ dim = sliceable_head_dims[i]+ if size is not None and size > dim:+ raise ValueError(f"size {size} has to be smaller or equal to {dim}.")++ # Recursively walk through all the children.+ # Any children which exposes the set_attention_slice method+ # gets the message+ def fn_recursive_set_attention_slice(+ module: torch.nn.Module, slice_size: List[int]+ ):+ if hasattr(module, "set_attention_slice"):+ module.set_attention_slice(slice_size.pop())++ for child in module.children():+ fn_recursive_set_attention_slice(child, slice_size)++ reversed_slice_size = list(reversed(slice_size))+ for module in self.children():+ fn_recursive_set_attention_slice(module, reversed_slice_size)++ def set_use_cross_frame_attention(self, value):++ def fn_recursive_set_use_cf_att(module: torch.nn.Module, value):+ if hasattr(module, "set_use_cross_frame_attention"):+ module.set_use_cross_frame_attention(value)++ for child in module.children():+ fn_recursive_set_use_cf_att(child, value)++ for module in self.children():+ fn_recursive_set_use_cf_att(module, value)++ def _set_gradient_checkpointing(self, module, value=False):+ if hasattr(module, "gradient_checkpointing"):+ module.gradient_checkpointing = value++ # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.set_attn_processor+ def set_attn_processor(+ self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]+ ):+ r"""+ Sets the attention processor to use to compute attention.++ Parameters:+ processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`):+ The instantiated processor class or a dictionary of processor classes that will be set as the processor+ for **all** `Attention` layers.++ If `processor` is a dict, the key needs to define the path to the corresponding cross attention+ processor. This is strongly recommended when setting trainable attention processors.++ """+ count = len(self.attn_processors.keys())++ if isinstance(processor, dict) and len(processor) != count:+ raise ValueError(+ f"A dict of processors was passed, but the number of processors {len(processor)} does not match the"+ f" number of attention layers: {count}. Please make sure to pass {count} processor classes."+ )++ def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor):+ if hasattr(module, "set_processor"):+ if not isinstance(processor, dict):+ module.set_processor(processor)+ else:+ module.set_processor(processor.pop(f"{name}.processor"))++ for sub_name, child in module.named_children():+ fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor)++ for name, module in self.named_children():+ fn_recursive_attn_processor(name, module, processor)++ def forward(+ self,+ sample: torch.FloatTensor,+ timestep: Union[torch.Tensor, float, int],+ encoder_hidden_states: torch.Tensor,+ pose_cond_fea : torch.Tensor,+ d00: torch.Tensor,+ d01: torch.Tensor,+ d10: torch.Tensor,+ d11: torch.Tensor,+ d20: torch.Tensor,+ d21: torch.Tensor,+ m: torch.Tensor,+ u10: torch.Tensor,+ u11: torch.Tensor,+ u12: torch.Tensor,+ u20: torch.Tensor,+ u21: torch.Tensor,+ u22: torch.Tensor,+ u30: torch.Tensor,+ u31: torch.Tensor,+ u32: torch.Tensor,+ class_labels: Optional[torch.Tensor] = None,+ attention_mask: Optional[torch.Tensor] = None,+ down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None,+ mid_block_additional_residual: Optional[torch.Tensor] = None,+ return_dict: bool = True,+ skip_mm: bool = False,+ ) -> Union[UNet3DConditionOutput, Tuple]:+ r"""+ Args:+ sample (`torch.FloatTensor`): (batch, channel, height, width) noisy inputs tensor+ timestep (`torch.FloatTensor` or `float` or `int`): (batch) timesteps+ encoder_hidden_states (`torch.FloatTensor`): (batch, sequence_length, feature_dim) encoder hidden states+ return_dict (`bool`, *optional*, defaults to `True`):+ Whether or not to return a [`models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain tuple.++ Returns:+ [`~models.unet_2d_condition.UNet2DConditionOutput`] or `tuple`:+ [`~models.unet_2d_condition.UNet2DConditionOutput`] if `return_dict` is True, otherwise a `tuple`. When+ returning a tuple, the first element is the sample tensor.+ """+ # By default samples have to be AT least a multiple of the overall upsampling factor.+ # The overall upsampling factor is equal to 2 ** (# num of upsampling layears).+ # However, the upsampling interpolation output size can be forced to fit any upsampling size+ # on the fly if necessary.+ default_overall_up_factor = 2**self.num_upsamplers++ # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor`+ forward_upsample_size = False+ upsample_size = None++ if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]):+ logger.info("Forward upsample size to force interpolation output size.")+ forward_upsample_size = True++ # prepare attention_mask+ if attention_mask is not None:+ attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0+ attention_mask = attention_mask.unsqueeze(1)++ # center input if necessary+ if self.config.center_input_sample:+ sample = 2 * sample - 1.0++ # time+ timesteps = timestep+ if not torch.is_tensor(timesteps):+ # This would be a good case for the `match` statement (Python 3.10+)+ is_mps = sample.device.type == "mps"+ if isinstance(timestep, float):+ dtype = torch.float32 if is_mps else torch.float64+ else:+ dtype = torch.int32 if is_mps else torch.int64+ timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device)+ elif len(timesteps.shape) == 0:+ timesteps = timesteps[None].to(sample.device)++ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML+ # try:+ # timesteps = timesteps.expand(sample.shape[0])+ # except Exception:+ # timesteps = timesteps.expand(sample.shape[0] * sample.shape[2])++ t_emb = self.time_proj(timesteps)++ # timesteps does not contain any weights and will always return f32 tensors+ # but time_embedding might actually be running in fp16. so we need to cast here.+ # there might be better ways to encapsulate this.+ t_emb = t_emb.to(dtype=self.dtype)+ emb = self.time_embedding(t_emb)+ if self.class_embedding is not None:+ if class_labels is None:+ raise ValueError(+ "class_labels should be provided when num_class_embeds > 0"+ )++ if self.config.class_embed_type == "timestep":+ class_labels = self.time_proj(class_labels)++ class_emb = self.class_embedding(class_labels).to(dtype=self.dtype)+ emb = emb + class_emb++ # pre-process+ sample = self.conv_in(sample)+ if pose_cond_fea is not None:+ sample = sample + pose_cond_fea++ # down+ down_block_res_samples = (sample,)+ for i, downsample_block in enumerate(self.down_blocks):+ down_reference = None+ if i==0:+ down_reference = [d00, d01]+ elif i==1:+ down_reference = [d10, d11]+ elif i==2:+ down_reference = [d20, d21]+ if (+ hasattr(downsample_block, "has_cross_attention")+ and downsample_block.has_cross_attention+ ):+ sample, res_samples = downsample_block(+ hidden_states=sample,+ temb=emb,+ encoder_hidden_states=encoder_hidden_states,+ attention_mask=attention_mask,+ skip_mm=skip_mm,+ down_reference=down_reference,+ )+ else:+ sample, res_samples = downsample_block(+ hidden_states=sample,+ temb=emb,+ encoder_hidden_states=encoder_hidden_states,+ skip_mm=skip_mm,+ )+ down_block_res_samples += res_samples++ if down_block_additional_residuals is not None:+ new_down_block_res_samples = ()++ for down_block_res_sample, down_block_additional_residual in zip(+ down_block_res_samples, down_block_additional_residuals+ ):+ down_block_res_sample = (+ down_block_res_sample + down_block_additional_residual+ )+ new_down_block_res_samples += (down_block_res_sample,)++ down_block_res_samples = new_down_block_res_samples++ # mid+ sample = self.mid_block(+ sample,+ emb,+ encoder_hidden_states=encoder_hidden_states,+ attention_mask=attention_mask,+ skip_mm=skip_mm,+ mid_reference=m,+ )++ if mid_block_additional_residual is not None:+ sample = sample + mid_block_additional_residual++ # up+ for i, upsample_block in enumerate(self.up_blocks):+ up_reference = None+ if i==1:+ up_reference = [u10, u11, u12]+ elif i==2:+ up_reference = [u20, u21, u22]+ elif i==3:+ up_reference = [u30, u31, u32]++ is_final_block = i == len(self.up_blocks) - 1++ res_samples = down_block_res_samples[-len(upsample_block.resnets) :]+ down_block_res_samples = down_block_res_samples[+ : -len(upsample_block.resnets)+ ]++ # if we have not reached the final block and need to forward the+ # upsample size, we do it here+ if not is_final_block and forward_upsample_size:+ upsample_size = down_block_res_samples[-1].shape[2:]++ if (+ hasattr(upsample_block, "has_cross_attention")+ and upsample_block.has_cross_attention+ ):+ sample = upsample_block(+ hidden_states=sample,+ temb=emb,+ res_hidden_states_tuple=res_samples,+ encoder_hidden_states=encoder_hidden_states,+ upsample_size=upsample_size,+ attention_mask=attention_mask,+ skip_mm=skip_mm,+ up_reference=up_reference,+ )+ else:+ sample = upsample_block(+ hidden_states=sample,+ temb=emb,+ res_hidden_states_tuple=res_samples,+ upsample_size=upsample_size,+ encoder_hidden_states=encoder_hidden_states,+ skip_mm=skip_mm,+ )++ # post-process+ sample = self.conv_norm_out(sample)+ sample = self.conv_act(sample)+ sample = self.conv_out(sample)++ return sample++ @classmethod+ def from_pretrained_2d(+ cls,+ pretrained_model_path: PathLike,+ motion_module_path: PathLike,+ subfolder=None,+ unet_additional_kwargs=None,+ mm_zero_proj_out=False,+ ):+ pretrained_model_path = Path(pretrained_model_path)+ motion_module_path = Path(motion_module_path)+ if subfolder is not None:+ pretrained_model_path = pretrained_model_path.joinpath(subfolder)+ logger.info(+ f"loaded temporal unet's pretrained weights from {pretrained_model_path} ..."+ )++ config_file = pretrained_model_path / "config.json"+ if not (config_file.exists() and config_file.is_file()):+ raise RuntimeError(f"{config_file} does not exist or is not a file")++ unet_config = cls.load_config(config_file)+ unet_config["_class_name"] = cls.__name__+ unet_config["down_block_types"] = [+ "CrossAttnDownBlock3D",+ "CrossAttnDownBlock3D",+ "CrossAttnDownBlock3D",+ "DownBlock3D",+ ]+ unet_config["up_block_types"] = [+ "UpBlock3D",+ "CrossAttnUpBlock3D",+ "CrossAttnUpBlock3D",+ "CrossAttnUpBlock3D",+ ]+ unet_config["mid_block_type"] = "UNetMidBlock3DCrossAttn"++ model = cls.from_config(unet_config, **unet_additional_kwargs)+ # load the vanilla weights+ if pretrained_model_path.joinpath(SAFETENSORS_WEIGHTS_NAME).exists():+ logger.debug(+ f"loading safeTensors weights from {pretrained_model_path} ..."+ )+ state_dict = load_file(+ pretrained_model_path.joinpath(SAFETENSORS_WEIGHTS_NAME), device="cpu"+ )++ elif pretrained_model_path.joinpath(WEIGHTS_NAME).exists():+ logger.debug(f"loading weights from {pretrained_model_path} ...")+ state_dict = torch.load(+ pretrained_model_path.joinpath(WEIGHTS_NAME),+ map_location="cpu",+ weights_only=True,+ )+ else:+ raise FileNotFoundError(f"no weights file found in {pretrained_model_path}")++ # load the motion module weights+ if motion_module_path.exists() and motion_module_path.is_file():+ if motion_module_path.suffix.lower() in [".pth", ".pt", ".ckpt"]:+ logger.info(f"Load motion module params from {motion_module_path}")+ motion_state_dict = torch.load(+ motion_module_path, map_location="cpu", weights_only=True+ )+ elif motion_module_path.suffix.lower() == ".safetensors":+ motion_state_dict = load_file(motion_module_path, device="cpu")+ else:+ raise RuntimeError(+ f"unknown file format for motion module weights: {motion_module_path.suffix}"+ )++ motion_state_dict = {+ k.replace('motion_modules.', 'temporal_modules.'): v for k, v in motion_state_dict.items() if not "pos_encoder" in k+ }++ if mm_zero_proj_out:+ logger.info(f"Zero initialize proj_out layers in motion module...")+ new_motion_state_dict = OrderedDict()+ for k in motion_state_dict:+ if "proj_out" in k:+ continue+ new_motion_state_dict[k] = motion_state_dict[k]+ motion_state_dict = new_motion_state_dict++ # merge the state dicts+ state_dict.update(motion_state_dict)++ # load the weights into the model+ m, u = model.load_state_dict(state_dict, strict=False)+ logger.debug(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};")++ params = [+ p.numel() if "temporal_modules" in n else 0+ for n, p in model.named_parameters()+ ]+ mm_params = [+ p.numel() if "motion_modules" in n else 0+ for n, p in model.named_parameters()+ ]+ logger.info(+ f"Loaded {sum(mm_params) / 1e6}M-parameter motion module, Loaded {sum(params) / 1e6}M-parameter temporal module"+ )++ return modeldiff --git a/src/pipelines/context.py b/src/pipelines/context.pynew file mode 100644index 0000000..543f09a--- /dev/null+++ b/src/pipelines/context.py@@ -0,0 +1,76 @@+from typing import Callable, List, Optional++import numpy as np+++def ordered_halving(val):+ bin_str = f"{val:064b}"+ bin_flip = bin_str[::-1]+ as_int = int(bin_flip, 2)++ return as_int / (1 << 64)+++def uniform(+ step: int = ...,+ num_steps: Optional[int] = None,+ num_frames: int = ...,+ context_size: Optional[int] = None,+ context_stride: int = 3,+ context_overlap: int = 4,+ closed_loop: bool = True,+ offset=0,+):+ if num_frames <= context_size:+ yield list(range(num_frames))+ return++ context_stride = min(+ context_stride, int(np.ceil(np.log2(num_frames / context_size))) + 1+ )++ for context_step in 1 << np.arange(context_stride):+ pad = int(round(num_frames * ordered_halving(step)))+ for j in range(+ int(ordered_halving(step) * context_step) + pad,+ num_frames + pad + (0 if closed_loop else -context_overlap),+ (context_size * context_step - context_overlap),+ ):+ yield [+ (e+offset) % num_frames+ for e in range(j, j + context_size * context_step, context_step)+ ]+++def get_context_scheduler(name: str) -> Callable:+ if name == "uniform":+ return uniform+ else:+ raise ValueError(f"Unknown context_overlap policy {name}")+++def get_total_steps(+ scheduler,+ timesteps: List[int],+ num_steps: Optional[int] = None,+ num_frames: int = ...,+ context_size: Optional[int] = None,+ context_stride: int = 3,+ context_overlap: int = 4,+ closed_loop: bool = True,+):+ return sum(+ len(+ list(+ scheduler(+ i,+ num_steps,+ num_frames,+ context_size,+ context_stride,+ context_overlap,+ )+ )+ )+ for i in range(len(timesteps))+ )diff --git a/src/pipelines/pipeline_pose2vid.py b/src/pipelines/pipeline_pose2vid.pynew file mode 100644index 0000000..fda15bc--- /dev/null+++ b/src/pipelines/pipeline_pose2vid.py@@ -0,0 +1,699 @@+import inspect+import math+from dataclasses import dataclass+from typing import Callable, List, Optional, Union+import random+import numpy as np+import torch+import torch.nn.functional as F+import torchvision.transforms as transforms+from diffusers import DiffusionPipeline+from diffusers import (+ AutoencoderKL,+ AutoencoderKLTemporalDecoder,+)+from diffusers.image_processor import VaeImageProcessor+from diffusers.schedulers import (+ DDIMScheduler,+ DPMSolverMultistepScheduler,+ EulerAncestralDiscreteScheduler,+ EulerDiscreteScheduler,+ LMSDiscreteScheduler,+ PNDMScheduler,+)+from diffusers.utils import BaseOutput, is_accelerate_available+from diffusers.utils.torch_utils import randn_tensor+from einops import rearrange+from tqdm import tqdm+from transformers import CLIPImageProcessor+from src.pipelines.utils import get_tensor_interpolation_method+from src.models.mutual_self_attention import ReferenceAttentionControl+from src.utils.util import draw_keypoints++@dataclass+class Pose2VideoPipelineOutput(BaseOutput):+ videos: Union[torch.Tensor, np.ndarray]+++class Pose2VideoPipeline(DiffusionPipeline):+ _optional_components = []++ def __init__(+ self,+ vae,+ # vae_tiny,+ image_encoder,+ reference_unet,+ denoising_unet,+ motion_encoder,+ pose_encoder,+ pose_guider,+ scheduler: Union[+ DDIMScheduler,+ PNDMScheduler,+ LMSDiscreteScheduler,+ EulerDiscreteScheduler,+ EulerAncestralDiscreteScheduler,+ DPMSolverMultistepScheduler,+ ],+ ):+ super().__init__()++ self.register_modules(+ vae=vae,+ # vae_tiny=vae_tiny,+ image_encoder=image_encoder,+ reference_unet=reference_unet,+ denoising_unet=denoising_unet,+ motion_encoder=motion_encoder,+ pose_encoder=pose_encoder,+ pose_guider=pose_guider,+ scheduler=scheduler,+ )+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)+ self.clip_image_processor = CLIPImageProcessor()+ self.ref_image_processor = VaeImageProcessor(+ vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True+ )+ self.cond_image_processor = VaeImageProcessor(+ vae_scale_factor=self.vae_scale_factor,+ do_convert_rgb=True,+ do_normalize=True,+ )++ def enable_vae_slicing(self):+ self.vae.enable_slicing()++ def disable_vae_slicing(self):+ self.vae.disable_slicing()++ def enable_sequential_cpu_offload(self, gpu_id=0):+ if is_accelerate_available():+ from accelerate import cpu_offload+ else:+ raise ImportError("Please install accelerate via `pip install accelerate`")++ device = torch.device(f"cuda:{gpu_id}")++ for cpu_offloaded_model in [self.unet, self.text_encoder, self.vae]:+ if cpu_offloaded_model is not None:+ cpu_offload(cpu_offloaded_model, device)++ @property+ def _execution_device(self):+ if self.device != torch.device("meta") or not hasattr(self.unet, "_hf_hook"):+ return self.device+ for module in self.unet.modules():+ if (+ hasattr(module, "_hf_hook")+ and hasattr(module._hf_hook, "execution_device")+ and module._hf_hook.execution_device is not None+ ):+ return torch.device(module._hf_hook.execution_device)+ return self.device++ def decode_latents_tiny(self, latents, decode_chunk_size=64):+ video_length = latents.shape[2]+ latents = rearrange(latents, "b c f h w -> (b f) c h w")+ video = []+ for frame_idx in tqdm(range(0, latents.shape[0], decode_chunk_size)):+ video.append(self.vae_tiny.decode(latents[frame_idx : frame_idx + decode_chunk_size]).sample)+ video = torch.cat(video)+ video = rearrange(video, "(b f) c h w -> b c f h w", f=video_length)+ video = (video / 2 + 0.5).clamp(0, 1)+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16+ video = video.cpu().float().numpy()+ return video++ def decode_latents(self, latents, decode_chunk_size=16):+ video_length = latents.shape[2]+ latents = 1 / 0.18215 * latents+ latents = rearrange(latents, "b c f h w -> (b f) c h w")+ video = []+ for frame_idx in range(0, latents.shape[0], decode_chunk_size):+ video.append(self.vae.decode(latents[frame_idx : frame_idx + decode_chunk_size]).sample)+ video = torch.cat(video)+ video = rearrange(video, "(b f) c h w -> b c f h w", f=video_length)+ video = (video / 2 + 0.5).clamp(0, 1)+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16+ video = video.cpu().float().numpy()+ return video++ def decode_latents_svd(+ self, latents: torch.FloatTensor, decode_chunk_size: int = 5+ ):+ video_length = latents.shape[2]+ latents = 1 / self.vae.config.scaling_factor * latents+ latents = rearrange(latents, "b c f h w -> (b f) c h w")++ # decode decode_chunk_size frames at a time to avoid OOM+ frames = []+ for i in range(0, latents.shape[0], decode_chunk_size):+ num_frames_in = latents[i : i + decode_chunk_size].shape[0]+ frame = self.vae.decode(+ latents[i : i + decode_chunk_size], num_frames_in+ ).sample+ frames.append(frame)+ video = torch.cat(frames, dim=0)+ video = rearrange(video, "(b f) c h w -> b c f h w", f=video_length)+ video = (video / 2 + 0.5).clamp(0, 1)+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16+ video = video.cpu().float().numpy()+ return video++ def prepare_extra_step_kwargs(self, generator, eta):+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.+ # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502+ # and should be between [0, 1]++ accepts_eta = "eta" in set(+ inspect.signature(self.scheduler.step).parameters.keys()+ )+ extra_step_kwargs = {}+ if accepts_eta:+ extra_step_kwargs["eta"] = eta++ # check if the scheduler accepts generator+ accepts_generator = "generator" in set(+ inspect.signature(self.scheduler.step).parameters.keys()+ )+ if accepts_generator:+ extra_step_kwargs["generator"] = generator+ return extra_step_kwargs++ def prepare_latents(+ self,+ batch_size,+ num_channels_latents,+ width,+ height,+ video_length,+ dtype,+ device,+ generator,+ latents=None,+ ):+ shape = (+ batch_size,+ num_channels_latents,+ video_length,+ height // self.vae_scale_factor,+ width // self.vae_scale_factor,+ )+ if isinstance(generator, list) and len(generator) != batch_size:+ raise ValueError(+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."+ )++ if latents is None:+ latents = randn_tensor(+ shape, generator=generator, device=device, dtype=dtype+ )+ else:+ latents = latents.to(device)++ # scale the initial noise by the standard deviation required by the scheduler+ latents = latents * self.scheduler.init_noise_sigma+ return latents++ def _encode_prompt(+ self,+ prompt,+ device,+ num_videos_per_prompt,+ do_classifier_free_guidance,+ negative_prompt,+ ):+ batch_size = len(prompt) if isinstance(prompt, list) else 1++ text_inputs = self.tokenizer(+ prompt,+ padding="max_length",+ max_length=self.tokenizer.model_max_length,+ truncation=True,+ return_tensors="pt",+ )+ text_input_ids = text_inputs.input_ids+ untruncated_ids = self.tokenizer(+ prompt, padding="longest", return_tensors="pt"+ ).input_ids++ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(+ text_input_ids, untruncated_ids+ ):+ removed_text = self.tokenizer.batch_decode(+ untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]+ )++ if (+ hasattr(self.text_encoder.config, "use_attention_mask")+ and self.text_encoder.config.use_attention_mask+ ):+ attention_mask = text_inputs.attention_mask.to(device)+ else:+ attention_mask = None++ text_embeddings = self.text_encoder(+ text_input_ids.to(device),+ attention_mask=attention_mask,+ )+ text_embeddings = text_embeddings[0]++ # duplicate text embeddings for each generation per prompt, using mps friendly method+ bs_embed, seq_len, _ = text_embeddings.shape+ text_embeddings = text_embeddings.repeat(1, num_videos_per_prompt, 1)+ text_embeddings = text_embeddings.view(+ bs_embed * num_videos_per_prompt, seq_len, -1+ )++ # get unconditional embeddings for classifier free guidance+ if do_classifier_free_guidance:+ uncond_tokens: List[str]+ if negative_prompt is None:+ uncond_tokens = [""] * batch_size+ elif type(prompt) is not type(negative_prompt):+ raise TypeError(+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="+ f" {type(prompt)}."+ )+ elif isinstance(negative_prompt, str):+ uncond_tokens = [negative_prompt]+ elif batch_size != len(negative_prompt):+ raise ValueError(+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"+ " the batch size of `prompt`."+ )+ else:+ uncond_tokens = negative_prompt++ max_length = text_input_ids.shape[-1]+ uncond_input = self.tokenizer(+ uncond_tokens,+ padding="max_length",+ max_length=max_length,+ truncation=True,+ return_tensors="pt",+ )++ if (+ hasattr(self.text_encoder.config, "use_attention_mask")+ and self.text_encoder.config.use_attention_mask+ ):+ attention_mask = uncond_input.attention_mask.to(device)+ else:+ attention_mask = None++ uncond_embeddings = self.text_encoder(+ uncond_input.input_ids.to(device),+ attention_mask=attention_mask,+ )+ uncond_embeddings = uncond_embeddings[0]++ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method+ seq_len = uncond_embeddings.shape[1]+ uncond_embeddings = uncond_embeddings.repeat(1, num_videos_per_prompt, 1)+ uncond_embeddings = uncond_embeddings.view(+ batch_size * num_videos_per_prompt, seq_len, -1+ )++ # For classifier free guidance, we need to do two forward passes.+ # Here we concatenate the unconditional and text embeddings into a single batch+ # to avoid doing two forward passes+ text_embeddings = torch.cat([uncond_embeddings, text_embeddings])++ return text_embeddings++ def downgrade_input(self, init_latents, generator, device, dtype):+ mask = (+ torch.rand(+ *init_latents.shape,+ generator=generator,+ device=device,+ )+ >= 0.5+ ).to(dtype=dtype)++ blur = transforms.GaussianBlur(kernel_size=(9, 9), sigma=(18, 18))+ # first_frame_latents = init_latents[:, :, 0:1, :, :].clone()+ video_length = init_latents.shape[2]+ init_latents = rearrange(init_latents, "b c f h w -> (b f) c h w")+ init_latents_blur = blur(init_latents)+ init_latents = rearrange(init_latents, "(b f) c h w -> b c f h w", f=video_length)+ init_latents_blur = rearrange(init_latents_blur, "(b f) c h w -> b c f h w", f=video_length)+ # init_latents = init_latents * mask + init_latents_blur * (1 - mask)+ # init_latents[:, :, 0:1, :, :] = first_frame_latents++ return init_latents_blur++ def interpolate_latents(+ self, latents: torch.Tensor, interpolation_factor: int, device+ ):+ if interpolation_factor < 2:+ return latents++ new_latents = torch.zeros(+ (+ latents.shape[0],+ latents.shape[1],+ ((latents.shape[2] - 1) * interpolation_factor) + 1,+ latents.shape[3],+ latents.shape[4],+ ),+ device=latents.device,+ dtype=latents.dtype,+ )++ org_video_length = latents.shape[2]+ rate = [i / interpolation_factor for i in range(interpolation_factor)][1:]++ new_index = 0++ v0 = None+ v1 = None++ for i0, i1 in zip(range(org_video_length), range(org_video_length)[1:]):+ v0 = latents[:, :, i0, :, :]+ v1 = latents[:, :, i1, :, :]++ new_latents[:, :, new_index, :, :] = v0+ new_index += 1++ for f in rate:+ v = get_tensor_interpolation_method()(+ v0.to(device=device), v1.to(device=device), f+ )+ new_latents[:, :, new_index, :, :] = v.to(latents.device)+ new_index += 1++ new_latents[:, :, new_index, :, :] = v1+ new_index += 1++ return new_latents++ def interpolate_tensors(self, a: torch.Tensor, b: torch.Tensor, num: int = 10) -> torch.Tensor:+ """+ Linear interpolation between tensors a and b.+ input shape: (B, 1, D1, D2, ...)+ output shape: (B, num, D1, D2, ...)+ """+ if a.shape != b.shape:+ raise ValueError(f"Shape mismatch: a.shape={a.shape}, b.shape={b.shape}")++ B, _, *rest = a.shape+ alphas = torch.linspace(0, 1, num, device=a.device, dtype=a.dtype)+ view_shape = (1, num) + (1,) * len(rest)+ alphas = alphas.view(view_shape) # (1, num, 1, 1, ...)++ # (B, num, D1, D2, ...)+ result = (1 - alphas) * a + alphas * b+ return result++ def calculate_dis(self, A, B, threshold=10.):+ """+ A: (b, f1, c1, c2) bank+ B: (b, f2, c1, c2) new data+ """++ A_flat = A.view(A.size(1), -1).clone()+ B_flat = B.view(B.size(1), -1).clone()++ dist = torch.cdist(B_flat.to(torch.float32), A_flat.to(torch.float32), p=2)++ min_dist, min_idx = dist.min(dim=1) # (f2,)++ idx_to_add = torch.nonzero(min_dist[:1] > threshold, as_tuple=False).squeeze(1).tolist()++ if len(idx_to_add) > 0:+ B_to_add = B[:, idx_to_add] # (1, k, c1, c2)+ A_new = torch.cat([A, B_to_add], dim=1) # (1, f1+k, c1, c2)+ add_flag = True+ else:+ A_new = A+ add_flag = False++ return add_flag, A_new++ @torch.no_grad()+ def __call__(+ self,+ tgt_images,+ ref_image,+ face_images,+ ref_face_image,+ width,+ height,+ video_length,+ num_inference_steps,+ guidance_scale,+ eta: float = 0.0,+ generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,+ output_type: Optional[str] = "tensor",+ return_dict: bool = True,+ temporal_window_size = 4,+ temporal_adaptive_step = 4,+ temporal_kv_cache=True,+ init_latents=None,+ **kwargs,+ ):+ assert num_inference_steps % temporal_adaptive_step == 0, "temporal_adaptive_step should be divisor of num_inference_steps"+ assert video_length % temporal_window_size == 0, "temporal_window_size should be divisor of video_length"+ # Default height and width to unet+ height = height or self.unet.config.sample_size * self.vae_scale_factor+ width = width or self.unet.config.sample_size * self.vae_scale_factor++ device = self._execution_device++ do_classifier_free_guidance = guidance_scale > 1.0++ # Prepare timesteps+ timesteps = torch.tensor([999, 666, 333, 0], device=device).long()+ self.scheduler.set_step_length(333)+ jump = num_inference_steps // temporal_adaptive_step+ windows = video_length // temporal_window_size++ batch_size = 1++ # Prepare clip image embeds+ clip_image = self.clip_image_processor.preprocess(+ ref_image.resize((224, 224)), return_tensors="pt"+ ).pixel_values+ clip_image_embeds = self.image_encoder(+ clip_image.to(device, dtype=self.image_encoder.dtype)+ ).image_embeds+ image_prompt_embeds = clip_image_embeds.unsqueeze(1)++ uncond_image_prompt_embeds = torch.zeros_like(image_prompt_embeds)++ if do_classifier_free_guidance:+ image_prompt_embeds = torch.cat(+ [uncond_image_prompt_embeds, image_prompt_embeds], dim=0+ )++ reference_control_writer = ReferenceAttentionControl(+ self.reference_unet,+ do_classifier_free_guidance=do_classifier_free_guidance,+ mode="write",+ batch_size=batch_size,+ fusion_blocks="full",+ )+ reference_control_reader = ReferenceAttentionControl(+ self.denoising_unet,+ do_classifier_free_guidance=do_classifier_free_guidance,+ mode="read",+ batch_size=batch_size,+ fusion_blocks="full",+ cache_kv=temporal_kv_cache,+ )++ # Prepare extra step kwargs.+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)++ # Prepare ref image latents+ ref_image_tensor = self.ref_image_processor.preprocess(+ ref_image, height=height, width=width+ ) # (bs, c, width, height)+ ref_image_tensor = ref_image_tensor.to(+ dtype=self.vae.dtype, device=self.vae.device+ )+ ref_image_latents = self.vae.encode(ref_image_tensor).latent_dist.mean+ ref_image_latents = ref_image_latents * 0.18215 # (b, 4, h, w)++ # initialize latents+ padding_num = (temporal_adaptive_step - 1) * temporal_window_size+ noisy_latents_first = []+ init_latents = ref_image_latents.unsqueeze(2).repeat(1, 1, padding_num, 1, 1)+ init_timesteps = reversed(timesteps[0::jump]).repeat_interleave(temporal_window_size, dim=0)+ noise = torch.randn_like(init_latents)+ noisy_latents_first.append(self.scheduler.add_noise(init_latents, noise, init_timesteps[:padding_num]))++ repeated_latents = ref_image_latents.unsqueeze(2).repeat(1, 1, video_length+padding_num, 1, 1)+ noise = torch.randn_like(repeated_latents)+ noisy_latents_first.append(self.scheduler.add_noise(repeated_latents, noise, timesteps[:1]))+ latents = torch.cat(noisy_latents_first, dim=2)++ # 3D implicit keypoints+ ref_cond_tensor = self.cond_image_processor.preprocess(+ ref_image, height=256, width=256+ ).to(device=device, dtype=self.pose_encoder.dtype) # (1, c, h, w)+ ref_cond_tensor = ref_cond_tensor / 2 + 0.5 # to [0, 1]++ tgt_cond_tensor = self.cond_image_processor.preprocess(+ tgt_images, height=256, width=256+ ).to(device=device, dtype=self.pose_encoder.dtype) # (1, c, h, w)+ tgt_cond_tensor = tgt_cond_tensor / 2 + 0.5 # to [0, 1]++ mot_bbox_param = self.pose_encoder.interpolate_kps(ref_cond_tensor, tgt_cond_tensor, num_interp=padding_num+1) # (t, c)+ keypoints = draw_keypoints(mot_bbox_param, device=device).unsqueeze(2)+ keypoints = rearrange(keypoints, 'f c b h w -> b c f h w')+ keypoints = keypoints.to(device=device, dtype=self.pose_guider.dtype)++ pose_feas = []+ for i in range(0, keypoints.shape[2], 256):+ pose_fea = self.pose_guider(keypoints[:,:, i:i+256, :, :]) # (b, c, f)+ pose_feas.append(pose_fea)+ pose_feas = torch.cat(pose_feas, dim=2)++ # motion embeddings+ face_cond_tensor = self.cond_image_processor.preprocess(+ face_images, height=224, width=224+ ).transpose(0, 1)+ face_cond_tensor = face_cond_tensor.unsqueeze(0) # (1, c, t, h, w)+ face_cond_tensor = face_cond_tensor.to(+ device=device, dtype=self.motion_encoder.dtype+ )++ motion_hidden_states = []+ for i in range(0, face_cond_tensor.shape[2], 256):+ motion_hidden_state = self.motion_encoder(face_cond_tensor[:,:, i:i+256, :, :]) # [b, c]+ motion_hidden_states.append(motion_hidden_state)+ motion_hidden_states = torch.cat(motion_hidden_states, dim=1) # [b, f, c]++ ref_face_cond_tensor = self.cond_image_processor.preprocess(+ ref_face_image, height=224, width=224+ ).to(device=device, dtype=self.motion_encoder.dtype)+ neg_motion_hidden_states = self.motion_encoder(ref_face_cond_tensor.unsqueeze(2)) # [b, c]++ if do_classifier_free_guidance:+ ref_motion_emb = mot_bbox_param[:1]+ ref_keypoints = draw_keypoints(ref_motion_emb).unsqueeze(2)+ ref_keypoints = ref_keypoints.to(device=device, dtype=self.pose_guider.dtype)+ ref_pose_fea = self.pose_guider(ref_keypoints)++ init_motion_hidden_states = self.interpolate_tensors(neg_motion_hidden_states, motion_hidden_states[:,:1], num=padding_num+1)[:,:-1]+ motion_hidden_states = torch.cat([init_motion_hidden_states, motion_hidden_states, motion_hidden_states[:,-1:].repeat(1, padding_num, 1, 1)], dim=1)+ pose_feas = torch.cat([pose_feas, pose_feas[:,:,-1:].repeat(1,1,padding_num,1,1)], dim=2)++ motion_bank = neg_motion_hidden_states++ # denoising loop+ with self.progress_bar(total=windows + temporal_adaptive_step - 1) as progress_bar:+ self.reference_unet(+ ref_image_latents.repeat((2 if do_classifier_free_guidance else 1), 1, 1, 1),+ torch.zeros((batch_size,),dtype=torch.float32,device=ref_image_latents.device),+ encoder_hidden_states=image_prompt_embeds,+ return_dict=False,+ )+ reference_control_reader.update(reference_control_writer)++ for i in range(windows + temporal_adaptive_step - 1):+ l = i * temporal_window_size+ r = (i + temporal_adaptive_step) * temporal_window_size++ motion_hidden_state = motion_hidden_states[:, l:r]+ pose_fea = pose_feas[:, :, l:r]++ add_flag = False+ if l > temporal_adaptive_step * temporal_window_size * 2 and motion_bank.shape[1] < 5:+ add_flag, motion_bank = self.calculate_dis(motion_bank, motion_hidden_state, threshold=17.)++ if do_classifier_free_guidance:+ motion_hidden_state = torch.cat(+ [+ neg_motion_hidden_states.unsqueeze(1).expand_as(motion_hidden_state),+ motion_hidden_state,+ ],+ dim=0,+ )++ pose_fea = torch.cat(+ [+ ref_pose_fea.expand_as(pose_fea),+ pose_fea,+ ],+ dim=0,+ )++ for j in range(jump):+ latent = latents[:,:,l:r,:,:]+ latent_model_input = (torch.cat([latent] * 2) if do_classifier_free_guidance else latent)+ ut = reversed(timesteps[j::jump]).repeat_interleave(temporal_window_size, dim=0)+ ut = torch.stack([ut] * batch_size * (2 if do_classifier_free_guidance else 1)).to(device)+ ut = rearrange(ut, 'b f -> (b f)')++ noise_pred = self.denoising_unet(+ latent_model_input,+ ut,+ encoder_hidden_states=[+ image_prompt_embeds,+ motion_hidden_state,+ ],+ pose_cond_fea=pose_fea,+ return_dict=False,+ )[0]++ if do_classifier_free_guidance:+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)+ noise_pred = noise_pred_uncond + guidance_scale * (+ noise_pred_text - noise_pred_uncond+ )++ clip_length = noise_pred.shape[2]+ mid_noise_pred = rearrange(noise_pred, 'b c f h w -> (b f) c h w')+ mid_latents = rearrange(latents[:,:,l:r,:,:], 'b c f h w -> (b f) c h w')+ ts = ut.chunk(2)[0] if do_classifier_free_guidance else ut+ mid_latents, pred_original_sample = self.scheduler.step(+ mid_noise_pred, ts, mid_latents, **extra_step_kwargs, return_dict=False+ )+ mid_latents = rearrange(mid_latents, '(b f) c h w -> b c f h w', f=clip_length)+ pred_original_sample = rearrange(pred_original_sample, '(b f) c h w -> b c f h w', f=clip_length)+ mid_latents = torch.cat([+ pred_original_sample[:,:,:temporal_window_size],+ mid_latents[:,:,temporal_window_size:]], dim=2)+ latents[:,:,l:r,:,:] = mid_latents++ # history keyframe mechanism+ if add_flag:+ reference_control_writer.clear()+ self.reference_unet(+ pred_original_sample[:,:,0].repeat(+ (2 if do_classifier_free_guidance else 1), 1, 1, 1+ ).to(self.reference_unet.dtype),+ torch.zeros((batch_size,),dtype=torch.float32,device=ref_image_latents.device),+ encoder_hidden_states=image_prompt_embeds,+ return_dict=False,+ )+ reference_control_reader.update_hkf(reference_control_writer)++ progress_bar.update()+ reference_control_reader.clear()+ reference_control_writer.clear()++ latents = latents[:, :, padding_num:video_length+padding_num, :, :]+ # Post-processing+ if isinstance(self.vae, AutoencoderKL):+ # images = self.decode_latents_tiny(latents) # (b, c, f, h, w)+ images = self.decode_latents(latents) # (b, c, f, h, w)+ elif isinstance(self.vae, AutoencoderKLTemporalDecoder):+ images = self.decode_latents_svd(latents)++ # Convert to tensor+ if output_type == "tensor":+ images = torch.from_numpy(images)++ if not return_dict:+ return images++ return Pose2VideoPipelineOutput(videos=images)diff --git a/src/pipelines/utils.py b/src/pipelines/utils.pynew file mode 100644index 0000000..3cd5076--- /dev/null+++ b/src/pipelines/utils.py@@ -0,0 +1,29 @@+import torch++tensor_interpolation = None+++def get_tensor_interpolation_method():+ return tensor_interpolation+++def set_tensor_interpolation_method(is_slerp):+ global tensor_interpolation+ tensor_interpolation = slerp if is_slerp else linear+++def linear(v1, v2, t):+ return (1.0 - t) * v1 + t * v2+++def slerp(+ v0: torch.Tensor, v1: torch.Tensor, t: float, DOT_THRESHOLD: float = 0.9995+) -> torch.Tensor:+ u0 = v0 / v0.norm()+ u1 = v1 / v1.norm()+ dot = (u0 * u1).sum()+ if dot.abs() > DOT_THRESHOLD:+ # logger.info(f'warning: v0 and v1 close to parallel, using linear interpolation instead.')+ return (1.0 - t) * v0 + t * v1+ omega = dot.acos()+ return (((1.0 - t) * omega).sin() * v0 + (t * omega).sin() * v1) / omega.sin()diff --git a/src/scheduler/scheduler_ddim.py b/src/scheduler/scheduler_ddim.pynew file mode 100644index 0000000..cc526d8--- /dev/null+++ b/src/scheduler/scheduler_ddim.py@@ -0,0 +1,550 @@+# Copyright 2023 Stanford University Team and The HuggingFace Team. All rights reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion+# and https://github.com/hojonathanho/diffusion++import math+from dataclasses import dataclass+from typing import List, Optional, Tuple, Union++from einops import rearrange+import numpy as np+import torch++from diffusers.configuration_utils import ConfigMixin, register_to_config+from diffusers.utils import BaseOutput+from diffusers.utils.torch_utils import randn_tensor+from diffusers.schedulers.scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin+++@dataclass+# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->DDIM+class DDIMSchedulerOutput(BaseOutput):+ """+ Output class for the scheduler's `step` function output.++ Args:+ prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):+ Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the+ denoising loop.+ pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):+ The predicted denoised sample `(x_{0})` based on the model output from the current timestep.+ `pred_original_sample` can be used to preview progress or for guidance.+ """++ prev_sample: torch.FloatTensor+ pred_original_sample: Optional[torch.FloatTensor] = None+++# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar+def betas_for_alpha_bar(+ num_diffusion_timesteps,+ max_beta=0.999,+ alpha_transform_type="cosine",+):+ """+ Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of+ (1-beta) over time from t = [0,1].++ Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up+ to that part of the diffusion process.+++ Args:+ num_diffusion_timesteps (`int`): the number of betas to produce.+ max_beta (`float`): the maximum beta to use; use values lower than 1 to+ prevent singularities.+ alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar.+ Choose from `cosine` or `exp`++ Returns:+ betas (`np.ndarray`): the betas used by the scheduler to step the model outputs+ """+ if alpha_transform_type == "cosine":++ def alpha_bar_fn(t):+ return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2++ elif alpha_transform_type == "exp":++ def alpha_bar_fn(t):+ return math.exp(t * -12.0)++ else:+ raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}")++ betas = []+ for i in range(num_diffusion_timesteps):+ t1 = i / num_diffusion_timesteps+ t2 = (i + 1) / num_diffusion_timesteps+ betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta))+ return torch.tensor(betas, dtype=torch.float32)+++def rescale_zero_terminal_snr(betas):+ """+ Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1)+++ Args:+ betas (`torch.FloatTensor`):+ the betas that the scheduler is being initialized with.++ Returns:+ `torch.FloatTensor`: rescaled betas with zero terminal SNR+ """+ # Convert betas to alphas_bar_sqrt+ alphas = 1.0 - betas+ alphas_cumprod = torch.cumprod(alphas, dim=0)+ alphas_bar_sqrt = alphas_cumprod.sqrt()++ # Store old values.+ alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone()+ alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone()++ # Shift so the last timestep is zero.+ alphas_bar_sqrt -= alphas_bar_sqrt_T++ # Scale so the first timestep is back to the old value.+ alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T)++ # Convert alphas_bar_sqrt to betas+ alphas_bar = alphas_bar_sqrt**2 # Revert sqrt+ alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod+ alphas = torch.cat([alphas_bar[0:1], alphas])+ betas = 1 - alphas++ return betas+++class DDIMScheduler(SchedulerMixin, ConfigMixin):+ """+ `DDIMScheduler` extends the denoising procedure introduced in denoising diffusion probabilistic models (DDPMs) with+ non-Markovian guidance.++ This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic+ methods the library implements for all schedulers such as loading and saving.++ Args:+ num_train_timesteps (`int`, defaults to 1000):+ The number of diffusion steps to train the model.+ beta_start (`float`, defaults to 0.0001):+ The starting `beta` value of inference.+ beta_end (`float`, defaults to 0.02):+ The final `beta` value.+ beta_schedule (`str`, defaults to `"linear"`):+ The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from+ `linear`, `scaled_linear`, or `squaredcos_cap_v2`.+ trained_betas (`np.ndarray`, *optional*):+ Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`.+ clip_sample (`bool`, defaults to `True`):+ Clip the predicted sample for numerical stability.+ clip_sample_range (`float`, defaults to 1.0):+ The maximum magnitude for sample clipping. Valid only when `clip_sample=True`.+ set_alpha_to_one (`bool`, defaults to `True`):+ Each diffusion step uses the alphas product value at that step and at the previous one. For the final step+ there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`,+ otherwise it uses the alpha value at step 0.+ steps_offset (`int`, defaults to 0):+ An offset added to the inference steps. You can use a combination of `offset=1` and+ `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable+ Diffusion.+ prediction_type (`str`, defaults to `epsilon`, *optional*):+ Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process),+ `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen+ Video](https://imagen.research.google/video/paper.pdf) paper).+ thresholding (`bool`, defaults to `False`):+ Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such+ as Stable Diffusion.+ dynamic_thresholding_ratio (`float`, defaults to 0.995):+ The ratio for the dynamic thresholding method. Valid only when `thresholding=True`.+ sample_max_value (`float`, defaults to 1.0):+ The threshold value for dynamic thresholding. Valid only when `thresholding=True`.+ timestep_spacing (`str`, defaults to `"leading"`):+ The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and+ Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.+ rescale_betas_zero_snr (`bool`, defaults to `False`):+ Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and+ dark samples instead of limiting it to samples with medium brightness. Loosely related to+ [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506).+ """++ _compatibles = [e.name for e in KarrasDiffusionSchedulers]+ order = 1++ @register_to_config+ def __init__(+ self,+ num_train_timesteps: int = 1000,+ beta_start: float = 0.0001,+ beta_end: float = 0.02,+ beta_schedule: str = "linear",+ trained_betas: Optional[Union[np.ndarray, List[float]]] = None,+ clip_sample: bool = True,+ set_alpha_to_one: bool = True,+ steps_offset: int = 0,+ prediction_type: str = "epsilon",+ thresholding: bool = False,+ dynamic_thresholding_ratio: float = 0.995,+ clip_sample_range: float = 1.0,+ sample_max_value: float = 1.0,+ timestep_spacing: str = "leading",+ rescale_betas_zero_snr: bool = False,+ ):+ if trained_betas is not None:+ self.betas = torch.tensor(trained_betas, dtype=torch.float32)+ elif beta_schedule == "linear":+ self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32)+ elif beta_schedule == "scaled_linear":+ # this schedule is very specific to the latent diffusion model.+ self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2+ elif beta_schedule == "squaredcos_cap_v2":+ # Glide cosine schedule+ self.betas = betas_for_alpha_bar(num_train_timesteps)+ else:+ raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}")++ # Rescale for zero SNR+ if rescale_betas_zero_snr:+ self.betas = rescale_zero_terminal_snr(self.betas)++ self.alphas = 1.0 - self.betas+ self.alphas_cumprod = torch.cumprod(self.alphas, dim=0)++ # At every step in ddim, we are looking into the previous alphas_cumprod+ # For the final step, there is no previous alphas_cumprod because we are already at 0+ # `set_alpha_to_one` decides whether we set this parameter simply to one or+ # whether we use the final alpha of the "non-previous" one.+ self.final_alpha_cumprod = torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0]++ # standard deviation of the initial noise distribution+ self.init_noise_sigma = 1.0++ # setable values+ self.num_inference_steps = None+ self.step_length = None+ self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy().astype(np.int64))++ def to(self, device):+ self.betas = self.betas.to(device)+ self.alphas = self.alphas.to(device)+ self.alphas_cumprod = self.alphas_cumprod.to(device)+ self.final_alpha_cumprod = self.final_alpha_cumprod.to(device)++ def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor:+ """+ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the+ current timestep.++ Args:+ sample (`torch.FloatTensor`):+ The input sample.+ timestep (`int`, *optional*):+ The current timestep in the diffusion chain.++ Returns:+ `torch.FloatTensor`:+ A scaled input sample.+ """+ return sample++ def _get_variance(self, alpha_prod_t, alpha_prod_t_prev):+ beta_prod_t = 1 - alpha_prod_t+ beta_prod_t_prev = 1 - alpha_prod_t_prev++ variance = (beta_prod_t_prev / beta_prod_t) * (1 - alpha_prod_t / alpha_prod_t_prev)++ return variance++ # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample+ def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor:+ """+ "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the+ prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by+ s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing+ pixels from saturation at each step. We find that dynamic thresholding results in significantly better+ photorealism as well as better image-text alignment, especially when using very large guidance weights."++ https://arxiv.org/abs/2205.11487+ """+ dtype = sample.dtype+ batch_size, channels, *remaining_dims = sample.shape++ if dtype not in (torch.float32, torch.float64):+ sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half++ # Flatten sample for doing quantile calculation along each image+ sample = sample.reshape(batch_size, channels * np.prod(remaining_dims))++ abs_sample = sample.abs() # "a certain percentile absolute pixel value"++ s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1)+ s = torch.clamp(+ s, min=1, max=self.config.sample_max_value+ ) # When clamped to min=1, equivalent to standard clipping to [-1, 1]+ s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0+ sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s"++ sample = sample.reshape(batch_size, channels, *remaining_dims)+ sample = sample.to(dtype)++ return sample++ def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None, num_train_timesteps = None):+ """+ Sets the discrete timesteps used for the diffusion chain (to be run before inference).++ Args:+ num_inference_steps (`int`):+ The number of diffusion steps used when generating samples with a pre-trained model.+ """+ if num_inference_steps > self.config.num_train_timesteps:+ raise ValueError(+ f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:"+ f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle"+ f" maximal {self.config.num_train_timesteps} timesteps."+ )++ self.num_inference_steps = num_inference_steps++ # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891+ if self.config.timestep_spacing == "linspace":+ timesteps = (+ np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps)+ .round()[::-1]+ .copy()+ .astype(np.int64)+ )+ elif self.config.timestep_spacing == "leading":+ step_ratio = self.config.num_train_timesteps // self.num_inference_steps+ # creates integer timesteps by multiplying by ratio+ # casting to int to avoid issues when num_inference_step is power of 3+ timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.int64)+ timesteps += self.config.steps_offset+ elif self.config.timestep_spacing == "trailing":+ step_ratio = self.config.num_train_timesteps / self.num_inference_steps+ # creates integer timesteps by multiplying by ratio+ # casting to int to avoid issues when num_inference_step is power of 3+ timesteps = np.round(np.arange(self.config.num_train_timesteps, 0, -step_ratio)).astype(np.int64)+ timesteps -= 1+ else:+ raise ValueError(+ f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'leading' or 'trailing'."+ )++ self.timesteps = torch.from_numpy(timesteps).to(device)++ def set_step_length(self, step_length):+ self.step_length = step_length++ def step(+ self,+ model_output: torch.FloatTensor,+ timestep: int,+ sample: torch.FloatTensor,+ eta: float = 0.0,+ use_clipped_model_output: bool = False,+ generator=None,+ variance_noise: Optional[torch.FloatTensor] = None,+ return_dict: bool = True,+ ) -> Union[DDIMSchedulerOutput, Tuple]:+ """+ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion+ process from the learned model outputs (most often the predicted noise).++ Args:+ model_output (`torch.FloatTensor`):+ The direct output from learned diffusion model.+ timestep (`float`):+ The current discrete timestep in the diffusion chain.+ sample (`torch.FloatTensor`):+ A current instance of a sample created by the diffusion process.+ eta (`float`):+ The weight of noise for added noise in diffusion step.+ use_clipped_model_output (`bool`, defaults to `False`):+ If `True`, computes "corrected" `model_output` from the clipped predicted original sample. Necessary+ because predicted original sample is clipped to [-1, 1] when `self.config.clip_sample` is `True`. If no+ clipping has happened, "corrected" `model_output` would coincide with the one provided as input and+ `use_clipped_model_output` has no effect.+ generator (`torch.Generator`, *optional*):+ A random number generator.+ variance_noise (`torch.FloatTensor`):+ Alternative to generating noise with `generator` by directly providing the noise for the variance+ itself. Useful for methods such as [`CycleDiffusion`].+ return_dict (`bool`, *optional*, defaults to `True`):+ Whether or not to return a [`~schedulers.scheduling_ddim.DDIMSchedulerOutput`] or `tuple`.++ Returns:+ [`~schedulers.scheduling_utils.DDIMSchedulerOutput`] or `tuple`:+ If return_dict is `True`, [`~schedulers.scheduling_ddim.DDIMSchedulerOutput`] is returned, otherwise a+ tuple is returned where the first element is the sample tensor.++ """+ if self.num_inference_steps is None and self.step_length is None:+ raise ValueError(+ "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler"+ )++ # See formulas (12) and (16) of DDIM paper https://arxiv.org/pdf/2010.02502.pdf+ # Ideally, read DDIM paper in-detail understanding++ # Notation (<variable name> -> <name in paper>+ # - pred_noise_t -> e_theta(x_t, t)+ # - pred_original_sample -> f_theta(x_t, t) or x_0+ # - std_dev_t -> sigma_t+ # - eta -> η+ # - pred_sample_direction -> "direction pointing to x_t"+ # - pred_prev_sample -> "x_t-1"++ # 1. get previous step value (=t-1)+ if self.step_length is not None:+ prev_timestep = timestep - self.step_length+ else:+ prev_timestep = timestep - self.config.num_train_timesteps // self.num_inference_steps++ # 2. compute alphas, betas+ alpha_prod_t = self.alphas_cumprod[timestep]+ if isinstance(prev_timestep, torch.Tensor):+ alpha_prod_t = alpha_prod_t.view(-1, 1, 1, 1)+ prev_timestep = prev_timestep.to(self.alphas_cumprod.device)+ alpha_prod_t_prev = torch.where(prev_timestep >= 0, self.alphas_cumprod[prev_timestep], self.final_alpha_cumprod)+ alpha_prod_t_prev = alpha_prod_t_prev.view(-1, 1, 1, 1)+ else:+ alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod++ # alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod++ beta_prod_t = 1 - alpha_prod_t++ # 3. compute predicted original sample from predicted noise also called+ # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf+ if self.config.prediction_type == "epsilon":+ pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5)+ pred_epsilon = model_output+ elif self.config.prediction_type == "sample":+ pred_original_sample = model_output+ pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5)+ elif self.config.prediction_type == "v_prediction":+ pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output+ pred_epsilon = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample+ else:+ raise ValueError(+ f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or"+ " `v_prediction`"+ )++ # 4. Clip or threshold "predicted x_0"+ if self.config.thresholding:+ pred_original_sample = self._threshold_sample(pred_original_sample)+ elif self.config.clip_sample:+ pred_original_sample = pred_original_sample.clamp(+ -self.config.clip_sample_range, self.config.clip_sample_range+ )++ # 5. compute variance: "sigma_t(η)" -> see formula (16)+ # σ_t = sqrt((1 − α_t−1)/(1 − α_t)) * sqrt(1 − α_t/α_t−1)+ variance = self._get_variance(alpha_prod_t, alpha_prod_t_prev)+ std_dev_t = eta * variance ** (0.5)++ if use_clipped_model_output:+ # the pred_epsilon is always re-derived from the clipped x_0 in Glide+ pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5)++ # 6. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf+ pred_sample_direction = (1 - alpha_prod_t_prev - std_dev_t**2) ** (0.5) * pred_epsilon++ # 7. compute x_t without "random noise" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf+ prev_sample = alpha_prod_t_prev ** (0.5) * pred_original_sample + pred_sample_direction++ if eta > 0:+ if variance_noise is not None and generator is not None:+ raise ValueError(+ "Cannot pass both generator and variance_noise. Please make sure that either `generator` or"+ " `variance_noise` stays `None`."+ )++ if variance_noise is None:+ variance_noise = randn_tensor(+ model_output.shape, generator=generator, device=model_output.device, dtype=model_output.dtype+ )+ variance = std_dev_t * variance_noise++ prev_sample = prev_sample + variance++ if not return_dict:+ return (prev_sample, pred_original_sample)++ return DDIMSchedulerOutput(prev_sample=prev_sample, pred_original_sample=pred_original_sample)++ # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise+ def add_noise(+ self,+ original_samples: torch.FloatTensor,+ noise: torch.FloatTensor,+ timesteps: torch.IntTensor,+ ) -> torch.FloatTensor:+ # Make sure alphas_cumprod and timestep have same device and dtype as original_samples+ # Move the self.alphas_cumprod to device to avoid redundant CPU to GPU data movement+ # for the subsequent add_noise calls+ shape_len = len(original_samples.shape)+ if shape_len == 5:+ b, c, f, h, w = original_samples.shape+ original_samples = rearrange(original_samples, "b c f h w -> (b f) c h w")+ noise = rearrange(noise, "b c f h w -> (b f) c h w")++ self.alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device)+ alphas_cumprod = self.alphas_cumprod.to(dtype=original_samples.dtype)+ timesteps = timesteps.to(original_samples.device)++ sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5+ sqrt_alpha_prod = sqrt_alpha_prod.flatten()+ while len(sqrt_alpha_prod.shape) < len(original_samples.shape):+ sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1)++ sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5+ sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten()+ while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape):+ sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1)++ noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise++ if shape_len == 5:+ noisy_samples = rearrange(noisy_samples, "(b f) c h w -> b c f h w", b=b)+ return noisy_samples++ # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.get_velocity+ def get_velocity(+ self, sample: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor+ ) -> torch.FloatTensor:+ # Make sure alphas_cumprod and timestep have same device and dtype as sample+ self.alphas_cumprod = self.alphas_cumprod.to(device=sample.device)+ alphas_cumprod = self.alphas_cumprod.to(dtype=sample.dtype)+ timesteps = timesteps.to(sample.device)++ sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5+ sqrt_alpha_prod = sqrt_alpha_prod.flatten()+ while len(sqrt_alpha_prod.shape) < len(sample.shape):+ sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1)++ sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5+ sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten()+ while len(sqrt_one_minus_alpha_prod.shape) < len(sample.shape):+ sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1)++ velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample+ return velocity++ def __len__(self):+ return self.config.num_train_timestepsdiff --git a/src/utils/util.py b/src/utils/util.pynew file mode 100644index 0000000..7f9ccbf--- /dev/null+++ b/src/utils/util.py@@ -0,0 +1,313 @@+import importlib+import os+import os.path as osp+import shutil+import sys+from pathlib import Path++import av+import numpy as np+import torch+import torchvision+from einops import rearrange+from PIL import Image+import cv2+++def save_checkpoint(model, save_dir, prefix, ckpt_num, logger, total_limit=None):+ save_path = osp.join(save_dir, f"{prefix}-{ckpt_num}.pth")++ if total_limit is not None:+ checkpoints = os.listdir(save_dir)+ checkpoints = [d for d in checkpoints if d.startswith(prefix)]+ checkpoints = sorted(+ checkpoints, key=lambda x: int(x.split("-")[1].split(".")[0])+ )++ if len(checkpoints) >= total_limit:+ num_to_remove = len(checkpoints) - total_limit + 1+ removing_checkpoints = checkpoints[0:num_to_remove]+ logger.info(+ f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints"+ )+ logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}")++ for removing_checkpoint in removing_checkpoints:+ removing_checkpoint = os.path.join(save_dir, removing_checkpoint)+ os.remove(removing_checkpoint)++ state_dict = model.state_dict()+ torch.save(state_dict, save_path)+++def create_code_snapshot(root, dst_path, extensions=(".py", ".h", ".cpp", ".cu", ".cc", ".cuh", ".json", ".sh", ".bat", ".yaml"), exclude=()):+ """Creates tarball with the source code"""+ import tarfile+ from pathlib import Path+ with tarfile.open(str(dst_path), "w:gz") as tar:+ for path in Path(root).rglob("*"):+ if '.git' in path.parts:+ continue+ exclude_flag = False+ if len(exclude) > 0:+ for k in exclude:+ if k in path.parts:+ exclude_flag = True+ if exclude_flag:+ continue+ if path.suffix.lower() in extensions:+ try:+ tar.add(path.as_posix(), arcname=path.relative_to(+ root).as_posix(), recursive=True)+ except:+ print(path)+ assert False, 'Error occur in create_code_snapshot'++def seed_everything(seed):+ import random++ import numpy as np++ torch.manual_seed(seed)+ torch.cuda.manual_seed_all(seed)+ np.random.seed(seed % (2**32))+ random.seed(seed)+++def import_filename(filename):+ spec = importlib.util.spec_from_file_location("mymodule", filename)+ module = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = module+ spec.loader.exec_module(module)+ return module+++def delete_additional_ckpt(base_path, num_keep):+ dirs = []+ for d in os.listdir(base_path):+ if d.startswith("checkpoint-"):+ dirs.append(d)+ num_tot = len(dirs)+ if num_tot <= num_keep:+ return+ # ensure ckpt is sorted and delete the ealier!+ del_dirs = sorted(dirs, key=lambda x: int(x.split("-")[-1]))[: num_tot - num_keep]+ for d in del_dirs:+ path_to_dir = osp.join(base_path, d)+ if osp.exists(path_to_dir):+ shutil.rmtree(path_to_dir)+++def save_videos_from_pil(pil_images, path, fps=8, crf=None):+ import av++ save_fmt = Path(path).suffix+ os.makedirs(os.path.dirname(path), exist_ok=True)+ width, height = pil_images[0].size++ if save_fmt == ".mp4":+ if True:+ codec = "libx264"+ container = av.open(path, "w")+ stream = container.add_stream(codec, rate=fps)++ stream.width = width+ stream.height = height+ if crf is not None:+ stream.options = {'crf': str(crf)}++ for pil_image in pil_images:+ # pil_image = Image.fromarray(image_arr).convert("RGB")+ av_frame = av.VideoFrame.from_image(pil_image)+ container.mux(stream.encode(av_frame))+ container.mux(stream.encode())+ container.close()+ else:++ video_writer = cv2.VideoWriter(+ path.replace('.mp4', '_cv.mp4'), cv2.VideoWriter_fourcc(*'mp4v'), fps, (width, height)+ )+ for pil_image in pil_images:+ img_np = cv2.cvtColor(np.array(pil_image), cv2.COLOR_RGB2BGR)+ video_writer.write(img_np)+ video_writer.release()++ elif save_fmt == ".gif":+ pil_images[0].save(+ fp=path,+ format="GIF",+ append_images=pil_images[1:],+ save_all=True,+ duration=(1 / fps * 1000),+ loop=0,+ )+ else:+ raise ValueError("Unsupported file type. Use .mp4 or .gif.")+++def save_videos_grid(videos_, path: str, rescale=False, n_rows=6, fps=8, crf=None):+ if not isinstance(videos_, list): videos_ = [videos_]++ outputs = []+ vid_len = videos_[0].shape[2]+ for i in range(vid_len):+ output = []+ for videos in videos_:+ videos = rearrange(videos, "b c t h w -> t b c h w")+ height, width = videos.shape[-2:]++ x = torchvision.utils.make_grid(videos[i], nrow=n_rows) # (c h w)+ x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) # (h w c)+ if rescale:+ x = (x + 1.0) / 2.0 # -1,1 -> 0,1+ x = (x * 255).numpy().astype(np.uint8)+ output.append(x)++ output = Image.fromarray(np.concatenate(output, axis=0))+ outputs.append(output)++ os.makedirs(os.path.dirname(path), exist_ok=True)+ save_videos_from_pil(outputs, path, fps, crf)+++def save_videos_grid_ori(videos: torch.Tensor, path: str, rescale=False, n_rows=6, fps=8):+ videos = rearrange(videos, "b c t h w -> t b c h w")+ height, width = videos.shape[-2:]+ outputs = []++ for x in videos:+ x = torchvision.utils.make_grid(x, nrow=n_rows) # (c h w)+ x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) # (h w c)+ if rescale:+ x = (x + 1.0) / 2.0 # -1,1 -> 0,1+ x = (x * 255).numpy().astype(np.uint8)+ x = Image.fromarray(x)++ outputs.append(x)++ os.makedirs(os.path.dirname(path), exist_ok=True)++ save_videos_from_pil(outputs, path, fps)+++def read_frames(video_path):+ container = av.open(video_path)++ video_stream = next(s for s in container.streams if s.type == "video")+ frames = []+ for packet in container.demux(video_stream):+ for frame in packet.decode():+ image = Image.frombytes(+ "RGB",+ (frame.width, frame.height),+ frame.to_rgb().to_ndarray(),+ )+ frames.append(image)++ return frames+++def get_fps(video_path):+ container = av.open(video_path)+ video_stream = next(s for s in container.streams if s.type == "video")+ fps = video_stream.average_rate+ container.close()+ return fps++def draw_keypoints(keypoints, height=512, width=512, device="cuda"):+ colors = torch.tensor([+ [255, 0, 0],+ [255, 255, 0],+ [0, 255, 0],+ [0, 255, 255],+ [0, 0, 255],+ [255, 0, 255],+ [255, 0, 85],+ ], device=device, dtype=torch.float32)++ selected = torch.tensor([1, 2, 3, 4, 12, 15, 20], device=device)+ B = keypoints.shape[0]++ # [B, len(selected), 2]+ pts = keypoints[:, selected] * 0.5 + 0.5+ pts[..., 0] *= width+ pts[..., 1] *= height+ pts = pts.long()++ canvas = torch.zeros((B, 3, height, width), device=device)+ radius = 4++ for i, color in enumerate(colors):+ x = pts[:, i, 0]+ y = pts[:, i, 1]+ mask = (+ (x[:, None, None] - torch.arange(width, device=device)) ** 2+ + (y[:, None, None] - torch.arange(height, device=device)[:, None]) ** 2+ ) <= radius**2+ canvas[:, 0] += color[0] / 255.0 * mask+ canvas[:, 1] += color[1] / 255.0 * mask+ canvas[:, 2] += color[2] / 255.0 * mask++ return canvas.clamp(0, 1)++def get_boxes(keypoints, height=512, width=512):+ selected = torch.tensor([1, 2, 3, 4, 12, 15, 20])++ # [B, len(selected), 2]+ pts = keypoints[:, selected] * 0.5 + 0.5+ pts[..., 0] *= width+ pts[..., 1] *= height+ pts = pts.long()++ cx = pts[..., 0].float().mean(dim=1) # [B]+ cy = pts[..., 1].float().mean(dim=1) # [B]++ min_y = pts[..., 1].float().min(dim=1)[0] # [B]++ side = (cy - min_y) * 2.0+ side = side * 1.7++ x1 = (cx - side / 2 * 0.95).clamp(0, width - 1).long()+ y1 = (cy - side / 2 * 0.95).clamp(0, height - 1).long()+ x2 = (cx + side / 2 * 1.05).clamp(0, width - 1).long()+ y2 = (cy + side / 2 * 1.05).clamp(0, height - 1).long()++ boxes = torch.stack([x1, y1, x2, y2], dim=1) # [B, 4]++ return boxes++def crop_face(image_pil, face_mesh):+ image = np.array(image_pil)+ h, w = image.shape[:2]+ results = face_mesh.process(image)+ face_landmarks = results.multi_face_landmarks[0]+ coords = [(int(l.x * w), int(l.y * h)) for l in face_landmarks.landmark]+ xs, ys = zip(*coords)+ x1, y1 = min(xs), min(ys)+ x2, y2 = max(xs), max(ys)+ face_box = (x1, y1, x2, y2)++ left, top, right, bot = scale_bb(face_box, scale=1.1, size=image.shape[:2])++ face_patch = image[int(top) : int(bot), int(left) : int(right)]++ return face_patch++def scale_bb(bbox, scale, size):+ left, top, right, bot = bbox+ width = right - left+ height = bot - top+ length = max(width, height) * scale+ center_X = (left + right) * 0.5+ center_Y = (top + bot) * 0.5+ left, top, right, bot = [+ center_X - length / 2,+ center_Y - length / 2,+ center_X + length / 2,+ center_Y + length / 2,+ ]+ left = max(0, left)+ top = max(0, top)+ right = min(size[1] - 1, right)+ bot = min(size[0] - 1, bot)+ return np.array([left, top, right, bot])\ No newline at end of filediff --git a/src/wrapper.py b/src/wrapper.pynew file mode 100644index 0000000..c6de821--- /dev/null+++ b/src/wrapper.py@@ -0,0 +1,400 @@+from omegaconf import OmegaConf+import os+import torch+import numpy as np+from PIL import Image+import time+import gc+import cv2+from src.models.unet_2d_condition import UNet2DConditionModel+from src.models.pose_guider import PoseGuider+from src.models.motion_encoder.encoder import MotEncoder+from src.models.unet_3d import UNet3DConditionModel+from src.models.mutual_self_attention import ReferenceAttentionControl+from src.scheduler.scheduler_ddim import DDIMScheduler+from src.liveportrait.motion_extractor import MotionExtractor+from diffusers import AutoencoderKL+from diffusers.image_processor import VaeImageProcessor+from transformers import CLIPVisionModelWithProjection, CLIPImageProcessor+from collections import deque+from threading import Lock, Thread+from torchvision import transforms as T+from einops import rearrange+from src.utils.util import draw_keypoints, get_boxes+import torch.nn.functional as F++def map_device(device_or_str):+ return device_or_str if isinstance(device_or_str, torch.device) else torch.device(device_or_str)++class PersonaLive:+ def __init__(self, config_path, device=None):+ cfg = OmegaConf.load(config_path)+ if(device is None):+ self.device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")+ else:+ self.device = map_device(device)++ self.temporal_adaptive_step = cfg.temporal_adaptive_step+ self.temporal_window_size = cfg.temporal_window_size++ if cfg.dtype == "fp16":+ self.numpy_dtype = np.float16+ self.dtype = torch.float16+ elif cfg.dtype == "fp32":+ self.numpy_dtype = np.float32+ self.dtype = torch.float32++ infer_config = OmegaConf.load(cfg.inference_config)+ sched_kwargs = OmegaConf.to_container(+ infer_config.noise_scheduler_kwargs+ )++ self.num_inference_steps = cfg.num_inference_steps++ # initialize models+ self.pose_guider = PoseGuider().to(device=self.device, dtype=self.dtype)+ pose_guider_state_dict = torch.load(cfg.pose_guider_path, map_location="cpu")+ self.pose_guider.load_state_dict(pose_guider_state_dict)+ del pose_guider_state_dict++ self.motion_encoder = MotEncoder().to(dtype=self.dtype, device=self.device).eval()+ motion_encoder_state_dict = torch.load(cfg.motion_encoder_path, map_location="cpu")+ self.motion_encoder.load_state_dict(motion_encoder_state_dict)+ del motion_encoder_state_dict++ self.pose_encoder = MotionExtractor(num_kp=21).to(device=self.device, dtype=self.dtype).eval()+ pose_encoder_state_dict = torch.load(cfg.pose_encoder_path, map_location="cpu")+ self.pose_encoder.load_state_dict(pose_encoder_state_dict, strict=False)+ del pose_encoder_state_dict++ self.denoising_unet = UNet3DConditionModel.from_pretrained_2d(+ cfg.pretrained_base_model_path,+ "",+ subfolder="unet",+ unet_additional_kwargs=infer_config.unet_additional_kwargs,+ ).to(dtype=self.dtype, device=self.device)++ self.reference_unet = UNet2DConditionModel.from_pretrained(+ cfg.pretrained_base_model_path,+ subfolder="unet",+ ).to(dtype=self.dtype, device=self.device)+ reference_unet_state_dict = torch.load(cfg.reference_unet_weight_path, map_location="cpu")+ self.reference_unet.load_state_dict(reference_unet_state_dict)+ del reference_unet_state_dict++ self.denoising_unet.load_state_dict(+ torch.load(cfg.denoising_unet_path, map_location="cpu"), strict=False+ )+ self.denoising_unet.load_state_dict(+ torch.load(+ cfg.temporal_module_path,+ map_location="cpu",+ ),+ strict=False,+ )++ self.reference_control_writer = ReferenceAttentionControl(+ self.reference_unet,+ do_classifier_free_guidance=False,+ mode="write",+ batch_size=cfg.batch_size,+ fusion_blocks="full",+ )+ self.reference_control_reader = ReferenceAttentionControl(+ self.denoising_unet,+ do_classifier_free_guidance=False,+ mode="read",+ batch_size=cfg.batch_size,+ fusion_blocks="full",+ cache_kv=True,+ )++ self.vae = AutoencoderKL.from_pretrained(cfg.vae_model_path).to(+ device=self.device, dtype=self.dtype+ )+ self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(+ cfg.image_encoder_path,+ ).to(device=self.device, dtype=self.dtype)++ # miscellaneous+ self.scheduler = DDIMScheduler(**sched_kwargs)+ self.timesteps = torch.tensor([999, 666, 333, 0], device=self.device).long()+ self.scheduler.set_step_length(333)++ self.generator = torch.Generator(self.device)+ self.generator.manual_seed(cfg.seed)++ self.batch_size = cfg.batch_size+ self.vae_scale_factor = 8+ self.ref_image_processor = VaeImageProcessor(+ vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True+ )+ self.clip_image_processor = CLIPImageProcessor()+ self.cond_image_processor = VaeImageProcessor(+ vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True, do_normalize=True)++ self.first_frame = True+ self.motion_bank = None+ self.count = 0+ self.num_khf = 0++ self.latents_pile = deque([])+ self.pose_pile = deque([])+ self.motion_pile = deque([])++ self.cfg = cfg+ torch.cuda.empty_cache()++ self.enable_xformers_memory_efficient_attention()++ def enable_xformers_memory_efficient_attention(self):+ self.reference_unet.enable_xformers_memory_efficient_attention()+ self.denoising_unet.enable_xformers_memory_efficient_attention()++ def fast_resize(self, images, target_width, target_height) -> torch.Tensor:+ tgt_cond_tensor = F.interpolate(+ images,+ size=(target_width, target_height),+ mode="bilinear",+ align_corners=False,+ )+ return tgt_cond_tensor++ @torch.no_grad()+ def fuse_reference(self, ref_image): # pil input+ clip_image = self.clip_image_processor.preprocess(+ ref_image, return_tensors="pt"+ ).pixel_values+ ref_image_tensor = self.ref_image_processor.preprocess(+ ref_image, height=self.cfg.reference_image_height, width=self.cfg.reference_image_width+ ) # (bs, c, width, height)+ clip_image_embeds = self.image_encoder(+ clip_image.to(self.image_encoder.device, dtype=self.image_encoder.dtype)+ ).image_embeds+ self.encoder_hidden_states = clip_image_embeds.unsqueeze(1)+ ref_image_tensor = ref_image_tensor.to(+ dtype=self.vae.dtype, device=self.vae.device+ )+ self.ref_image_tensor = ref_image_tensor.squeeze(0)+ ref_image_latents = self.vae.encode(ref_image_tensor).latent_dist.mean+ ref_image_latents = ref_image_latents * 0.18215 # (b, 4, h, w)+ self.reference_unet(+ ref_image_latents.to(self.reference_unet.device),+ torch.zeros((self.batch_size,),dtype=self.dtype,device=self.reference_unet.device),+ encoder_hidden_states=self.encoder_hidden_states,+ return_dict=False,+ )+ self.reference_control_reader.update(self.reference_control_writer)+ self.encoder_hidden_states = self.encoder_hidden_states.to(self.device)++ ref_cond_tensor = self.cond_image_processor.preprocess(+ ref_image, height=256, width=256+ ).to(device=self.device, dtype=self.pose_encoder.dtype) # (1, c, h, w)+ self.ref_cond_tensor = ref_cond_tensor / 2 + 0.5 # to [0, 1]+ self.ref_image_latents = ref_image_latents++ padding_num = (self.temporal_adaptive_step - 1) * self.temporal_window_size+ init_latents = ref_image_latents.unsqueeze(2).repeat(1, 1, padding_num, 1, 1)+ noise = torch.randn_like(init_latents)+ init_timesteps = reversed(self.timesteps).repeat_interleave(self.temporal_window_size, dim=0)+ noisy_latents_first = self.scheduler.add_noise(init_latents, noise, init_timesteps[:padding_num])+ for i in range(self.temporal_adaptive_step-1):+ l = i * self.temporal_window_size+ r = (i+1) * self.temporal_window_size+ self.latents_pile.append(noisy_latents_first[:,:,l:r])++ def crop_face(self, image_pil, boxes):+ image = np.array(image_pil)++ left, top, right, bot = boxes++ face_patch = image[int(top) : int(bot), int(left) : int(right)]+ face_patch = Image.fromarray(face_patch).convert("RGB")+ return face_patch++ def crop_face_tensor(self, image_tensor, boxes):+ left, top, right, bot = boxes+ left, top, right, bottom = map(int, (left, top, right, bot))++ face_patch = image_tensor[:, top:bottom, left:right]+ face_patch = F.interpolate(+ face_patch.unsqueeze(0),+ size=(224, 224),+ mode="bilinear",+ align_corners=False,+ )+ return face_patch++ def interpolate_tensors(self, a: torch.Tensor, b: torch.Tensor, num: int = 10) -> torch.Tensor:+ """+ 在张量 a 和 b 之间线性插值。+ 输入 shape: (B, 1, D1, D2, ...)+ 输出 shape: (B, num, D1, D2, ...)+ """+ if a.shape != b.shape:+ raise ValueError(f"Shape mismatch: a.shape={a.shape}, b.shape={b.shape}")++ B, _, *rest = a.shape+ # 插值系数 (num,) → reshape 成 (1, num, 1, 1, ...)+ alphas = torch.linspace(0, 1, num, device=a.device, dtype=a.dtype)+ view_shape = (1, num) + (1,) * len(rest)+ alphas = alphas.view(view_shape) # (1, num, 1, 1, ...)++ # 插值 (B, num, D1, D2, ...)+ result = (1 - alphas) * a + alphas * b+ return result++ def calculate_dis(self, A, B, threshold=10.):+ """+ A: (b, f1, c1, c2) bank+ B: (b, f2, c1, c2) new data+ """++ A_flat = A.view(A.size(1), -1).clone()+ B_flat = B.view(B.size(1), -1).clone()++ dist = torch.cdist(B_flat.to(torch.float32), A_flat.to(torch.float32), p=2)++ min_dist, min_idx = dist.min(dim=1) # (f2,)++ idx_to_add = torch.nonzero(min_dist[:1] > threshold, as_tuple=False).squeeze(1).tolist()++ if len(idx_to_add) > 0: # 有需要添加的元素+ B_to_add = B[:, idx_to_add] # (1, k, c1, c2)+ A_new = torch.cat([A, B_to_add], dim=1) # (1, f1+k, c1, c2)+ else:+ A_new = A # 没有需要添加的++ return idx_to_add, A_new, min_idx++ @torch.no_grad()+ def process_input(self, images):+ batch_size = self.batch_size+ device = self.device+ temporal_window_size = self.temporal_window_size+ temporal_adaptive_step = self.temporal_adaptive_step++ tgt_cond_tensor = self.fast_resize(images, 256, 256)+ tgt_cond_tensor = tgt_cond_tensor / 2 + 0.5++ if self.first_frame:+ mot_bbox_param, kps_ref, kps_frame1, kps_dri = self.pose_encoder.interpolate_kps_online(self.ref_cond_tensor, tgt_cond_tensor, num_interp=12+1)+ self.kps_ref = kps_ref+ self.kps_frame1 = kps_frame1+ else:+ mot_bbox_param, kps_dri = self.pose_encoder.get_kps(self.kps_ref, self.kps_frame1, tgt_cond_tensor)++ keypoints = draw_keypoints(mot_bbox_param, device=device)+ boxes = get_boxes(kps_dri)+ keypoints = rearrange(keypoints.unsqueeze(2), 'f c b h w -> b c f h w')+ keypoints = keypoints.to(device=device, dtype=self.pose_guider.dtype)++ if self.first_frame:+ ref_box = get_boxes(mot_bbox_param[:1])+ ref_face = self.crop_face_tensor(self.ref_image_tensor, ref_box[0])+ motion_face = [ref_face]+ for i, frame in enumerate(images):+ motion_face.append(self.crop_face_tensor(frame, boxes[i]))+ pose_cond_tensor = torch.cat(motion_face, dim=0).transpose(0, 1)+ pose_cond_tensor = pose_cond_tensor.unsqueeze(0)+ # pose_cond_tensor = pose_cond_tensor.to(+ # device=device, dtype=self.motion_encoder.dtype+ # )+ motion_hidden_states = self.motion_encoder(pose_cond_tensor)+ ref_motion = motion_hidden_states[:, :1]+ dri_motion = motion_hidden_states[:, 1:]++ init_motion_hidden_states = self.interpolate_tensors(ref_motion, dri_motion[:,:1], num=12+1)[:,:-1]+ for i in range(temporal_adaptive_step-1):+ l = i * temporal_window_size+ r = (i+1) * temporal_window_size+ self.motion_pile.append(init_motion_hidden_states[:,l:r])+ self.motion_pile.append(dri_motion)++ self.motion_bank = ref_motion+ else:+ motion_face = []+ for i, frame in enumerate(images):+ motion_face.append(self.crop_face_tensor(frame, boxes[i]))+ pose_cond_tensor = torch.cat(motion_face, dim=0).transpose(0, 1)+ pose_cond_tensor = pose_cond_tensor.unsqueeze(0)+ motion_hidden_states = self.motion_encoder(pose_cond_tensor)+ self.motion_pile.append(motion_hidden_states)++ pose_fea = self.pose_guider(keypoints)+ if self.first_frame:+ for i in range(temporal_adaptive_step):+ l = i * temporal_window_size+ r = (i+1) * temporal_window_size+ self.pose_pile.append(pose_fea[:,:,l:r])+ self.first_frame = False+ else:+ self.pose_pile.append(pose_fea)++ latents = self.ref_image_latents.unsqueeze(2).repeat(1, 1, temporal_window_size, 1, 1)+ noise = torch.randn_like(latents)+ latents = self.scheduler.add_noise(latents, noise, self.timesteps[:1])+ self.latents_pile.append(latents)++ jump = 1+ motion_hidden_state = torch.cat(list(self.motion_pile), dim=1)+ pose_cond_fea=torch.cat(list(self.pose_pile), dim=2)++ idx_to_add = []+ if self.count > 8:+ idx_to_add, self.motion_bank, idx_his = self.calculate_dis(self.motion_bank, motion_hidden_state, threshold=17.)++ latents_model_input = torch.cat(list(self.latents_pile), dim=2)+ for j in range(jump):+ timesteps = reversed(self.timesteps[j::jump]).repeat_interleave(temporal_window_size, dim=0)+ timesteps = torch.stack([timesteps] * batch_size)#.to(device)+ timesteps = rearrange(timesteps, 'b f -> (b f)')+ noise_pred = self.denoising_unet(+ latents_model_input,+ timesteps,+ encoder_hidden_states=[self.encoder_hidden_states,+ motion_hidden_state],+ pose_cond_fea=pose_cond_fea,+ return_dict=False,+ )[0]++ clip_length = noise_pred.shape[2]+ mid_noise_pred = rearrange(noise_pred, 'b c f h w -> (b f) c h w')+ mid_latents = rearrange(latents_model_input, 'b c f h w -> (b f) c h w')+ latents_model_input, pred_original_sample = self.scheduler.step(+ mid_noise_pred, timesteps, mid_latents, generator=self.generator, return_dict=False+ )+ latents_model_input = rearrange(latents_model_input, '(b f) c h w -> b c f h w', f=clip_length)+ pred_original_sample = rearrange(pred_original_sample, '(b f) c h w -> b c f h w', f=clip_length)+ latents_model_input = torch.cat([+ pred_original_sample[:,:,:temporal_window_size],+ latents_model_input[:,:,temporal_window_size:]], dim=2)+ latents_model_input = latents_model_input.to(dtype=self.dtype)++ if len(idx_to_add) > 0 and self.num_khf < 3:+ self.reference_control_writer.clear()+ self.reference_unet(+ pred_original_sample[:,:,0].to(self.reference_unet.dtype),+ torch.zeros((batch_size,),dtype=self.dtype,device=self.reference_unet.device),+ encoder_hidden_states=self.encoder_hidden_states,+ return_dict=False,+ )+ self.reference_control_reader.update_hkf(self.reference_control_writer)+ print('add_keyframes')+ self.num_khf += 1+++ for i in range(len(self.latents_pile)):+ self.latents_pile[i] = latents_model_input[:, :, i * temporal_adaptive_step : (i + 1) * temporal_adaptive_step, :, :]++ self.pose_pile.popleft()+ self.motion_pile.popleft()+ latents = self.latents_pile.popleft()+ latents = 1 / 0.18215 * latents+ latents = rearrange(latents, "b c f h w -> (b f) c h w")+ video = self.vae.decode(latents).sample+ video = rearrange(video, "b c h w -> b h w c")+ video = (video / 2 + 0.5).clamp(0, 1)+ video = video.cpu().numpy()+ self.count += 1+ return video\ No newline at end of filediff --git a/src/wrapper_trt.py b/src/wrapper_trt.pynew file mode 100644index 0000000..fa1b534--- /dev/null+++ b/src/wrapper_trt.py@@ -0,0 +1,336 @@+from omegaconf import OmegaConf+import os+import torch+import numpy as np+from PIL import Image+import time+import gc+import cv2+from src.models.unet_2d_condition import UNet2DConditionModel+from src.models.pose_guider import PoseGuider+from src.models.motion_encoder.encoder import MotEncoder+from src.models.unet_3d import UNet3DConditionModel+from src.models.mutual_self_attention import ReferenceAttentionControl+from src.scheduler.scheduler_ddim import DDIMScheduler+from src.liveportrait.motion_extractor import MotionExtractor+from diffusers import AutoencoderKL+from diffusers.image_processor import VaeImageProcessor+from transformers import CLIPVisionModelWithProjection, CLIPImageProcessor+from collections import deque+from threading import Lock, Thread+from torchvision import transforms as T+from einops import rearrange+from src.utils.util import draw_keypoints, get_boxes+import torch.nn.functional as F+from src.modeling.engine_model import EngineModel++def map_device(device_or_str):+ return device_or_str if isinstance(device_or_str, torch.device) else torch.device(device_or_str)++class PersonaLive:+ def __init__(self, config_path, device=None):+ cfg = OmegaConf.load(config_path)+ if(device is None):+ self.device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")+ else:+ self.device = map_device(device)++ self.temporal_adaptive_step = cfg.temporal_adaptive_step+ self.temporal_window_size = cfg.temporal_window_size++ if cfg.dtype == "fp16":+ self.numpy_dtype = np.float16+ self.dtype = torch.float16+ elif cfg.dtype == "fp32":+ self.numpy_dtype = np.float32+ self.dtype = torch.float32++ infer_config = OmegaConf.load(cfg.inference_config)+ sched_kwargs = OmegaConf.to_container(+ infer_config.noise_scheduler_kwargs+ )++ self.num_inference_steps = cfg.num_inference_steps++ # initialize models+ self.pose_guider = PoseGuider().to(device=self.device, dtype=self.dtype)+ pose_guider_state_dict = torch.load(cfg.pose_guider_path, map_location="cpu")+ self.pose_guider.load_state_dict(pose_guider_state_dict)+ del pose_guider_state_dict++ self.motion_encoder = MotEncoder().to(dtype=self.dtype, device=self.device).eval()+ motion_encoder_state_dict = torch.load(cfg.motion_encoder_path, map_location="cpu")+ self.motion_encoder.load_state_dict(motion_encoder_state_dict)+ del motion_encoder_state_dict++ self.pose_encoder = MotionExtractor(num_kp=21).to(device=self.device, dtype=self.dtype).eval()+ pose_encoder_state_dict = torch.load(cfg.pose_encoder_path, map_location="cpu")+ self.pose_encoder.load_state_dict(pose_encoder_state_dict, strict=False)+ del pose_encoder_state_dict++ self.reference_unet = UNet2DConditionModel.from_pretrained(+ cfg.pretrained_base_model_path,+ subfolder="unet",+ ).to(dtype=self.dtype, device=self.device)+ reference_unet_state_dict = torch.load(cfg.reference_unet_weight_path, map_location="cpu")+ self.reference_unet.load_state_dict(reference_unet_state_dict)+ del reference_unet_state_dict++ self.reference_control_writer = ReferenceAttentionControl(+ self.reference_unet,+ do_classifier_free_guidance=False,+ mode="write",+ batch_size=cfg.batch_size,+ fusion_blocks="full",+ )++ self.vae = AutoencoderKL.from_pretrained(cfg.vae_model_path).to(+ device=self.device, dtype=self.dtype+ )+ self.image_encoder = CLIPVisionModelWithProjection.from_pretrained(+ cfg.image_encoder_path,+ ).to(device=self.device, dtype=self.dtype)++ #----------------------- TensorRT -----------------------#+ self.unet_work = EngineModel(engine_file_path=cfg.tensorrt_target_model, device_int=self.device.index)+ self.unet_work.bind({+ "motion_hidden_states_out": "motion_hidden_states",+ "pose_cond_fea_out": "pose_cond_fea",+ "latents" : "sample",+ })+ #------------------------------------------------------------#++ # miscellaneous+ self.scheduler = DDIMScheduler(**sched_kwargs)+ timesteps = torch.tensor([0, 333, 666, 999], device=self.device)+ self.timesteps = timesteps.repeat_interleave(cfg.temporal_window_size, dim=0).long()+ self.scheduler.set_step_length(333)++ self.batch_size = cfg.batch_size+ self.vae_scale_factor = 8+ self.ref_image_processor = VaeImageProcessor(+ vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True+ )+ self.clip_image_processor = CLIPImageProcessor()+ self.cond_image_processor = VaeImageProcessor(+ vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True, do_normalize=True)++ self.first_frame = True+ self.motion_bank = None+ self.count = 0+ self.num_khf = 0++ self.cfg = cfg+ self.reference_hidden_states_names = ["d00", "d01", "d10", "d11",+ "d20", "d21", "m", "u10", "u11", "u12",+ "u20", "u21", "u22", "u30", "u31", "u32"]+ torch.cuda.empty_cache()++ self.enable_xformers_memory_efficient_attention()++ def enable_xformers_memory_efficient_attention(self):+ self.reference_unet.enable_xformers_memory_efficient_attention()++ def fast_resize(self, images, target_width, target_height) -> torch.Tensor:+ tgt_cond_tensor = F.interpolate(+ images,+ size=(target_width, target_height),+ mode="bilinear",+ align_corners=False,+ )+ return tgt_cond_tensor++ @torch.no_grad()+ def fuse_reference(self, ref_image): # pil input+ clip_image = self.clip_image_processor.preprocess(+ ref_image, return_tensors="pt"+ ).pixel_values+ ref_image_tensor = self.ref_image_processor.preprocess(+ ref_image, height=self.cfg.reference_image_height, width=self.cfg.reference_image_width+ ) # (bs, c, width, height)+ clip_image_embeds = self.image_encoder(+ clip_image.to(self.image_encoder.device, dtype=self.image_encoder.dtype)+ ).image_embeds+ encoder_hidden_states = clip_image_embeds.unsqueeze(1)+ self.unet_work.prefill(encoder_hidden_states = encoder_hidden_states)+ self.encoder_hidden_states = encoder_hidden_states++ ref_image_tensor = ref_image_tensor.to(+ dtype=self.vae.dtype, device=self.vae.device+ )+ self.ref_image_tensor = ref_image_tensor.squeeze(0)+ ref_image_latents = self.vae.encode(ref_image_tensor).latent_dist.mean+ ref_image_latents = ref_image_latents * 0.18215 # (b, 4, h, w)+ self.reference_unet(+ ref_image_latents.to(self.reference_unet.device),+ torch.zeros((self.batch_size,),dtype=self.dtype,device=self.reference_unet.device),+ encoder_hidden_states=encoder_hidden_states,+ return_dict=False,+ )+ self.reference_hidden_states = self.reference_control_writer.output()+ self.unet_work.prefill(**{name: self.reference_hidden_states[name] for name in self.reference_hidden_states_names})++ ref_cond_tensor = self.cond_image_processor.preprocess(+ ref_image, height=256, width=256+ ).to(device=self.device, dtype=self.pose_encoder.dtype) # (1, c, h, w)+ self.ref_cond_tensor = ref_cond_tensor / 2 + 0.5 # to [0, 1]+ self.ref_image_latents = ref_image_latents.unsqueeze(2).repeat(1, 1, self.temporal_window_size, 1, 1)++ padding_num = (self.temporal_adaptive_step - 1) * self.temporal_window_size+ init_latents = ref_image_latents.unsqueeze(2).repeat(1, 1, padding_num, 1, 1)+ noise = torch.randn_like(init_latents)+ self.noisy_latents_first = self.scheduler.add_noise(init_latents, noise, self.timesteps[:padding_num])++ def crop_face(self, image_pil, boxes):+ image = np.array(image_pil)++ left, top, right, bot = boxes++ face_patch = image[int(top) : int(bot), int(left) : int(right)]+ face_patch = Image.fromarray(face_patch).convert("RGB")+ return face_patch++ def crop_face_tensor(self, image_tensor, boxes):+ left, top, right, bot = boxes+ left, top, right, bottom = map(int, (left, top, right, bot))++ face_patch = image_tensor[:, top:bottom, left:right]+ face_patch = F.interpolate(+ face_patch.unsqueeze(0),+ size=(224, 224),+ mode="bilinear",+ align_corners=False,+ )+ return face_patch++ def interpolate_tensors(self, a: torch.Tensor, b: torch.Tensor, num: int = 10) -> torch.Tensor:+ """+ 在张量 a 和 b 之间线性插值。+ 输入 shape: (B, 1, D1, D2, ...)+ 输出 shape: (B, num, D1, D2, ...)+ """+ if a.shape != b.shape:+ raise ValueError(f"Shape mismatch: a.shape={a.shape}, b.shape={b.shape}")++ B, _, *rest = a.shape+ # 插值系数 (num,) → reshape 成 (1, num, 1, 1, ...)+ alphas = torch.linspace(0, 1, num, device=a.device, dtype=a.dtype)+ view_shape = (1, num) + (1,) * len(rest)+ alphas = alphas.view(view_shape) # (1, num, 1, 1, ...)++ # 插值 (B, num, D1, D2, ...)+ result = (1 - alphas) * a + alphas * b+ return result++ def calculate_dis(self, A, B, threshold=10.):+ """+ A: (b, f1, c1, c2) bank+ B: (b, f2, c1, c2) new data+ """++ A_flat = A.view(A.size(1), -1).clone()+ B_flat = B.view(B.size(1), -1).clone()++ dist = torch.cdist(B_flat.to(torch.float32), A_flat.to(torch.float32), p=2)++ min_dist, min_idx = dist.min(dim=1) # (f2,)++ idx_to_add = torch.nonzero(min_dist[:1] > threshold, as_tuple=False).squeeze(1).tolist()++ if len(idx_to_add) > 0: # 有需要添加的元素+ B_to_add = B[:, idx_to_add] # (1, k, c1, c2)+ A_new = torch.cat([A, B_to_add], dim=1) # (1, f1+k, c1, c2)+ else:+ A_new = A # 没有需要添加的++ return idx_to_add, A_new, min_idx++ @torch.no_grad()+ def process_input(self, images):+ batch_size = self.batch_size+ device = self.device++ tgt_cond_tensor = self.fast_resize(images, 256, 256)+ tgt_cond_tensor = tgt_cond_tensor / 2 + 0.5++ if self.first_frame:+ mot_bbox_param, kps_ref, kps_frame1, kps_dri = self.pose_encoder.interpolate_kps_online(self.ref_cond_tensor, tgt_cond_tensor, num_interp=12+1)+ self.kps_ref = kps_ref+ self.kps_frame1 = kps_frame1+ else:+ mot_bbox_param, kps_dri = self.pose_encoder.get_kps(self.kps_ref, self.kps_frame1, tgt_cond_tensor)++ keypoints = draw_keypoints(mot_bbox_param, device=device)+ boxes = get_boxes(kps_dri)+ keypoints = rearrange(keypoints.unsqueeze(2), 'f c b h w -> b c f h w')+ keypoints = keypoints.to(device=device, dtype=self.pose_guider.dtype)++ if self.first_frame:+ pose_cond_fea = self.pose_guider(keypoints[:,:, :12])+ pose = keypoints[:,:,12:]++ ref_box = get_boxes(mot_bbox_param[:1])+ ref_face = self.crop_face_tensor(self.ref_image_tensor, ref_box[0])+ motion_face = [ref_face]+ for i, frame in enumerate(images):+ motion_face.append(self.crop_face_tensor(frame, boxes[i]))+ motion_cond_tensor = torch.cat(motion_face, dim=0).transpose(0, 1)+ motion_cond_tensor = motion_cond_tensor.unsqueeze(0)+ motion = motion_cond_tensor[:,:,1:]+ motion_hidden_states = self.motion_encoder(motion_cond_tensor[:,:,:2])+ ref_motion = motion_hidden_states[:, :1]+ dri_motion = motion_hidden_states[:, 1:]++ motion_hidden_states = self.interpolate_tensors(ref_motion, dri_motion[:,:1], num=12+1)[:,:-1]+ self.motion_bank = ref_motion++ latents = self.ref_image_latents+ noise = torch.randn_like(latents)+ latents = self.scheduler.add_noise(latents, noise, self.timesteps[-1:])+ sample = torch.cat([self.noisy_latents_first, latents], dim=2)++ self.unet_work.prefill(latents=sample)+ self.unet_work.prefill(motion_hidden_states_out=motion_hidden_states)+ self.unet_work.prefill(pose_cond_fea_out=pose_cond_fea)+ self.first_frame = False+ else:+ pose = keypoints++ motion_face = []+ for i, frame in enumerate(images):+ motion_face.append(self.crop_face_tensor(frame, boxes[i]))+ motion = torch.cat(motion_face, dim=0).transpose(0, 1)+ motion = motion.unsqueeze(0)++ motion = motion.to(dtype = self.dtype)+ latents = self.ref_image_latents+ noise = torch.randn_like(latents)+ new_noise = self.scheduler.add_noise(latents, noise, self.timesteps[-1:])++ results = self.unet_work(output_list=["pred_video", "motion_out", "latent_first"], return_tensor=True, pose=pose, motion=motion, new_noise=new_noise)+ video = results['pred_video'].cpu().numpy()+ motion_out = results['motion_out']++ idx_to_add = []+ if self.count > 8:+ idx_to_add, self.motion_bank, idx_his = self.calculate_dis(self.motion_bank, motion_out, threshold=17.)++ if len(idx_to_add) > 0 and self.num_khf < 3:+ latents_first = results['latent_first']+ self.reference_control_writer.clear()+ self.reference_unet(+ latents_first.to(self.reference_unet.dtype),+ torch.zeros((batch_size,),dtype=self.dtype,device=self.reference_unet.device),+ encoder_hidden_states=self.encoder_hidden_states,+ return_dict=False,+ )+ reference_hidden_states = self.reference_control_writer.output()+ for name in self.reference_hidden_states_names:+ self.reference_hidden_states[name] = torch.cat([self.reference_hidden_states[name], reference_hidden_states[name]], dim=1)++ self.unet_work.prefill(**{name: self.reference_hidden_states[name] for name in self.reference_hidden_states_names})+ print('add_keyframes')+ self.num_khf += 1++ self.count += 1+ return video\ No newline at end of file