Open-Sora: Democratizing Efficient Video Production for All
Python
update
zhengzangw5c8139521a40
diff --git a/opensora/models/vae/vae_3d.py b/opensora/models/vae/vae_3d.pyindex 41613f2..0cf288a 100644--- a/opensora/models/vae/vae_3d.py+++ b/opensora/models/vae/vae_3d.py@@ -6,7 +6,7 @@ import numpy as npimport torchimport torch.nn as nnimport torch.nn.functional as F-from einops import pack, rearrange, repeat, unpack+from einops import pack, rearrange, repeatfrom opensora.registry import MODELSfrom opensora.utils.ckpt_utils import find_model, load_checkpoint@@ -675,18 +675,21 @@ class Encoder(nn.Module):# print("encoder:", x.size())- for i in range(self.num_blocks):- for j in range(self.num_res_blocks):- x = self.block_res_blocks[i][j](x)- # print("encoder:", x.size())+ try:+ for i in range(self.num_blocks):+ for j in range(self.num_res_blocks):+ x = self.block_res_blocks[i][j](x)+ # print("encoder:", x.size())- if i < self.num_blocks - 1:- x = self.conv_blocks[i](x)- # print("encoder:", x.size())+ if i < self.num_blocks - 1:+ x = self.conv_blocks[i](x)+ # print("encoder:", x.size())- for i in range(self.num_res_blocks):- x = self.res_blocks[i](x)- # print("encoder:", x.size())+ for i in range(self.num_res_blocks):+ x = self.res_blocks[i](x)+ # print("encoder:", x.size())+ except:+ breakpoint()x = self.norm1(x)x = self.activate(x)@@ -940,27 +943,27 @@ class VAE_3D_V2(nn.Module): # , ModelMixinvideo,video_contains_first_frame=True,):- encode_first_frame_separately = self.separate_first_frame_encoding and video_contains_first_frame+ self.separate_first_frame_encoding and video_contains_first_frame# whether to pad video or not- if video_contains_first_frame:- video_len = video.shape[2]- video = pad_at_dim(video, (self.time_padding, 0), value=0.0, dim=2)- video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]+ # if video_contains_first_frame:+ # video_len = video.shape[2]+ # video = pad_at_dim(video, (self.time_padding, 0), value=0.0, dim=2)+ # video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]# print("pre-encoder:", video.size())# NOTE: moved encoder conv1 here for separate first frame encoding- if encode_first_frame_separately:- pad, first_frame, video = unpack(video, video_packed_shape, "b c * h w")- first_frame = self.conv_in_first_frame(first_frame)+ # if encode_first_frame_separately:+ # pad, first_frame, video = unpack(video, video_packed_shape, "b c * h w")+ # first_frame = self.conv_in_first_frame(first_frame)video = self.conv_in(video)# print("pre-encoder:", video.size())- if encode_first_frame_separately:- video, _ = pack([first_frame, video], "b c * h w")- video = pad_at_dim(video, (self.time_padding, 0), dim=2)+ # if encode_first_frame_separately:+ # video, _ = pack([first_frame, video], "b c * h w")+ # video = pad_at_dim(video, (self.time_padding, 0), dim=2)encoded_feature = self.encoder(video)@@ -1020,9 +1023,9 @@ class VAE_3D_V2(nn.Module): # , ModelMixin# split = "train",):batch, channels, frames = video.shape[:3]- assert divisible_by(- frames - int(video_contains_first_frame), self.time_downsample_factor- ), f"number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}"+ # assert divisible_by(+ # frames - int(video_contains_first_frame), self.time_downsample_factor+ # ), f"number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}"posterior = self.encode(video,diff --git a/scripts/inference-vae.py b/scripts/inference-vae.pyindex bcef072..b9731d2 100644--- a/scripts/inference-vae.py+++ b/scripts/inference-vae.py@@ -5,7 +5,6 @@ import torchimport torch.distributed as distfrom colossalai.cluster import DistCoordinatorfrom colossalai.utils import get_current_device-from einops import rearrangefrom tqdm import tqdmfrom opensora.acceleration.parallel_states import get_data_parallel_group@@ -148,13 +147,7 @@ def main():for step in pbar:batch = next(dataloader_iter)x = batch["video"].to(device, dtype) # [B, C, T, H, W]-- is_image = x.ndim == 4- if is_image:- video = rearrange(x, "b c ... -> b c 1 ...")- video_contains_first_frame = True- else:- video = x+ video = x# ===== Spatial VAE =====if cfg.get("use_pipeline") == True: