Open-Sora: Democratizing Efficient Video Production for All
Python
debug
Shen-Chenhui5abb9239fba8
diff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.pyindex 15ea9a9..ad38e63 100644--- a/opensora/models/vae/vae_3d_v2.py+++ b/opensora/models/vae/vae_3d_v2.py@@ -317,16 +317,20 @@ class StyleGANDiscriminator(nn.Module):self.apply(xavier_uniform_weight_init)def forward(self, x):-++ breakpoint()x = self.conv1(x)x = self.activation_fn(x)+ breakpoint()for i in range(self.num_blocks):x = self.res_block_list[i](x)+ breakpoint()x = self.conv2(x)x = self.norm1(x)x = self.activation_fn(x)+ breakpoint()x = x.reshape((x.shape[0], -1)) # SCH: [B, (C * T * W * H)] ?x = self.linear1(x)@@ -429,16 +433,13 @@ class Encoder(nn.Module):for i in range(self.num_blocks):for j in range(self.num_res_blocks):- breakpoint()x = self.block_res_blocks[i][j](x)if i < self.num_blocks - 1:- breakpoint()x = self.conv_blocks[i](x)for i in range(self.num_res_blocks):- breakpoint()x = self.res_blocks[i](x)x = self.norm1(x)@@ -550,20 +551,17 @@ class Decoder(nn.Module):x,**kwargs,):- dtype, device = x.dtype, x.device+ # dtype, device = x.dtype, x.devicex = self.conv1(x)for i in range(self.num_res_blocks):- breakpoint()x = self.res_blocks[i](x)for i in reversed(range(self.num_blocks)): # reverse here to make decoder symmetric with encoderfor j in range(self.num_res_blocks):- breakpoint()x = self.block_res_blocks[i][j](x)if i > 0:t_stride = 2 if self.temporal_downsample[i - 1] else 1# SCH: T-Causal Conv 3x3x3, f -> (t_stride * 2 * 2) * f, depth to space t_stride x 2 x 2- breakpoint()x = self.conv_blocks[i-1](x)x = rearrange(x, "B (C ts hs ws) T H W -> B C (T ts) (H hs) (W ws)", ts=t_stride, hs=2, ws=2)@@ -635,6 +633,8 @@ class VAE_3D_V2(nn.Module):self.image_size = cast_tuple(image_size, 2)self.time_downsample_factor = 2**sum(temporal_downsample)self.time_padding = self.time_downsample_factor - 1+ self.disc_time_downsample_factor = 2**len(discriminator_channel_multipliers)+ self.disc_time_padding = self.disc_time_downsample_factor - 1self.separate_first_frame_encoding = separate_first_frame_encodingimage_down = 2 ** len(temporal_downsample)@@ -744,7 +744,6 @@ class VAE_3D_V2(nn.Module):video, _ = pack([first_frame, video], 'b c * h w')video = pad_at_dim(video, (self.time_padding, 0), dim = 2)- breakpoint()encoded_feature = self.encoder(video)moments = self.quant_conv(encoded_feature).to(video.dtype)@@ -761,7 +760,7 @@ class VAE_3D_V2(nn.Module):z = self.post_quant_conv(z)dec = self.decoder(z)- breakpoint()+# SCH: moved decoder last conv layer here for separate first frame decodingif decode_first_frame_separately:left_pad, dec_ff, dec = dec[:, :, :self.time_padding], dec[:, :, self.time_padding], dec[:, :, (self.time_padding + 1):]@@ -796,7 +795,6 @@ class VAE_3D_V2(nn.Module):batch, channels, frames = video.shape[:3]assert divisible_by(frames - int(video_contains_first_frame), self.time_downsample_factor), f'number of frames {frames} minus the first frame ({frames - int(video_contains_first_frame)}) must be divisible by the total downsample factor across time {self.time_downsample_factor}'- breakpoint()posterior = self.encode(video,video_contains_first_frame = video_contains_first_frame,@@ -807,14 +805,12 @@ class VAE_3D_V2(nn.Module):else:z = posterior.mode()- breakpoint()recon_video = self.decode(z,video_contains_first_frame = video_contains_first_frame)- breakpoint()recon_loss = F.mse_loss(video, recon_video)kl_loss = posterior.kl()@@ -825,9 +821,9 @@ class VAE_3D_V2(nn.Module):if self.adversarial_loss_weight is not None and self.adversarial_loss_weight > 0:if video_contains_first_frame:# video_len = video.shape[2]- # real_video = pad_at_dim(video, (self.time_padding, 0), value = 0., dim = 2)- # video_packed_shape = [torch.Size([self.time_padding]), torch.Size([]), torch.Size([video_len - 1])]- fake_video = pad_at_dim(recon_video, (self.time_padding, 0), value = 0., dim = 2)+ # real_video = pad_at_dim(video, (self.disc_time_padding, 0), value = 0., dim = 2)+ # video_packed_shape = [torch.Size([self.disc_time_padding]), torch.Size([]), torch.Size([video_len - 1])]+ fake_video = pad_at_dim(recon_video, (self.disc_time_padding, 0), value = 0., dim = 2)# real_logits = self.disc(real_video)fake_logits = self.disc(fake_video.detach())# dsicr_loss = hinge_discr_loss(fake_logits, real_logits)