Open-Sora: Democratizing Efficient Video Production for All
Python
strides debug
Shen-Chenhui1bbef18f15e9
diff --git a/opensora/models/vae/vae_3d.py b/opensora/models/vae/vae_3d.pyindex 29abaec..1907fb4 100644--- a/opensora/models/vae/vae_3d.py+++ b/opensora/models/vae/vae_3d.py@@ -161,7 +161,7 @@ class Encoder(nn.Module):if self.conv_downsample:t_stride = 2 if self.temporal_downsample[i] else 1t_pad = 1 if self.temporal_downsample[i] else 0- self.conv_blocks.append(self.conv_fn(prev_filters, filters, kernel_size=(4, 4, 4), strides=(t_stride, 2, 2)), padding=(t_pad,1,1)) # SCH: should be same in_channel and out_channel+ self.conv_blocks.append(self.conv_fn(prev_filters, filters, kernel_size=(4, 4, 4), stride=(t_stride, 2, 2)), padding=(t_pad,1,1)) # SCH: should be same in_channel and out_channelprev_filters = filters # update in_channels# NOTE: downsample, dimensions T, H, Wdiff --git a/opensora/models/vae/vae_3d_v2.py b/opensora/models/vae/vae_3d_v2.pyindex a067304..5d0c806 100644--- a/opensora/models/vae/vae_3d_v2.py+++ b/opensora/models/vae/vae_3d_v2.py@@ -39,6 +39,7 @@ class CausalConv3d(nn.Module):chan_out,kernel_size: Union[int, Tuple[int, int, int]],pad_mode = 'constant',+ strides = None, # allow custom stride**kwargs):super().__init__()@@ -49,7 +50,7 @@ class CausalConv3d(nn.Module):assert is_odd(height_kernel_size) and is_odd(width_kernel_size)dilation = kwargs.pop('dilation', 1)- stride = kwargs.pop('stride', 1)+ stride = strides[0] if strides is not None else kwargs.pop('stride', 1)self.pad_mode = pad_modetime_pad = dilation * (time_kernel_size - 1) + (1 - stride)@@ -59,7 +60,7 @@ class CausalConv3d(nn.Module):self.time_pad = time_padself.time_causal_padding = (width_pad, width_pad, height_pad, height_pad, time_pad, 0)- stride = (stride, 1, 1)+ stride = strides if strides is not None else (stride, 1, 1)dilation = (dilation, 1, 1)self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride = stride, dilation = dilation, **kwargs)@@ -69,7 +70,6 @@ class CausalConv3d(nn.Module):x = nn.F.pad(x, self.time_causal_padding, mode = pad_mode)return self.conv(x)-# TODO: CausalConvTranspose3dclass ResBlock(nn.Module):def __init__(@@ -187,6 +187,7 @@ class Encoder(nn.Module):if i < self.num_blocks - 1: # SCH: T-Causal Conv 3x3x3, 128->128, stride t x 2 x 2t_stride = 2 if self.temporal_downsample[i] else 1+ # TODO: conv_fn usess default stride t x 1 x 1, cannotself.conv_blocks.append(self.conv_fn(prev_filters, filters, kernel_size=(3, 3, 3), strides=(t_stride, 2, 2))) # SCH: should be same in_channel and out_channelprev_filters = filters # update in_channels@@ -200,7 +201,6 @@ class Encoder(nn.Module):# MAGVIT uses Group Normalizationself.norm1 = nn.GroupNorm(self.num_groups, prev_filters, dtype=dtype, device=device) # SCH: separate <prev_filters> channels into 32 groups- # TODO: check if this is correct ??self.conv2 = nn.Conv3d(prev_filters, self.embedding_dim, kernel_size=(1, 1, 1), dtype=dtype, device=device, padding="same")def forward(self, x):