Open-Sora: Democratizing Efficient Video Production for All
Python
[fix] micro batch size not for training
zhengzangwb89c072eb518
diff --git a/configs/vae/train/image.py b/configs/vae/train/image.pyindex 8541c25..77ef011 100644--- a/configs/vae/train/image.py+++ b/configs/vae/train/image.py@@ -25,7 +25,6 @@ model = dict(type="VideoAutoencoderKL",from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",subfolder="vae",- micro_batch_size=4,local_files_only=True,),vae_temporal=dict(diff --git a/configs/vae/train/video.py b/configs/vae/train/video.pyindex 5414242..b330059 100644--- a/configs/vae/train/video.py+++ b/configs/vae/train/video.py@@ -19,13 +19,12 @@ plugin = "zero2"# Define modelmodel = dict(type="VideoAutoencoderPipeline",- freeze_vae_2d=False,+ freeze_vae_2d=True,from_pretrained=None,vae_2d=dict(type="VideoAutoencoderKL",from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",subfolder="vae",- micro_batch_size=4,local_files_only=True,),vae_temporal=dict(diff --git a/opensora/models/vae/utils.py b/opensora/models/vae/utils.pyindex fd57391..6c0c07c 100644--- a/opensora/models/vae/utils.py+++ b/opensora/models/vae/utils.py@@ -1,10 +1,8 @@import numpy as npimport torch-import torch.nn as nn# from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers# from taming.modules.discriminator.model import NLayerDiscriminator, weights_init-from einops import rearrange"""Stripped version of https://github.com/richzhang/PerceptualSimilarity/tree/master/models"""@@ -30,7 +28,7 @@ class DiagonalGaussianDistribution(object):deterministic=False,):self.parameters = parameters- self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) # SCH: channels dim+ self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)self.logvar = torch.clamp(self.logvar, -30.0, 20.0)self.deterministic = deterministicself.std = torch.exp(0.5 * self.logvar)@@ -65,6 +63,5 @@ class DiagonalGaussianDistribution(object):logtwopi = np.log(2.0 * np.pi)return 0.5 * torch.sum(logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var, dim=dims)- def mode(self): # SCH: used for vae inference?+ def mode(self):return self.mean-diff --git a/opensora/models/vae/vae.py b/opensora/models/vae/vae.pyindex c54aa60..e7912e7 100644--- a/opensora/models/vae/vae.py+++ b/opensora/models/vae/vae.py@@ -31,6 +31,7 @@ class VideoAutoencoderKL(nn.Module):if self.micro_batch_size is None:x = self.module.encode(x).latent_dist.sample().mul_(0.18215)else:+ # NOTE: cannot be used for trainingbs = self.micro_batch_sizex_out = []for i in range(0, x.shape[0], bs):@@ -48,6 +49,7 @@ class VideoAutoencoderKL(nn.Module):if self.micro_batch_size is None:x = self.module.decode(x / 0.18215).sampleelse:+ # NOTE: cannot be used for trainingbs = self.micro_batch_sizex_out = []for i in range(0, x.shape[0], bs):