Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[fix] micro batch size not for training

zhengzangwb89c072eb518
diff --git a/configs/vae/train/image.py b/configs/vae/train/image.py
index 8541c25..77ef011 100644
--- a/configs/vae/train/image.py
+++ b/configs/vae/train/image.py
@@ -25,7 +25,6 @@ model = dict(
type="VideoAutoencoderKL",
from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
subfolder="vae",
- micro_batch_size=4,
local_files_only=True,
),
vae_temporal=dict(
diff --git a/configs/vae/train/video.py b/configs/vae/train/video.py
index 5414242..b330059 100644
--- a/configs/vae/train/video.py
+++ b/configs/vae/train/video.py
@@ -19,13 +19,12 @@ plugin = "zero2"
# Define model
model = dict(
type="VideoAutoencoderPipeline",
- freeze_vae_2d=False,
+ freeze_vae_2d=True,
from_pretrained=None,
vae_2d=dict(
type="VideoAutoencoderKL",
from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
subfolder="vae",
- micro_batch_size=4,
local_files_only=True,
),
vae_temporal=dict(
diff --git a/opensora/models/vae/utils.py b/opensora/models/vae/utils.py
index fd57391..6c0c07c 100644
--- a/opensora/models/vae/utils.py
+++ b/opensora/models/vae/utils.py
@@ -1,10 +1,8 @@
import numpy as np
import torch
-import torch.nn as nn
# from taming.modules.losses.lpips import LPIPS # need to pip install https://github.com/CompVis/taming-transformers
# from taming.modules.discriminator.model import NLayerDiscriminator, weights_init
-from einops import rearrange
"""Stripped version of https://github.com/richzhang/PerceptualSimilarity/tree/master/models"""
@@ -30,7 +28,7 @@ class DiagonalGaussianDistribution(object):
deterministic=False,
):
self.parameters = parameters
- self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) # SCH: channels dim
+ self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)
self.logvar = torch.clamp(self.logvar, -30.0, 20.0)
self.deterministic = deterministic
self.std = torch.exp(0.5 * self.logvar)
@@ -65,6 +63,5 @@ class DiagonalGaussianDistribution(object):
logtwopi = np.log(2.0 * np.pi)
return 0.5 * torch.sum(logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var, dim=dims)
- def mode(self): # SCH: used for vae inference?
+ def mode(self):
return self.mean
-
diff --git a/opensora/models/vae/vae.py b/opensora/models/vae/vae.py
index c54aa60..e7912e7 100644
--- a/opensora/models/vae/vae.py
+++ b/opensora/models/vae/vae.py
@@ -31,6 +31,7 @@ class VideoAutoencoderKL(nn.Module):
if self.micro_batch_size is None:
x = self.module.encode(x).latent_dist.sample().mul_(0.18215)
else:
+ # NOTE: cannot be used for training
bs = self.micro_batch_size
x_out = []
for i in range(0, x.shape[0], bs):
@@ -48,6 +49,7 @@ class VideoAutoencoderKL(nn.Module):
if self.micro_batch_size is None:
x = self.module.decode(x / 0.18215).sample
else:
+ # NOTE: cannot be used for training
bs = self.micro_batch_size
x_out = []
for i in range(0, x.shape[0], bs):