Open-Sora: Democratizing Efficient Video Production for All
Python
[feature] support normal sample without cfg (#16)
Hongxin Liu0e60f3375fd6
diff --git a/open_sora/modeling/dit.py b/open_sora/modeling/dit.pyindex dfcb640..5710abd 100644--- a/open_sora/modeling/dit.py+++ b/open_sora/modeling/dit.py@@ -237,7 +237,7 @@ class TextEmbedder(nn.Module):self.proj = nn.Linear(in_features, embed_dim, bias=bias)def drop_sample(self, x: torch.Tensor) -> torch.Tensor:- drop_ids = torch.rand(x.shape[0], device=x.device) < self.dropout_prob+ drop_ids = torch.rand(x.shape[0], 1, 1, device=x.device) < self.dropout_probx = torch.where(drop_ids, torch.zeros_like(x), x)return x@@ -517,13 +517,18 @@ class DiT(nn.Module):# three channels by default. The standard approach to cfg applies it to all channels.# This can be done by uncommenting the following line and commenting-out the line following that.# eps, rest = model_out[:, :self.in_channels], model_out[:, self.in_channels:]- c = model_out.shape[2]- assert c == 2 * self.in_channels- eps, rest = model_out.chunk(2, dim=2)+ if self.learn_sigma:+ c = model_out.shape[2]+ assert c == 2 * self.in_channels+ eps, rest = model_out.chunk(2, dim=2)+ else:+ eps = model_outcond_eps, uncond_eps = torch.split(eps, len(eps) // 2, dim=0)half_eps = uncond_eps + cfg_scale * (cond_eps - uncond_eps)eps = torch.cat([half_eps, half_eps], dim=0)- return torch.cat([eps, rest], dim=2)+ if self.learn_sigma:+ return torch.cat([eps, rest], dim=2)+ return eps#################################################################################diff --git a/open_sora/utils/data.py b/open_sora/utils/data.pyindex 640f723..832a6f0 100644--- a/open_sora/utils/data.py+++ b/open_sora/utils/data.py@@ -7,8 +7,10 @@ import torch.nn.functional as Ffrom colossalai.utils import get_current_devicefrom datasets import Dataset as HFDatasetfrom datasets import dataset_dict, load_from_disk+from diffusers.models import AutoencoderKLfrom torch.utils.data import ConcatDataset, Datasetfrom torchvision.io import read_video+from transformers import AutoModelDatasetType = Union[Dataset, ConcatDataset, dataset_dict.Dataset]PathType = Union[str, os.PathLike]@@ -155,11 +157,116 @@ def unnormalize_video(video: torch.Tensor) -> torch.Tensor:return (video + 0.5) * 255+class VideoCompressor:+ t_factor: int+ h_w_factor: int+ out_channels: int++ def encode(self, video: torch.Tensor) -> torch.Tensor:+ """Encode a video.++ Args:+ video (torch.Tensor): [T, H, W, C]++ Returns:+ torch.Tensor: [T, C, H, W]+ """+ raise NotImplementedError++ def decode(self, latent: torch.Tensor) -> torch.Tensor:+ """Decode a latent tensor.++ Args:+ latent (torch.Tensor): [T, C, H, W]++ Returns:+ torch.Tensor: [T, H, W, C]+ """+ raise NotImplementedError+++class RawVideoCompressor(VideoCompressor):+ t_factor = 1+ h_w_factor = 1+ out_channels = 3++ def encode(self, video: torch.Tensor) -> torch.Tensor:+ # [T, H, W, C] -> [T, C, H, W]+ return video.permute(0, 3, 1, 2).contiguous()++ def decode(self, latent: torch.Tensor) -> torch.Tensor:+ # [T, C, H, W] -> [T, H, W, C]+ return latent.permute(0, 2, 3, 1).contiguous()+++class VqvaeVideoCompressor(VideoCompressor):+ t_factor = 2+ h_w_factor = 4++ def __init__(self, vqvae: nn.Module):+ self.vqvae = vqvae+ self.out_channels = vqvae.embedding_dim++ def encode(self, video: torch.Tensor) -> torch.Tensor:+ # [T, H, W, C] -> [B, C, T, H, W]+ video = video.permute(3, 0, 1, 2).unsqueeze(0)+ latent_indices, embeddings = self.vqvae.encode(video, include_embeddings=True)+ # [B, C, T, H, W] -> [T, C, H, W]+ return embeddings.squeeze(0).permute(1, 0, 2, 3)++ def decode(self, latent: torch.Tensor) -> torch.Tensor:+ # [T, C, H, W] -> [B, C, T, H, W]+ latent = latent.permute(1, 0, 2, 3).unsqueeze(0)+ video = self.vqvae.decode_from_embeddings(latent)+ # [B, C, T, H, W] -> [T, H, W, C]+ video = video.squeeze(0).permute(1, 2, 3, 0)+ return video+++class VaeVideoCompressor(VideoCompressor):+ t_factor = 1+ h_w_factor = 8+ out_channels = 4++ def __init__(self, vae: nn.Module):+ self.vae = vae++ def encode(self, video: torch.Tensor) -> torch.Tensor:+ # [T, H, W, C] -> [T, C, H, W]+ video = video.permute(0, 3, 1, 2)+ return self.vae.encode(video).latent_dist.sample()++ def decode(self, latent: torch.Tensor) -> torch.Tensor:+ video = self.vae.decode(latent).sample+ # [T, C, H, W] -> [T, H, W, C]+ return video.permute(0, 2, 3, 1).contiguous()+++def create_video_compressor(+ compressor_type: str,+ vqvae_path="hpcai-tech/vqvae",+ vae_path="stabilityai/sd-vae-ft-mse",+) -> VideoCompressor:+ if compressor_type == "raw":+ return RawVideoCompressor()+ if compressor_type == "vqvae":+ vqvae = (+ AutoModel.from_pretrained(vqvae_path, trust_remote_code=True)+ .to(get_current_device())+ .eval()+ )+ return VqvaeVideoCompressor(vqvae)+ if compressor_type == "vae":+ vae = AutoencoderKL.from_pretrained(vae_path).to(get_current_device()).eval()+ return VaeVideoCompressor(vae)+ raise ValueError(f"Unsupported video compressor type {compressor_type}")++@torch.no_grad()def preprocess_batch(batch: dict,patch_size: int,- vqvae: Optional[nn.Module] = None,+ video_compressor: VideoCompressor,device=None,use_cross_attn=True,) -> dict:@@ -169,18 +276,8 @@ def preprocess_batch(for video in batch.pop("videos"):video = video.to(device)video = normalize_video(video)- if vqvae is not None:- # [T, H, W, C] -> [B, C, T, H, W]- video = video.permute(3, 0, 1, 2)- video = video.unsqueeze(0)- latent_indices, embeddings = vqvae.encode(video, include_embeddings=True)- # [B, C, T, H, W] -> [T, C, H, W]- embeddings = embeddings.squeeze(0).permute(1, 0, 2, 3)- videos.append(embeddings)- else:- # [T, H, W, C] -> [T, C, H, W]- video = video.permute(0, 3, 1, 2).contiguous()- videos.append(video)+ video = video_compressor.encode(video)+ videos.append(video)video_latent_states, video_padding_mask = patchify_batch(videos, patch_size)batch["video_latent_states"] = video_latent_statesbatch["video_padding_mask"] = video_padding_maskdiff --git a/sample.py b/sample.pyindex ffb4f2f..5acd53a 100644--- a/sample.py+++ b/sample.py@@ -15,11 +15,11 @@ import argparsefrom colossalai.utils import get_current_devicefrom torchvision.io import write_video-from transformers import AutoModel, AutoTokenizer, CLIPTextModel+from transformers import AutoTokenizer, CLIPTextModelfrom open_sora.diffusion import create_diffusionfrom open_sora.modeling import DiT_models-from open_sora.utils.data import col2video, unnormalize_video+from open_sora.utils.data import col2video, create_video_compressor, unnormalize_videodef main(args):@@ -27,25 +27,14 @@ def main(args):torch.manual_seed(args.seed)torch.set_grad_enabled(False)device = get_current_device()- if len(args.vqvae) > 0:- vqvae = (- AutoModel.from_pretrained(args.vqvae, trust_remote_code=True)- .to(device)- .eval()- )- in_channels = vqvae.embedding_dim- w_h_factor = 4- t_factor = 2- else:- # disable VQ-VAE if not provided, just use raw video frames- vqvae = None- in_channels = 3- w_h_factor = 1- t_factor = 1++ video_compressor = create_video_compressor(args.compressor)+ model_kwargs = {"in_channels": video_compressor.out_channels}+text_model = CLIPTextModel.from_pretrained(args.text_model).to(device).eval()tokenizer = AutoTokenizer.from_pretrained(args.text_model)- model = DiT_models[args.model](in_channels=in_channels).to(device).eval()+ model = DiT_models[args.model](**model_kwargs).to(device).eval()patch_size = model.patch_sizemodel.load_state_dict(torch.load(args.ckpt))diffusion = create_diffusion(str(args.num_sampling_steps))@@ -58,10 +47,10 @@ def main(args):num_frames = args.fps * args.secz = torch.randn(1,- (args.height // patch_size // w_h_factor)- * (args.width // patch_size // w_h_factor)- * (num_frames // t_factor),- in_channels,+ (args.height // patch_size // video_compressor.h_w_factor)+ * (args.width // patch_size // video_compressor.h_w_factor)+ * (num_frames // video_compressor.t_factor),+ video_compressor.out_channels,patch_size,patch_size,device=device,@@ -69,18 +58,26 @@ def main(args):# Setup classifier-free guidance:model_kwargs = {}- z = torch.cat([z, z], 0)- model_kwargs["text_latent_states"] = torch.cat(- [text_latent_states, torch.zeros_like(text_latent_states)], 0- )- model_kwargs["cfg_scale"] = args.cfg_scale+ if not args.disable_cfg:+ z = torch.cat([z, z], 0)+ model_kwargs["text_latent_states"] = torch.cat(+ [text_latent_states, torch.zeros_like(text_latent_states)], 0+ )+ model_kwargs["cfg_scale"] = args.cfg_scale+ else:+ model_kwargs["text_latent_states"] = text_latent_statesmodel_kwargs["attention_mask"] = torch.ones(- 2, 1, z.shape[1], text_latent_states.shape[1], device=device, dtype=torch.int+ z.shape[0],+ 1,+ z.shape[1],+ text_latent_states.shape[1],+ device=device,+ dtype=torch.int,)# Sample images:samples = diffusion.p_sample_loop(- model.forward_with_cfg,+ model if args.disable_cfg else model.forward_with_cfg,z.shape,z,clip_denoised=False,@@ -88,25 +85,18 @@ def main(args):progress=True,device=device,)- samples, _ = samples.chunk(2, dim=0) # Remove null class samples+ if not args.disable_cfg:+ samples, _ = samples.chunk(2, dim=0) # Remove null class samplessamples = col2video(samples.squeeze(),(- num_frames // t_factor,- in_channels,- args.height // w_h_factor,- args.width // w_h_factor,+ num_frames // video_compressor.t_factor,+ video_compressor.out_channels,+ args.height // video_compressor.h_w_factor,+ args.width // video_compressor.h_w_factor,),)- if vqvae is not None:- # [T, C, H, W] -> [B, C, T, H, W]- samples = samples.permute(1, 0, 2, 3).unsqueeze(0)- samples = vqvae.decode_from_embeddings(samples)- # [B, C, T, H, W] -> [T, H, W, C]- samples = samples.squeeze(0).permute(1, 2, 3, 0)- else:- # [T, C, H, W] -> [T, H, W, C]- samples = samples.permute(0, 2, 3, 1)+ samples = video_compressor.decode(samples)samples = unnormalize_video(samples).to(torch.uint8)write_video("sample.mp4", samples.cpu(), args.fps)@@ -131,7 +121,9 @@ if __name__ == "__main__":required=True,help="Optional path to a DiT checkpoint (default: auto-download a pre-trained DiT-XL/2 model).",)- parser.add_argument("--vqvae", default="hpcai-tech/vqvae")+ parser.add_argument(+ "-c", "--compressor", choices=["raw", "vqvae", "vae"], default="raw"+ )parser.add_argument("--text_model", type=str, default="openai/clip-vit-base-patch32")@@ -139,5 +131,6 @@ if __name__ == "__main__":parser.add_argument("--height", type=int, default=320)parser.add_argument("--fps", type=int, default=15)parser.add_argument("--sec", type=int, default=8)+ parser.add_argument("--disable-cfg", action="store_true", default=False)args = parser.parse_args()main(args)diff --git a/train.py b/train.pyindex ad2ff74..82cfb33 100644--- a/train.py+++ b/train.py@@ -20,14 +20,19 @@ from colossalai.booster import Boosterfrom colossalai.booster.plugin import LowLevelZeroPluginfrom colossalai.cluster import DistCoordinatorfrom colossalai.logging import get_dist_logger+from colossalai.nn.lr_scheduler import CosineAnnealingLRfrom colossalai.utils import get_current_devicefrom torch.utils.tensorboard import SummaryWriterfrom tqdm import tqdm-from transformers import AutoModelfrom open_sora.diffusion import create_diffusionfrom open_sora.modeling import DiT_models-from open_sora.utils.data import load_datasets, make_batch, preprocess_batch+from open_sora.utils.data import (+ create_video_compressor,+ load_datasets,+ make_batch,+ preprocess_batch,+)################################################################################## Training Helper Functions #@@ -99,18 +104,9 @@ def main(args):os.makedirs(args.tensorboard_dir, exist_ok=True)writer = SummaryWriter(args.tensorboard_dir)- # Step 3: Create VQ-VAE- if len(args.vqvae) > 0:- vqvae = (- AutoModel.from_pretrained(args.vqvae, trust_remote_code=True)- .to(get_current_device())- .eval()- )- model_kwargs = {"in_channels": vqvae.embedding_dim}- else:- # disable VQ-VAE if not provided, just use raw video frames- vqvae = None- model_kwargs = {}+ # Step 3: Create video compressor+ video_compressor = create_video_compressor(args.compressor)+ model_kwargs = {"in_channels": video_compressor.out_channels}# Step 4: Create DiT and EMAmodel = DiT_models[args.model](**model_kwargs).to(get_current_device())@@ -142,6 +138,9 @@ def main(args):shuffle=True,drop_last=True,)+ lr_scheduler = CosineAnnealingLR(+ opt, args.epochs * len(dataloader) // args.accumulation_steps+ )logger.info(f"Dataset contains {len(dataset)} samples", ranks=[0])# Step 8: setup booster@@ -167,7 +166,7 @@ def main(args):) as pbar:total_loss = torch.tensor(0.0, device=get_current_device())for step, batch in enumerate(dataloader):- batch = preprocess_batch(batch, patch_size, vqvae)+ batch = preprocess_batch(batch, patch_size, video_compressor)video_inputs = batch.pop("video_latent_states")mask = batch.pop("video_padding_mask")t = torch.randint(@@ -186,6 +185,7 @@ def main(args):if (step + 1) % args.accumulation_steps == 0:opt.step()opt.zero_grad()+ lr_scheduler.step()update_ema(ema, model)all_reduce_mean(total_loss)@@ -239,7 +239,9 @@ if __name__ == "__main__":parser.add_argument("--save_interval", type=int, default=20)parser.add_argument("--checkpoint_dir", type=str, default="checkpoints")parser.add_argument("--tensorboard_dir", type=str, default="runs")- parser.add_argument("--vqvae", default="hpcai-tech/vqvae")+ parser.add_argument(+ "-c", "--compressor", choices=["raw", "vqvae", "vae"], default="raw"+ )parser.add_argument("--load_model", default=None)parser.add_argument("--load_optimizer", default=None)args = parser.parse_args()