Open-Sora: Democratizing Efficient Video Production for All
Python
added open_sora package (#66)
xyupeng9aab3ad343fe
diff --git a/.gitignore b/.gitignoreindex 47e4823..b5fc089 100644--- a/.gitignore+++ b/.gitignore@@ -157,7 +157,7 @@ cython_debug/# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore# and can be added to the global gitignore or merged into this file. For a more nuclear# option (not recommended) you can uncomment the following to ignore the entire idea folder.-#.idea/+.idea/.vscode/# misc filesdiff --git a/opensora/acceleration/__init__.py b/opensora/acceleration/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/opensora/acceleration/checkpoint.py b/opensora/acceleration/checkpoint.pynew file mode 100644index 0000000..d832a01--- /dev/null+++ b/opensora/acceleration/checkpoint.py@@ -0,0 +1,24 @@+from collections.abc import Iterable++import torch.nn as nn+from torch.utils.checkpoint import checkpoint, checkpoint_sequential+++def set_grad_checkpoint(model, use_fp32_attention=False, gc_step=1):+ assert isinstance(model, nn.Module)++ def set_attr(module):+ module.grad_checkpointing = True+ module.fp32_attention = use_fp32_attention+ module.grad_checkpointing_step = gc_step++ model.apply(set_attr)+++def auto_grad_checkpoint(module, *args, **kwargs):+ if getattr(module, "grad_checkpointing", False):+ if not isinstance(module, Iterable):+ return checkpoint(module, *args, **kwargs)+ gc_step = module[0].grad_checkpointing_step+ return checkpoint_sequential(module, gc_step, *args, **kwargs)+ return module(*args, **kwargs)diff --git a/opensora/acceleration/communications.py b/opensora/acceleration/communications.pynew file mode 100644index 0000000..b07d4c0--- /dev/null+++ b/opensora/acceleration/communications.py@@ -0,0 +1,187 @@+import torch+import torch.distributed as dist+++# ====================+# All-To-All+# ====================+def _all_to_all(+ input_: torch.Tensor,+ world_size: int,+ group: dist.ProcessGroup,+ scatter_dim: int,+ gather_dim: int,+):+ input_list = [+ t.contiguous() for t in torch.tensor_split(input_, world_size, scatter_dim)+ ]+ output_list = [torch.empty_like(input_list[0]) for _ in range(world_size)]+ dist.all_to_all(output_list, input_list, group=group)+ return torch.cat(output_list, dim=gather_dim).contiguous()++class _AllToAll(torch.autograd.Function):+ """All-to-all communication.++ Args:+ input_: input matrix+ process_group: communication group+ scatter_dim: scatter dimension+ gather_dim: gather dimension+ """++ @staticmethod+ def forward(ctx, input_, process_group, scatter_dim, gather_dim):+ ctx.process_group = process_group+ ctx.scatter_dim = scatter_dim+ ctx.gather_dim = gather_dim+ ctx.world_size = dist.get_world_size(process_group)+ output = _all_to_all(+ input_, ctx.world_size, process_group, scatter_dim, gather_dim+ )+ return output++ @staticmethod+ def backward(ctx, grad_output):+ grad_output = _all_to_all(+ grad_output,+ ctx.world_size,+ ctx.process_group,+ ctx.gather_dim,+ ctx.scatter_dim,+ )+ return (+ grad_output,+ None,+ None,+ None,+ )+++def all_to_all(+ input_: torch.Tensor,+ process_group: dist.ProcessGroup,+ scatter_dim: int = 2,+ gather_dim: int = 1,+):+ return _AllToAll.apply(input_, process_group, scatter_dim, gather_dim)++def _gather(+ input_: torch.Tensor,+ world_size: int,+ group: dist.ProcessGroup,+ gather_dim: int,+):+ if gather_list is None:+ gather_list = [torch.empty_like(input_) for _ in range(world_size)]+ dist.gather(input_, gather_list, group=group, gather_dim=gather_dim)+ return gather_list++# ====================+# Gather-Split+# ====================++def _split(input_, pg: dist.ProcessGroup, dim=-1):+ # skip if only one rank involved+ world_size = dist.get_world_size(pg)+ rank = dist.get_rank(pg)+ if world_size == 1:+ return input_++ # Split along last dimension.+ dim_size = input_.size(dim)+ assert dim_size % world_size == 0, (+ f"The dimension to split ({dim_size}) is not a multiple of world size ({world_size}), "+ f"cannot split tensor evenly"+ )++ tensor_list = torch.split(input_, dim_size // world_size, dim=dim)+ output = tensor_list[rank].contiguous()++ return output+++def _gather(input_, pg: dist.ProcessGroup, dim=-1):+ # skip if only one rank involved+ input_ = input_.contiguous()+ world_size = dist.get_world_size(pg)+ rank = dist.get_rank(pg)++ if world_size == 1:+ return input_++ # all gather+ tensor_list = [torch.empty_like(input_) for _ in range(world_size)]+ assert input_.device.type == "cuda"+ torch.distributed.all_gather(tensor_list, input_, group=pg)++ # concat+ output = torch.cat(tensor_list, dim=dim).contiguous()++ return output++class _GatherForwardSplitBackward(torch.autograd.Function):+ """Gather the input from model parallel region and concatenate.++ Args:+ input_: input matrix.+ process_group: parallel mode.+ dim: dimension+ """++ @staticmethod+ def symbolic(graph, input_):++ return _gather(input_)++ @staticmethod+ def forward(ctx, input_, process_group, dim, grad_scale):+ ctx.mode = process_group+ ctx.dim = dim+ ctx.grad_scale = grad_scale+ return _gather(input_, process_group, dim)++ @staticmethod+ def backward(ctx, grad_output):+ if ctx.grad_scale == "up":+ grad_output = grad_output * dist.get_world_size(ctx.mode)+ elif ctx.grad_scale == "down":+ grad_output = grad_output / dist.get_world_size(ctx.mode)++ return _split(grad_output, ctx.mode, ctx.dim), None, None, None+++class _SplitForwardGatherBackward(torch.autograd.Function):+ """+ Split the input and keep only the corresponding chuck to the rank.++ Args:+ input_: input matrix.+ process_group: parallel mode.+ dim: dimension+ """++ @staticmethod+ def symbolic(graph, input_):+ return _split(input_)++ @staticmethod+ def forward(ctx, input_, process_group, dim, grad_scale):+ ctx.mode = process_group+ ctx.dim = dim+ ctx.grad_scale = grad_scale+ return _split(input_, process_group, dim)++ @staticmethod+ def backward(ctx, grad_output):+ if ctx.grad_scale == "up":+ grad_output = grad_output * dist.get_world_size(ctx.mode)+ elif ctx.grad_scale == "down":+ grad_output = grad_output / dist.get_world_size(ctx.mode)+ return _gather(grad_output, ctx.mode, ctx.dim), None, None, None+++def split_forward_gather_backward(input_, process_group, dim, grad_scale=1.0):+ return _SplitForwardGatherBackward.apply(input_, process_group, dim, grad_scale)++def gather_forward_split_backward(input_, process_group, dim, grad_scale=None):+ return _GatherForwardSplitBackward.apply(input_, process_group, dim, grad_scale)diff --git a/opensora/acceleration/parallel_states.py b/opensora/acceleration/parallel_states.pynew file mode 100644index 0000000..a8d71e8--- /dev/null+++ b/opensora/acceleration/parallel_states.py@@ -0,0 +1,15 @@+import torch.distributed as dist++_GLOBAL_PARALLEL_GROUPS = dict()++def set_data_parallel_group(group: dist.ProcessGroup):+ _GLOBAL_PARALLEL_GROUPS["data"] = group++def get_data_parallel_group():+ return _GLOBAL_PARALLEL_GROUPS.get("data", None)++def set_sequence_parallel_group(group: dist.ProcessGroup):+ _GLOBAL_PARALLEL_GROUPS["sequence"] = group++def get_sequence_parallel_group():+ return _GLOBAL_PARALLEL_GROUPS.get("sequence", None)\ No newline at end of filediff --git a/opensora/acceleration/plugin.py b/opensora/acceleration/plugin.pynew file mode 100644index 0000000..7c9be5d--- /dev/null+++ b/opensora/acceleration/plugin.py@@ -0,0 +1,102 @@+import random+from typing import Optional++import numpy as np+import torch+from colossalai.booster.plugin import LowLevelZeroPlugin+from colossalai.cluster import ProcessGroupMesh+from torch.utils.data import DataLoader+from torch.utils.data.distributed import DistributedSampler++DP_AXIS, SP_AXIS = 0, 1+++class ZeroSeqParallelPlugin(LowLevelZeroPlugin):+ def __init__(+ self,+ sp_size: int = 1,+ stage: int = 2,+ precision: str = "fp16",+ initial_scale: float = 2**32,+ min_scale: float = 1,+ growth_factor: float = 2,+ backoff_factor: float = 0.5,+ growth_interval: int = 1000,+ hysteresis: int = 2,+ max_scale: float = 2**32,+ max_norm: float = 0.0,+ norm_type: float = 2.0,+ reduce_bucket_size_in_m: int = 12,+ communication_dtype: Optional[torch.dtype] = None,+ overlap_communication: bool = True,+ cpu_offload: bool = False,+ master_weights: bool = True,+ verbose: bool = False,+ ) -> None:+ super().__init__(+ stage=stage,+ precision=precision,+ initial_scale=initial_scale,+ min_scale=min_scale,+ growth_factor=growth_factor,+ backoff_factor=backoff_factor,+ growth_interval=growth_interval,+ hysteresis=hysteresis,+ max_scale=max_scale,+ max_norm=max_norm,+ norm_type=norm_type,+ reduce_bucket_size_in_m=reduce_bucket_size_in_m,+ communication_dtype=communication_dtype,+ overlap_communication=overlap_communication,+ cpu_offload=cpu_offload,+ master_weights=master_weights,+ verbose=verbose,+ )+ self.sp_size = sp_size+ assert self.world_size % sp_size == 0, "world_size must be divisible by sp_size"+ self.dp_size = self.world_size // sp_size+ self.pg_mesh = ProcessGroupMesh(self.dp_size, self.sp_size)+ self.dp_group = self.pg_mesh.get_group_along_axis(DP_AXIS)+ self.sp_group = self.pg_mesh.get_group_along_axis(SP_AXIS)+ self.dp_rank = self.pg_mesh.coordinate(DP_AXIS)+ self.sp_rank = self.pg_mesh.coordinate(SP_AXIS)++ def __del__(self):+ """Destroy the prcess groups in ProcessGroupMesh"""+ self.pg_mesh.destroy_mesh_process_groups()++ def prepare_dataloader(+ self,+ dataset,+ batch_size,+ shuffle=False,+ seed=1024,+ drop_last=False,+ pin_memory=False,+ num_workers=0,+ distributed_sampler_cls=None,+ **kwargs+ ):+ _kwargs = kwargs.copy()+ distributed_sampler_cls = distributed_sampler_cls or DistributedSampler+ sampler = distributed_sampler_cls(+ dataset, num_replicas=self.dp_size, rank=self.dp_rank, shuffle=shuffle+ )++ # Deterministic dataloader+ def seed_worker(worker_id):+ worker_seed = seed+ np.random.seed(worker_seed)+ torch.manual_seed(worker_seed)+ random.seed(worker_seed)++ return DataLoader(+ dataset,+ batch_size=batch_size,+ sampler=sampler,+ worker_init_fn=seed_worker,+ drop_last=drop_last,+ pin_memory=pin_memory,+ num_workers=num_workers,+ **_kwargs,+ )\ No newline at end of filediff --git a/opensora/acceleration/shardformer/modeling/__init__.py b/opensora/acceleration/shardformer/modeling/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/opensora/acceleration/shardformer/modeling/t5.py b/opensora/acceleration/shardformer/modeling/t5.pynew file mode 100644index 0000000..d494c1c--- /dev/null+++ b/opensora/acceleration/shardformer/modeling/t5.py@@ -0,0 +1,38 @@+import torch+import torch.nn as nn+++class T5LayerNorm(nn.Module):+ def __init__(self, hidden_size, eps=1e-6):+ """+ Construct a layernorm module in the T5 style. No bias and no subtraction of mean.+ """+ super().__init__()+ self.weight = nn.Parameter(torch.ones(hidden_size))+ self.variance_epsilon = eps++ def forward(self, hidden_states):+ # T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean+ # Square Layer Normalization https://arxiv.org/abs/1910.07467 thus varience is calculated+ # w/o mean and there is no bias. Additionally we want to make sure that the accumulation for+ # half-precision inputs is done in fp32++ variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)++ # convert into half-precision if necessary+ if self.weight.dtype in [torch.float16, torch.bfloat16]:+ hidden_states = hidden_states.to(self.weight.dtype)++ return self.weight * hidden_states++ @staticmethod+ def from_native_module(module, *args, **kwargs):+ assert module.__class__.__name__ == "FusedRMSNorm", \+ "Recovering T5LayerNorm requires the original layer to be apex's Fused RMS Norm." \+ "Apex's fused norm is automatically used by Hugging Face Transformers https://github.com/huggingface/transformers/blob/main/src/transformers/models/t5/modeling_t5.py#L265C5-L265C48"++ layer_norm = T5LayerNorm(module.normalized_shape, eps=module.eps)+ layer_norm.weight.data.copy_(module.weight.data)+ layer_norm = layer_norm.to(module.weight.device)+ return layer_normdiff --git a/opensora/acceleration/shardformer/policy/__init__.py b/opensora/acceleration/shardformer/policy/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/opensora/acceleration/shardformer/policy/t5_encoder.py b/opensora/acceleration/shardformer/policy/t5_encoder.pynew file mode 100644index 0000000..e7ce3ac--- /dev/null+++ b/opensora/acceleration/shardformer/policy/t5_encoder.py@@ -0,0 +1,81 @@+from colossalai.shardformer.modeling.jit import get_jit_fused_dropout_add_func+from colossalai.shardformer.modeling.t5 import (+ get_jit_fused_T5_layer_ff_forward,+ get_T5_layer_self_attention_forward,+)+from colossalai.shardformer.policies.base_policy import Policy, SubModuleReplacementDescription++class T5EncoderPolicy(Policy):++ def config_sanity_check(self):+ assert not self.shard_config.enable_tensor_parallelism+ assert not self.shard_config.enable_flash_attention++ def preprocess(self):+ return self.model++ def module_policy(self):+ from transformers.models.t5.modeling_t5 import (+ T5LayerFF,+ T5LayerSelfAttention,+ T5Stack,+ )++ policy = {}++ # check whether apex is installed+ try:+ from apex.normalization import FusedRMSNorm+ from opensora.acceleration.shardformer.modeling.t5 import T5LayerNorm++ # recover hf from fused rms norm to T5 norm which is faster+ self.append_or_create_submodule_replacement(+ description=SubModuleReplacementDescription(+ suffix="layer_norm",+ target_module=T5LayerNorm,+ ),+ policy=policy,+ target_key=T5LayerFF,+ )+ self.append_or_create_submodule_replacement(+ description=SubModuleReplacementDescription(+ suffix="layer_norm",+ target_module=T5LayerNorm+ ),+ policy=policy,+ target_key=T5LayerSelfAttention,+ )+ self.append_or_create_submodule_replacement(+ description=SubModuleReplacementDescription(+ suffix="final_layer_norm",+ target_module=T5LayerNorm+ ),+ policy=policy,+ target_key=T5Stack,+ )+ except (ImportError, ModuleNotFoundError):+ pass++ # use jit operator+ if self.shard_config.enable_jit_fused:+ self.append_or_create_method_replacement(+ description={+ "forward": get_jit_fused_T5_layer_ff_forward(),+ "dropout_add": get_jit_fused_dropout_add_func(),+ },+ policy=policy,+ target_key=T5LayerFF,+ )+ self.append_or_create_method_replacement(+ description={+ "forward": get_T5_layer_self_attention_forward(),+ "dropout_add": get_jit_fused_dropout_add_func(),+ },+ policy=policy,+ target_key=T5LayerSelfAttention,+ )++ return policy++ def postprocess(self):+ return self.modeldiff --git a/opensora/datasets/__init__.py b/opensora/datasets/__init__.pynew file mode 100644index 0000000..c9b3395--- /dev/null+++ b/opensora/datasets/__init__.py@@ -0,0 +1,2 @@+from .datasets import DatasetFromCSV, get_transforms_image, get_transforms_video+from .utils import prepare_dataloader, save_samplediff --git a/opensora/datasets/datasets.py b/opensora/datasets/datasets.pynew file mode 100644index 0000000..c709685--- /dev/null+++ b/opensora/datasets/datasets.py@@ -0,0 +1,114 @@+import csv+import os++import numpy as np+import torch+import torchvision+import torchvision.transforms as transforms+from torchvision.datasets.folder import IMG_EXTENSIONS, pil_loader++from . import video_transforms+from .utils import center_crop_arr+++def get_transforms_video(resolution=256):+ transform_video = transforms.Compose(+ [+ video_transforms.ToTensorVideo(), # TCHW+ video_transforms.RandomHorizontalFlipVideo(),+ video_transforms.UCFCenterCropVideo(resolution),+ transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True),+ ]+ )+ return transform_video+++def get_transforms_image(image_size=256):+ transform = transforms.Compose(+ [+ transforms.Lambda(lambda pil_image: center_crop_arr(pil_image, image_size)),+ transforms.RandomHorizontalFlip(),+ transforms.ToTensor(),+ transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True),+ ]+ )+ return transform+++class DatasetFromCSV(torch.utils.data.Dataset):+ """load video according to the csv file.++ Args:+ target_video_len (int): the number of video frames will be load.+ align_transform (callable): Align different videos in a specified size.+ temporal_sample (callable): Sample the target length of a video.+ """++ def __init__(+ self,+ csv_path,+ num_frames=16,+ frame_interval=1,+ transform=None,+ root=None,+ ):+ self.csv_path = csv_path+ with open(csv_path, "r") as f:+ reader = csv.reader(f)+ self.samples = list(reader)++ ext = self.samples[0][0].split(".")[-1]+ if ext.lower() in ["mp4", "avi", "mov", "mkv"]:+ self.is_video = True+ else:+ assert f".{ext.lower()}" in IMG_EXTENSIONS+ self.is_video = False++ self.transform = transform++ self.num_frames = num_frames+ self.frame_interval = frame_interval+ self.temporal_sample = video_transforms.TemporalRandomCrop(num_frames * frame_interval)+ self.root = root++ def getitem(self, index):+ sample = self.samples[index]+ path = sample[0]+ if self.root:+ path = os.path.join(self.root, path)+ text = sample[1]++ if self.is_video:+ vframes, aframes, info = torchvision.io.read_video(filename=path, pts_unit="sec", output_format="TCHW")+ total_frames = len(vframes)++ # Sampling video frames+ start_frame_ind, end_frame_ind = self.temporal_sample(total_frames)+ assert (+ end_frame_ind - start_frame_ind >= self.num_frames+ ), f"{path} with index {index} has not enough frames."+ frame_indice = np.linspace(start_frame_ind, end_frame_ind - 1, self.num_frames, dtype=int)++ video = vframes[frame_indice]+ video = self.transform(video) # T C H W+ else:+ image = pil_loader(path)+ image = self.transform(image)+ video = image.unsqueeze(0).repeat(self.num_frames, 1, 1, 1)++ # TCHW -> CTHW+ video = video.permute(1, 0, 2, 3)++ return {"video": video, "text": text}++ def __getitem__(self, index):+ for _ in range(10):+ try:+ return self.getitem(index)+ except Exception as e:+ print(e)+ index = np.random.randint(len(self))+ raise RuntimeError("Too many bad data.")++ def __len__(self):+ return len(self.samples)diff --git a/opensora/datasets/utils.py b/opensora/datasets/utils.pynew file mode 100644index 0000000..00f8fe3--- /dev/null+++ b/opensora/datasets/utils.py@@ -0,0 +1,134 @@+import random+from typing import Iterator, Optional++import numpy as np+import torch+from PIL import Image+from torch.distributed import ProcessGroup+from torch.distributed.distributed_c10d import _get_default_group+from torch.utils.data import DataLoader, Dataset+from torch.utils.data.distributed import DistributedSampler+from torchvision.io import write_video+from torchvision.utils import save_image+++def save_sample(x, fps=8, save_path=None, normalize=True, value_range=(-1, 1)):+ """+ Args:+ x (Tensor): shape [C, T, H, W]+ """+ assert x.ndim == 4++ if x.shape[1] == 1: # T = 1: save as image+ save_path += ".png"+ x = x.squeeze(1)+ save_image([x], save_path, normalize=normalize, value_range=value_range)+ else:+ save_path += ".mp4"+ if normalize:+ low, high = value_range+ x.clamp_(min=low, max=high)+ x.sub_(low).div_(max(high - low, 1e-5))++ x = x.mul(255).add_(0.5).clamp_(0, 255).permute(1, 2, 3, 0).to("cpu", torch.uint8)+ write_video(save_path, x, fps=fps, video_codec="h264")+ print(f"Saved to {save_path}")+++class StatefulDistributedSampler(DistributedSampler):+ def __init__(+ self,+ dataset: Dataset,+ num_replicas: Optional[int] = None,+ rank: Optional[int] = None,+ shuffle: bool = True,+ seed: int = 0,+ drop_last: bool = False,+ ) -> None:+ super().__init__(dataset, num_replicas, rank, shuffle, seed, drop_last)+ self.start_index: int = 0++ def __iter__(self) -> Iterator:+ iterator = super().__iter__()+ indices = list(iterator)+ indices = indices[self.start_index :]+ return iter(indices)++ def __len__(self) -> int:+ return self.num_samples - self.start_index++ def set_start_index(self, start_index: int) -> None:+ self.start_index = start_index+++def prepare_dataloader(+ dataset,+ batch_size,+ shuffle=False,+ seed=1024,+ drop_last=False,+ pin_memory=False,+ num_workers=0,+ process_group: Optional[ProcessGroup] = None,+ **kwargs,+):+ r"""+ Prepare a dataloader for distributed training. The dataloader will be wrapped by+ `torch.utils.data.DataLoader` and `StatefulDistributedSampler`.+++ Args:+ dataset (`torch.utils.data.Dataset`): The dataset to be loaded.+ shuffle (bool, optional): Whether to shuffle the dataset. Defaults to False.+ seed (int, optional): Random worker seed for sampling, defaults to 1024.+ add_sampler: Whether to add ``DistributedDataParallelSampler`` to the dataset. Defaults to True.+ drop_last (bool, optional): Set to True to drop the last incomplete batch, if the dataset size+ is not divisible by the batch size. If False and the size of dataset is not divisible by+ the batch size, then the last batch will be smaller, defaults to False.+ pin_memory (bool, optional): Whether to pin memory address in CPU memory. Defaults to False.+ num_workers (int, optional): Number of worker threads for this dataloader. Defaults to 0.+ kwargs (dict): optional parameters for ``torch.utils.data.DataLoader``, more details could be found in+ `DataLoader <https://pytorch.org/docs/stable/_modules/torch/utils/data/dataloader.html#DataLoader>`_.++ Returns:+ :class:`torch.utils.data.DataLoader`: A DataLoader used for training or testing.+ """+ _kwargs = kwargs.copy()+ process_group = process_group or _get_default_group()+ sampler = StatefulDistributedSampler(+ dataset, num_replicas=process_group.size(), rank=process_group.rank(), shuffle=shuffle+ )++ # Deterministic dataloader+ def seed_worker(worker_id):+ worker_seed = seed+ np.random.seed(worker_seed)+ torch.manual_seed(worker_seed)+ random.seed(worker_seed)++ return DataLoader(+ dataset,+ batch_size=batch_size,+ sampler=sampler,+ worker_init_fn=seed_worker,+ drop_last=drop_last,+ pin_memory=pin_memory,+ num_workers=num_workers,+ **_kwargs,+ )++def center_crop_arr(pil_image, image_size):+ """+ Center cropping implementation from ADM.+ https://github.com/openai/guided-diffusion/blob/8fb3ad9197f16bbc40620447b2742e13458d2831/guided_diffusion/image_datasets.py#L126+ """+ while min(*pil_image.size) >= 2 * image_size:+ pil_image = pil_image.resize(tuple(x // 2 for x in pil_image.size), resample=Image.BOX)++ scale = image_size / min(*pil_image.size)+ pil_image = pil_image.resize(tuple(round(x * scale) for x in pil_image.size), resample=Image.BICUBIC)++ arr = np.array(pil_image)+ crop_y = (arr.shape[0] - image_size) // 2+ crop_x = (arr.shape[1] - image_size) // 2+ return Image.fromarray(arr[crop_y : crop_y + image_size, crop_x : crop_x + image_size])diff --git a/opensora/datasets/video_transforms.py b/opensora/datasets/video_transforms.pynew file mode 100644index 0000000..21ee925--- /dev/null+++ b/opensora/datasets/video_transforms.py@@ -0,0 +1,486 @@+# Modified from Latte++import numbers+import random++import torch+++def _is_tensor_video_clip(clip):+ if not torch.is_tensor(clip):+ raise TypeError("clip should be Tensor. Got %s" % type(clip))++ if not clip.ndimension() == 4:+ raise ValueError("clip should be 4D. Got %dD" % clip.dim())++ return True+++def center_crop_arr(pil_image, image_size):+ """+ Center cropping implementation from ADM.+ https://github.com/openai/guided-diffusion/blob/8fb3ad9197f16bbc40620447b2742e13458d2831/guided_diffusion/image_datasets.py#L126+ """+ while min(*pil_image.size) >= 2 * image_size:+ pil_image = pil_image.resize(tuple(x // 2 for x in pil_image.size), resample=Image.BOX)++ scale = image_size / min(*pil_image.size)+ pil_image = pil_image.resize(tuple(round(x * scale) for x in pil_image.size), resample=Image.BICUBIC)++ arr = np.array(pil_image)+ crop_y = (arr.shape[0] - image_size) // 2+ crop_x = (arr.shape[1] - image_size) // 2+ return Image.fromarray(arr[crop_y : crop_y + image_size, crop_x : crop_x + image_size])+++def crop(clip, i, j, h, w):+ """+ Args:+ clip (torch.tensor): Video clip to be cropped. Size is (T, C, H, W)+ """+ if len(clip.size()) != 4:+ raise ValueError("clip should be a 4D tensor")+ return clip[..., i : i + h, j : j + w]+++def resize(clip, target_size, interpolation_mode):+ if len(target_size) != 2:+ raise ValueError(f"target size should be tuple (height, width), instead got {target_size}")+ return torch.nn.functional.interpolate(clip, size=target_size, mode=interpolation_mode, align_corners=False)+++def resize_scale(clip, target_size, interpolation_mode):+ if len(target_size) != 2:+ raise ValueError(f"target size should be tuple (height, width), instead got {target_size}")+ H, W = clip.size(-2), clip.size(-1)+ scale_ = target_size[0] / min(H, W)+ return torch.nn.functional.interpolate(clip, scale_factor=scale_, mode=interpolation_mode, align_corners=False)+++def resized_crop(clip, i, j, h, w, size, interpolation_mode="bilinear"):+ """+ Do spatial cropping and resizing to the video clip+ Args:+ clip (torch.tensor): Video clip to be cropped. Size is (T, C, H, W)+ i (int): i in (i,j) i.e coordinates of the upper left corner.+ j (int): j in (i,j) i.e coordinates of the upper left corner.+ h (int): Height of the cropped region.+ w (int): Width of the cropped region.+ size (tuple(int, int)): height and width of resized clip+ Returns:+ clip (torch.tensor): Resized and cropped clip. Size is (T, C, H, W)+ """+ if not _is_tensor_video_clip(clip):+ raise ValueError("clip should be a 4D torch.tensor")+ clip = crop(clip, i, j, h, w)+ clip = resize(clip, size, interpolation_mode)+ return clip+++def center_crop(clip, crop_size):+ if not _is_tensor_video_clip(clip):+ raise ValueError("clip should be a 4D torch.tensor")+ h, w = clip.size(-2), clip.size(-1)+ th, tw = crop_size+ if h < th or w < tw:+ raise ValueError("height and width must be no smaller than crop_size")++ i = int(round((h - th) / 2.0))+ j = int(round((w - tw) / 2.0))+ return crop(clip, i, j, th, tw)+++def center_crop_using_short_edge(clip):+ if not _is_tensor_video_clip(clip):+ raise ValueError("clip should be a 4D torch.tensor")+ h, w = clip.size(-2), clip.size(-1)+ if h < w:+ th, tw = h, h+ i = 0+ j = int(round((w - tw) / 2.0))+ else:+ th, tw = w, w+ i = int(round((h - th) / 2.0))+ j = 0+ return crop(clip, i, j, th, tw)+++def random_shift_crop(clip):+ """+ Slide along the long edge, with the short edge as crop size+ """+ if not _is_tensor_video_clip(clip):+ raise ValueError("clip should be a 4D torch.tensor")+ h, w = clip.size(-2), clip.size(-1)++ if h <= w:+ short_edge = h+ else:+ short_edge = w++ th, tw = short_edge, short_edge++ i = torch.randint(0, h - th + 1, size=(1,)).item()+ j = torch.randint(0, w - tw + 1, size=(1,)).item()+ return crop(clip, i, j, th, tw)+++def to_tensor(clip):+ """+ Convert tensor data type from uint8 to float, divide value by 255.0 and+ permute the dimensions of clip tensor+ Args:+ clip (torch.tensor, dtype=torch.uint8): Size is (T, C, H, W)+ Return:+ clip (torch.tensor, dtype=torch.float): Size is (T, C, H, W)+ """+ _is_tensor_video_clip(clip)+ if not clip.dtype == torch.uint8:+ raise TypeError("clip tensor should have data type uint8. Got %s" % str(clip.dtype))+ # return clip.float().permute(3, 0, 1, 2) / 255.0+ return clip.float() / 255.0+++def normalize(clip, mean, std, inplace=False):+ """+ Args:+ clip (torch.tensor): Video clip to be normalized. Size is (T, C, H, W)+ mean (tuple): pixel RGB mean. Size is (3)+ std (tuple): pixel standard deviation. Size is (3)+ Returns:+ normalized clip (torch.tensor): Size is (T, C, H, W)+ """+ if not _is_tensor_video_clip(clip):+ raise ValueError("clip should be a 4D torch.tensor")+ if not inplace:+ clip = clip.clone()+ mean = torch.as_tensor(mean, dtype=clip.dtype, device=clip.device)+ # print(mean)+ std = torch.as_tensor(std, dtype=clip.dtype, device=clip.device)+ clip.sub_(mean[:, None, None, None]).div_(std[:, None, None, None])+ return clip+++def hflip(clip):+ """+ Args:+ clip (torch.tensor): Video clip to be normalized. Size is (T, C, H, W)+ Returns:+ flipped clip (torch.tensor): Size is (T, C, H, W)+ """+ if not _is_tensor_video_clip(clip):+ raise ValueError("clip should be a 4D torch.tensor")+ return clip.flip(-1)+++class RandomCropVideo:+ def __init__(self, size):+ if isinstance(size, numbers.Number):+ self.size = (int(size), int(size))+ else:+ self.size = size++ def __call__(self, clip):+ """+ Args:+ clip (torch.tensor): Video clip to be cropped. Size is (T, C, H, W)+ Returns:+ torch.tensor: randomly cropped video clip.+ size is (T, C, OH, OW)+ """+ i, j, h, w = self.get_params(clip)+ return crop(clip, i, j, h, w)++ def get_params(self, clip):+ h, w = clip.shape[-2:]+ th, tw = self.size++ if h < th or w < tw:+ raise ValueError(f"Required crop size {(th, tw)} is larger than input image size {(h, w)}")++ if w == tw and h == th:+ return 0, 0, h, w++ i = torch.randint(0, h - th + 1, size=(1,)).item()+ j = torch.randint(0, w - tw + 1, size=(1,)).item()++ return i, j, th, tw++ def __repr__(self) -> str:+ return f"{self.__class__.__name__}(size={self.size})"+++class CenterCropResizeVideo:+ """+ First use the short side for cropping length,+ center crop video, then resize to the specified size+ """++ def __init__(+ self,+ size,+ interpolation_mode="bilinear",+ ):+ if isinstance(size, tuple):+ if len(size) != 2:+ raise ValueError(f"size should be tuple (height, width), instead got {size}")+ self.size = size+ else:+ self.size = (size, size)++ self.interpolation_mode = interpolation_mode++ def __call__(self, clip):+ """+ Args:+ clip (torch.tensor): Video clip to be cropped. Size is (T, C, H, W)+ Returns:+ torch.tensor: scale resized / center cropped video clip.+ size is (T, C, crop_size, crop_size)+ """+ clip_center_crop = center_crop_using_short_edge(clip)+ clip_center_crop_resize = resize(+ clip_center_crop, target_size=self.size, interpolation_mode=self.interpolation_mode+ )+ return clip_center_crop_resize++ def __repr__(self) -> str:+ return f"{self.__class__.__name__}(size={self.size}, interpolation_mode={self.interpolation_mode}"+++class UCFCenterCropVideo:+ """+ First scale to the specified size in equal proportion to the short edge,+ then center cropping+ """++ def __init__(+ self,+ size,+ interpolation_mode="bilinear",+ ):+ if isinstance(size, tuple):+ if len(size) != 2:+ raise ValueError(f"size should be tuple (height, width), instead got {size}")+ self.size = size+ else:+ self.size = (size, size)++ self.interpolation_mode = interpolation_mode++ def __call__(self, clip):+ """+ Args:+ clip (torch.tensor): Video clip to be cropped. Size is (T, C, H, W)+ Returns:+ torch.tensor: scale resized / center cropped video clip.+ size is (T, C, crop_size, crop_size)+ """+ clip_resize = resize_scale(clip=clip, target_size=self.size, interpolation_mode=self.interpolation_mode)+ clip_center_crop = center_crop(clip_resize, self.size)+ return clip_center_crop++ def __repr__(self) -> str:+ return f"{self.__class__.__name__}(size={self.size}, interpolation_mode={self.interpolation_mode}"+++class KineticsRandomCropResizeVideo:+ """+ Slide along the long edge, with the short edge as crop size. And resie to the desired size.+ """++ def __init__(+ self,+ size,+ interpolation_mode="bilinear",+ ):+ if isinstance(size, tuple):+ if len(size) != 2:+ raise ValueError(f"size should be tuple (height, width), instead got {size}")+ self.size = size+ else:+ self.size = (size, size)++ self.interpolation_mode = interpolation_mode++ def __call__(self, clip):+ clip_random_crop = random_shift_crop(clip)+ clip_resize = resize(clip_random_crop, self.size, self.interpolation_mode)+ return clip_resize+++class CenterCropVideo:+ def __init__(+ self,+ size,+ interpolation_mode="bilinear",+ ):+ if isinstance(size, tuple):+ if len(size) != 2:+ raise ValueError(f"size should be tuple (height, width), instead got {size}")+ self.size = size+ else:+ self.size = (size, size)++ self.interpolation_mode = interpolation_mode++ def __call__(self, clip):+ """+ Args:+ clip (torch.tensor): Video clip to be cropped. Size is (T, C, H, W)+ Returns:+ torch.tensor: center cropped video clip.+ size is (T, C, crop_size, crop_size)+ """+ clip_center_crop = center_crop(clip, self.size)+ return clip_center_crop++ def __repr__(self) -> str:+ return f"{self.__class__.__name__}(size={self.size}, interpolation_mode={self.interpolation_mode}"+++class NormalizeVideo:+ """+ Normalize the video clip by mean subtraction and division by standard deviation+ Args:+ mean (3-tuple): pixel RGB mean+ std (3-tuple): pixel RGB standard deviation+ inplace (boolean): whether do in-place normalization+ """++ def __init__(self, mean, std, inplace=False):+ self.mean = mean+ self.std = std+ self.inplace = inplace++ def __call__(self, clip):+ """+ Args:+ clip (torch.tensor): video clip must be normalized. Size is (C, T, H, W)+ """+ return normalize(clip, self.mean, self.std, self.inplace)++ def __repr__(self) -> str:+ return f"{self.__class__.__name__}(mean={self.mean}, std={self.std}, inplace={self.inplace})"+++class ToTensorVideo:+ """+ Convert tensor data type from uint8 to float, divide value by 255.0 and+ permute the dimensions of clip tensor+ """++ def __init__(self):+ pass++ def __call__(self, clip):+ """+ Args:+ clip (torch.tensor, dtype=torch.uint8): Size is (T, C, H, W)+ Return:+ clip (torch.tensor, dtype=torch.float): Size is (T, C, H, W)+ """+ return to_tensor(clip)++ def __repr__(self) -> str:+ return self.__class__.__name__+++class RandomHorizontalFlipVideo:+ """+ Flip the video clip along the horizontal direction with a given probability+ Args:+ p (float): probability of the clip being flipped. Default value is 0.5+ """++ def __init__(self, p=0.5):+ self.p = p++ def __call__(self, clip):+ """+ Args:+ clip (torch.tensor): Size is (T, C, H, W)+ Return:+ clip (torch.tensor): Size is (T, C, H, W)+ """+ if random.random() < self.p:+ clip = hflip(clip)+ return clip++ def __repr__(self) -> str:+ return f"{self.__class__.__name__}(p={self.p})"+++# ------------------------------------------------------------+# --------------------- Sampling ---------------------------+# ------------------------------------------------------------+class TemporalRandomCrop(object):+ """Temporally crop the given frame indices at a random location.++ Args:+ size (int): Desired length of frames will be seen in the model.+ """++ def __init__(self, size):+ self.size = size++ def __call__(self, total_frames):+ rand_end = max(0, total_frames - self.size - 1)+ begin_index = random.randint(0, rand_end)+ end_index = min(begin_index + self.size, total_frames)+ return begin_index, end_index+++if __name__ == "__main__":+ import os++ import numpy as np+ import torchvision.io as io+ from torchvision import transforms+ from torchvision.utils import save_image++ vframes, aframes, info = io.read_video(filename="./v_Archery_g01_c03.avi", pts_unit="sec", output_format="TCHW")++ trans = transforms.Compose(+ [+ ToTensorVideo(),+ RandomHorizontalFlipVideo(),+ UCFCenterCropVideo(512),+ # NormalizeVideo(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True),+ transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True),+ ]+ )++ target_video_len = 32+ frame_interval = 1+ total_frames = len(vframes)+ print(total_frames)++ temporal_sample = TemporalRandomCrop(target_video_len * frame_interval)++ # Sampling video frames+ start_frame_ind, end_frame_ind = temporal_sample(total_frames)+ # print(start_frame_ind)+ # print(end_frame_ind)+ assert end_frame_ind - start_frame_ind >= target_video_len+ frame_indice = np.linspace(start_frame_ind, end_frame_ind - 1, target_video_len, dtype=int)+ print(frame_indice)++ select_vframes = vframes[frame_indice]+ print(select_vframes.shape)+ print(select_vframes.dtype)++ select_vframes_trans = trans(select_vframes)+ print(select_vframes_trans.shape)+ print(select_vframes_trans.dtype)++ select_vframes_trans_int = ((select_vframes_trans * 0.5 + 0.5) * 255).to(dtype=torch.uint8)+ print(select_vframes_trans_int.dtype)+ print(select_vframes_trans_int.permute(0, 2, 3, 1).shape)++ io.write_video("./test.avi", select_vframes_trans_int.permute(0, 2, 3, 1), fps=8)++ for i in range(target_video_len):+ save_image(+ select_vframes_trans[i], os.path.join("./test000", "%04d.png" % i), normalize=True, value_range=(-1, 1)+ )diff --git a/opensora/models/__init__.py b/opensora/models/__init__.pynew file mode 100644index 0000000..2aa793a--- /dev/null+++ b/opensora/models/__init__.py@@ -0,0 +1,6 @@+from .dit import *+from .latte import *+from .stdit import *+from .pixart import *+from .text_encoder import *+from .vae import *diff --git a/opensora/models/dit/__init__.py b/opensora/models/dit/__init__.pynew file mode 100644index 0000000..2019831--- /dev/null+++ b/opensora/models/dit/__init__.py@@ -0,0 +1 @@+from .dit import DiT, DiT_XL_2, DiT_XL_2x2diff --git a/opensora/models/dit/dit.py b/opensora/models/dit/dit.pynew file mode 100644index 0000000..106f660--- /dev/null+++ b/opensora/models/dit/dit.py@@ -0,0 +1,284 @@+# Modified from Meta DiT++# This source code is licensed under the license found in the+# LICENSE file in the root directory of this source tree.+# --------------------------------------------------------+# References:+# DiT: https://github.com/facebookresearch/DiT/tree/main+# GLIDE: https://github.com/openai/glide-text2im+# MAE: https://github.com/facebookresearch/mae/blob/main/models_mae.py+# --------------------------------------------------------++import numpy as np+import torch+import torch.nn as nn+import torch.utils.checkpoint+from einops import rearrange+from timm.models.vision_transformer import Mlp++from opensora.models.layers.blocks import (+ Attention,+ CaptionEmbedder,+ FinalLayer,+ LabelEmbedder,+ PatchEmbed3D,+ TimestepEmbedder,+ approx_gelu,+ get_1d_sincos_pos_embed,+ get_2d_sincos_pos_embed,+ get_layernorm,+ modulate,+)+from opensora.registry import MODELS+from opensora.utils.ckpt_utils import load_checkpoint+from opensora.acceleration.checkpoint import auto_grad_checkpoint+++class DiTBlock(nn.Module):+ """+ A DiT block with adaptive layer norm zero (adaLN-Zero) conditioning.+ """++ def __init__(+ self,+ hidden_size,+ num_heads,+ mlp_ratio=4.0,+ enable_flashattn=False,+ enable_layernorm_kernel=False,+ ):+ super().__init__()+ self.hidden_size = hidden_size+ self.num_heads = num_heads+ self.enable_flashattn = enable_flashattn+ mlp_hidden_dim = int(hidden_size * mlp_ratio)++ self.norm1 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)+ self.attn = Attention(+ hidden_size,+ num_heads=num_heads,+ qkv_bias=True,+ enable_flashattn=enable_flashattn,+ )+ self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)+ self.mlp = Mlp(in_features=hidden_size, hidden_features=mlp_hidden_dim, act_layer=approx_gelu, drop=0)+ self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True))++ def forward(self, x, c):+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN_modulation(c).chunk(6, dim=1)+ x = x + gate_msa.unsqueeze(1) * self.attn(modulate(self.norm1, x, shift_msa, scale_msa))+ x = x + gate_mlp.unsqueeze(1) * self.mlp(modulate(self.norm2, x, shift_mlp, scale_mlp))+ return x++[email protected]_module()+class DiT(nn.Module):+ """+ Diffusion model with a Transformer backbone.+ """++ def __init__(+ self,+ input_size=(16, 32, 32),+ in_channels=4,+ patch_size=(1, 2, 2),+ hidden_size=1152,+ depth=28,+ num_heads=16,+ mlp_ratio=4.0,+ class_dropout_prob=0.1,+ learn_sigma=True,+ condition="text",+ no_temporal_pos_emb=False,+ caption_channels=512,+ model_max_length=77,+ dtype=torch.float32,+ enable_flashattn=False,+ enable_layernorm_kernel=False,+ ):+ super().__init__()+ self.learn_sigma = learn_sigma+ self.in_channels = in_channels+ self.out_channels = in_channels * 2 if learn_sigma else in_channels+ self.hidden_size = hidden_size+ self.patch_size = patch_size+ self.input_size = input_size+ num_patches = np.prod([input_size[i] // patch_size[i] for i in range(3)])+ self.num_patches = num_patches+ self.num_temporal = input_size[0] // patch_size[0]+ self.num_spatial = num_patches // self.num_temporal+ self.num_heads = num_heads+ self.dtype = dtype+ self.use_text_encoder = not condition.startswith("label")+ if enable_flashattn:+ assert dtype in [+ torch.float16,+ torch.bfloat16,+ ], f"Flash attention only supports float16 and bfloat16, but got {self.dtype}"+ self.no_temporal_pos_emb = no_temporal_pos_emb+ self.mlp_ratio = mlp_ratio+ self.depth = depth++ self.register_buffer("pos_embed_spatial", self.get_spatial_pos_embed())+ self.register_buffer("pos_embed_temporal", self.get_temporal_pos_embed())++ self.x_embedder = PatchEmbed3D(patch_size, in_channels, embed_dim=hidden_size)+ if not self.use_text_encoder:+ num_classes = int(condition.split("_")[-1])+ self.y_embedder = LabelEmbedder(num_classes, hidden_size, class_dropout_prob)+ else:+ self.y_embedder = CaptionEmbedder(+ in_channels=caption_channels,+ hidden_size=hidden_size,+ uncond_prob=class_dropout_prob,+ act_layer=approx_gelu,+ token_num=1, # pooled token+ )+ self.t_embedder = TimestepEmbedder(hidden_size)+ self.blocks = nn.ModuleList(+ [+ DiTBlock(+ hidden_size,+ num_heads,+ mlp_ratio=mlp_ratio,+ enable_flashattn=enable_flashattn,+ enable_layernorm_kernel=enable_layernorm_kernel,+ )+ for _ in range(depth)+ ]+ )+ self.final_layer = FinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)++ self.initialize_weights()+ self.enable_flashattn = enable_flashattn+ self.enable_layernorm_kernel = enable_layernorm_kernel++ def get_spatial_pos_embed(self):+ pos_embed = get_2d_sincos_pos_embed(+ self.hidden_size,+ self.input_size[1] // self.patch_size[1],+ )+ pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)+ return pos_embed++ def get_temporal_pos_embed(self):+ pos_embed = get_1d_sincos_pos_embed(+ self.hidden_size,+ self.input_size[0] // self.patch_size[0],+ )+ pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)+ return pos_embed++ def unpatchify(self, x):+ c = self.out_channels+ t, h, w = [self.input_size[i] // self.patch_size[i] for i in range(3)]+ pt, ph, pw = self.patch_size++ x = x.reshape(shape=(x.shape[0], t, h, w, pt, ph, pw, c))+ x = rearrange(x, "n t h w r p q c -> n c t r h p w q")+ imgs = x.reshape(shape=(x.shape[0], c, t * pt, h * ph, w * pw))+ return imgs++ def forward(self, x, t, y):+ """+ Forward pass of DiT.+ x: (B, C, T, H, W) tensor of inputs+ t: (B,) tensor of diffusion timesteps+ y: list of text+ """+ # origin inputs should be float32, cast to specified dtype+ x = x.to(self.dtype)++ # embedding+ x = self.x_embedder(x) # (B, N, D)+ x = rearrange(x, "b (t s) d -> b t s d", t=self.num_temporal, s=self.num_spatial)+ x = x + self.pos_embed_spatial+ if not self.no_temporal_pos_emb:+ x = rearrange(x, "b t s d -> b s t d")+ x = x + self.pos_embed_temporal+ x = rearrange(x, "b s t d -> b (t s) d")+ else:+ x = rearrange(x, "b t s d -> b (t s) d")++ t = self.t_embedder(t, dtype=x.dtype) # (N, D)+ y = self.y_embedder(y, self.training) # (N, D)+ if self.use_text_encoder:+ y = y.squeeze(1).squeeze(1)+ condition = t + y++ # blocks+ for _, block in enumerate(self.blocks):+ c = condition+ x = auto_grad_checkpoint(block, x, c) # (B, N, D)++ # final process+ x = self.final_layer(x, condition) # (B, N, num_patches * out_channels)+ x = self.unpatchify(x) # (B, out_channels, T, H, W)++ # cast to float32 for better accuracy+ x = x.to(torch.float32)+ return x++ def initialize_weights(self):+ # Initialize transformer layers:+ def _basic_init(module):+ if isinstance(module, nn.Linear):+ if module.weight.requires_grad_:+ torch.nn.init.xavier_uniform_(module.weight)+ if module.bias is not None:+ nn.init.constant_(module.bias, 0)++ self.apply(_basic_init)++ # Initialize patch_embed like nn.Linear (instead of nn.Conv2d):+ w = self.x_embedder.proj.weight.data+ nn.init.xavier_uniform_(w.view([w.shape[0], -1]))+ nn.init.constant_(self.x_embedder.proj.bias, 0)++ # Initialize timestep embedding MLP:+ nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02)+ nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02)++ # Zero-out adaLN modulation layers in DiT blocks:+ for block in self.blocks:+ nn.init.constant_(block.adaLN_modulation[-1].weight, 0)+ nn.init.constant_(block.adaLN_modulation[-1].bias, 0)++ # Zero-out output layers:+ nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0)+ nn.init.constant_(self.final_layer.adaLN_modulation[-1].bias, 0)+ nn.init.constant_(self.final_layer.linear.weight, 0)+ nn.init.constant_(self.final_layer.linear.bias, 0)++ # Zero-out text embedding layers:+ if self.use_text_encoder:+ nn.init.normal_(self.y_embedder.y_proj.fc1.weight, std=0.02)+ nn.init.normal_(self.y_embedder.y_proj.fc2.weight, std=0.02)++[email protected]_module("DiT-XL/2")+def DiT_XL_2(from_pretrained=None, **kwargs):+ model = DiT(+ depth=28,+ hidden_size=1152,+ patch_size=(1, 2, 2),+ num_heads=16,+ **kwargs,+ )+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)+ return model++[email protected]_module("DiT-XL/2x2")+def DiT_XL_2x2(from_pretrained=None, **kwargs):+ model = DiT(+ depth=28,+ hidden_size=1152,+ patch_size=(2, 2, 2),+ num_heads=16,+ **kwargs,+ )+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)+ return modeldiff --git a/opensora/models/latte/__init__.py b/opensora/models/latte/__init__.pynew file mode 100644index 0000000..3115f47--- /dev/null+++ b/opensora/models/latte/__init__.py@@ -0,0 +1 @@+from .latte import Latte, Latte_XL_2, Latte_XL_2x2diff --git a/opensora/models/latte/latte.py b/opensora/models/latte/latte.pynew file mode 100644index 0000000..11259de--- /dev/null+++ b/opensora/models/latte/latte.py@@ -0,0 +1,100 @@+# Adapted from Latte+#+# This source code is licensed under the license found in the+# LICENSE file in the root directory of this source tree.+# --------------------------------------------------------+# References:+# Latte: https://github.com/Vchitect/Latte+# DiT: https://github.com/facebookresearch/DiT/tree/main+# --------------------------------------------------------++import torch+from einops import rearrange, repeat++from opensora.models.dit import DiT+from opensora.registry import MODELS+from opensora.utils.ckpt_utils import load_checkpoint+from opensora.acceleration.checkpoint import auto_grad_checkpoint++[email protected]_module()+class Latte(DiT):+ def forward(self, x, t, y):+ """+ Forward pass of DiT.+ x: (B, C, T, H, W) tensor of inputs+ t: (B,) tensor of diffusion timesteps+ y: list of text+ """+ # origin inputs should be float32, cast to specified dtype+ x = x.to(self.dtype)++ # embedding+ x = self.x_embedder(x) # (B, N, D)+ x = rearrange(x, "b (t s) d -> b t s d", t=self.num_temporal, s=self.num_spatial)+ x = x + self.pos_embed_spatial+ x = rearrange(x, "b t s d -> b (t s) d")++ t = self.t_embedder(t, dtype=x.dtype) # (N, D)+ y = self.y_embedder(y, self.training) # (N, D)+ if self.use_text_encoder:+ y = y.squeeze(1).squeeze(1)+ condition = t + y+ condition_spatial = repeat(condition, "b d -> (b t) d", t=self.num_temporal)+ condition_temporal = repeat(condition, "b d -> (b s) d", s=self.num_spatial)++ # blocks+ for i, block in enumerate(self.blocks):+ if i % 2 == 0:+ # spatial+ x = rearrange(x, "b (t s) d -> (b t) s d", t=self.num_temporal, s=self.num_spatial)+ c = condition_spatial+ else:+ # temporal+ x = rearrange(x, "b (t s) d -> (b s) t d", t=self.num_temporal, s=self.num_spatial)+ c = condition_temporal+ if i == 1:+ x = x + self.pos_embed_temporal++ x = auto_grad_checkpoint(block, x, c) # (B, N, D)++ if i % 2 == 0:+ x = rearrange(x, "(b t) s d -> b (t s) d", t=self.num_temporal, s=self.num_spatial)+ else:+ x = rearrange(x, "(b s) t d -> b (t s) d", t=self.num_temporal, s=self.num_spatial)++ # final process+ x = self.final_layer(x, condition) # (B, N, num_patches * out_channels)+ x = self.unpatchify(x) # (B, out_channels, T, H, W)++ # cast to float32 for better accuracy+ x = x.to(torch.float32)+ return x++[email protected]_module("Latte-XL/2")+def Latte_XL_2(from_pretrained=None, **kwargs):+ model = Latte(+ depth=28,+ hidden_size=1152,+ patch_size=(1, 2, 2),+ num_heads=16,+ **kwargs,+ )+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)+ return model++[email protected]_module("Latte-XL/2x2")+def Latte_XL_2x2(from_pretrained=None, **kwargs):+ model = Latte(+ depth=28,+ hidden_size=1152,+ patch_size=(2, 2, 2),+ num_heads=16,+ **kwargs,+ )+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)+ return modeldiff --git a/opensora/models/layers/__init__.py b/opensora/models/layers/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/opensora/models/layers/blocks.py b/opensora/models/layers/blocks.pynew file mode 100644index 0000000..f06d696--- /dev/null+++ b/opensora/models/layers/blocks.py@@ -0,0 +1,591 @@+# This source code is licensed under the license found in the+# LICENSE file in the root directory of this source tree.+# --------------------------------------------------------+# References:+# PixArt: https://github.com/PixArt-alpha/PixArt-alpha+# Latte: https://github.com/Vchitect/Latte+# DiT: https://github.com/facebookresearch/DiT/tree/main+# GLIDE: https://github.com/openai/glide-text2im+# MAE: https://github.com/facebookresearch/mae/blob/main/models_mae.py+# --------------------------------------------------------++import math++import numpy as np+import torch+import torch.distributed as dist+import torch.nn as nn+import torch.nn.functional as F+import torch.utils.checkpoint+import xformers.ops+from einops import rearrange+from flash_attn import flash_attn_varlen_func+from timm.models.vision_transformer import Mlp++from opensora.acceleration.communications import all_to_all, split_forward_gather_backward+from opensora.acceleration.parallel_states import get_sequence_parallel_group++approx_gelu = lambda: nn.GELU(approximate="tanh")+++def get_layernorm(hidden_size: torch.Tensor, eps: float, affine: bool, use_kernel: bool):+ if use_kernel:+ try:+ from apex.normalization import FusedLayerNorm++ return FusedLayerNorm(hidden_size, elementwise_affine=affine, eps=eps)+ except ImportError:+ raise RuntimeError("FusedLayerNorm not available. Please install apex.")+ else:+ return nn.LayerNorm(hidden_size, eps, elementwise_affine=affine)+++def modulate(norm_func, x, shift, scale):+ # Suppose x is (B, N, D), shift is (B, D), scale is (B, D)+ dtype = x.dtype+ x = norm_func(x.to(torch.float32)).to(dtype)+ x = x * (scale.unsqueeze(1) + 1) + shift.unsqueeze(1)+ x = x.to(dtype)+ return x+++def t2i_modulate(x, shift, scale):+ return x * (1 + scale) + shift+++# ===============================================+# General-purpose Layers+# ===============================================+++class PatchEmbed3D(nn.Module):+ """Video to Patch Embedding.++ Args:+ patch_size (int): Patch token size. Default: (2,4,4).+ in_chans (int): Number of input video channels. Default: 3.+ embed_dim (int): Number of linear projection output channels. Default: 96.+ norm_layer (nn.Module, optional): Normalization layer. Default: None+ """++ def __init__(+ self,+ patch_size=(2, 4, 4),+ in_chans=3,+ embed_dim=96,+ norm_layer=None,+ flatten=True,+ ):+ super().__init__()+ self.patch_size = patch_size+ self.flatten = flatten++ self.in_chans = in_chans+ self.embed_dim = embed_dim++ self.proj = nn.Conv3d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)+ if norm_layer is not None:+ self.norm = norm_layer(embed_dim)+ else:+ self.norm = None++ def forward(self, x):+ """Forward function."""+ # padding+ _, _, D, H, W = x.size()+ if W % self.patch_size[2] != 0:+ x = F.pad(x, (0, self.patch_size[2] - W % self.patch_size[2]))+ if H % self.patch_size[1] != 0:+ x = F.pad(x, (0, 0, 0, self.patch_size[1] - H % self.patch_size[1]))+ if D % self.patch_size[0] != 0:+ x = F.pad(x, (0, 0, 0, 0, 0, self.patch_size[0] - D % self.patch_size[0]))++ x = self.proj(x) # (B C T H W)+ if self.norm is not None:+ D, Wh, Ww = x.size(2), x.size(3), x.size(4)+ x = x.flatten(2).transpose(1, 2)+ x = self.norm(x)+ x = x.transpose(1, 2).view(-1, self.embed_dim, D, Wh, Ww)+ if self.flatten:+ x = x.flatten(2).transpose(1, 2) # BCTHW -> BNC+ return x+++class Attention(nn.Module):+ def __init__(+ self,+ dim: int,+ num_heads: int = 8,+ qkv_bias: bool = False,+ qk_norm: bool = False,+ attn_drop: float = 0.0,+ proj_drop: float = 0.0,+ norm_layer: nn.Module = nn.LayerNorm,+ enable_flashattn: bool = False,+ ) -> None:+ super().__init__()+ assert dim % num_heads == 0, "dim should be divisible by num_heads"+ self.dim = dim+ self.num_heads = num_heads+ self.head_dim = dim // num_heads+ self.scale = self.head_dim**-0.5+ self.enable_flashattn = enable_flashattn++ self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)+ self.q_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()+ self.k_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()+ self.attn_drop = nn.Dropout(attn_drop)+ self.proj = nn.Linear(dim, dim)+ self.proj_drop = nn.Dropout(proj_drop)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ B, N, C = x.shape+ qkv = self.qkv(x)+ qkv_shape = (B, N, 3, self.num_heads, self.head_dim)+ if self.enable_flashattn:+ qkv_permute_shape = (2, 0, 1, 3, 4)+ else:+ qkv_permute_shape = (2, 0, 3, 1, 4)+ qkv = qkv.view(qkv_shape).permute(qkv_permute_shape)+ q, k, v = qkv.unbind(0)+ q, k = self.q_norm(q), self.k_norm(k)+ if self.enable_flashattn:+ from flash_attn import flash_attn_func++ x = flash_attn_func(+ q,+ k,+ v,+ dropout_p=self.attn_drop.p if self.training else 0.0,+ softmax_scale=self.scale,+ )+ else:+ dtype = q.dtype+ q = q * self.scale+ attn = q @ k.transpose(-2, -1) # translate attn to float32+ attn = attn.to(torch.float32)+ attn = attn.softmax(dim=-1)+ attn = attn.to(dtype) # cast back attn to original dtype+ attn = self.attn_drop(attn)+ x = attn @ v++ x_output_shape = (B, N, C)+ if not self.enable_flashattn:+ x = x.transpose(1, 2)+ x = x.reshape(x_output_shape)+ x = self.proj(x)+ x = self.proj_drop(x)+ return x+++class SeqParallelAttention(Attention):++ def __init__(+ self,+ dim: int,+ num_heads: int = 8,+ qkv_bias: bool = False,+ qk_norm: bool = False,+ attn_drop: float = 0.0,+ proj_drop: float = 0.0,+ norm_layer: nn.Module = nn.LayerNorm,+ enable_flashattn: bool = False,+ ) -> None:+ super().__init__(+ dim=dim,+ num_heads=num_heads,+ qkv_bias=qkv_bias,+ qk_norm=qk_norm,+ attn_drop=attn_drop,+ proj_drop=proj_drop,+ norm_layer=norm_layer,+ enable_flashattn=enable_flashattn,+ )++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ B, N, C = x.shape # for sequence parallel here, the N is a local sequence length+ qkv = self.qkv(x)+ qkv_shape = (B, N, 3, self.num_heads, self.head_dim)++ qkv = qkv.view(qkv_shape)++ sp_group = get_sequence_parallel_group()++ # apply all_to_all to gather sequence and split attention heads+ # [B, SUB_N, 3, NUM_HEAD, HEAD_DIM] -> [B, N, 3, NUM_HEAD_PER_DEVICE, HEAD_DIM]+ qkv = all_to_all(qkv, sp_group, scatter_dim=3, gather_dim=1)++ if self.enable_flashattn:+ qkv_permute_shape = (2, 0, 1, 3, 4) # [3, B, N, NUM_HEAD_PER_DEVICE, HEAD_DIM]+ else:+ qkv_permute_shape = (2, 0, 3, 1, 4) # [3, B, NUM_HEAD_PER_DEVICE, N, HEAD_DIM]+ qkv = qkv.permute(qkv_permute_shape)++ q, k, v = qkv.unbind(0)+ q, k = self.q_norm(q), self.k_norm(k)+ if self.enable_flashattn:+ from flash_attn import flash_attn_func++ x = flash_attn_func(+ q,+ k,+ v,+ dropout_p=self.attn_drop.p if self.training else 0.0,+ softmax_scale=self.scale,+ )+ else:+ dtype = q.dtype+ q = q * self.scale+ attn = q @ k.transpose(-2, -1) # translate attn to float32+ attn = attn.to(torch.float32)+ attn = attn.softmax(dim=-1)+ attn = attn.to(dtype) # cast back attn to original dtype+ attn = self.attn_drop(attn)+ x = attn @ v++ if not self.enable_flashattn:+ x = x.transpose(1, 2)++ # apply all to all to gather back attention heads and split sequence+ # [B, N, NUM_HEAD_PER_DEVICE, HEAD_DIM] -> [B, SUB_N, NUM_HEAD, HEAD_DIM]+ x = all_to_all(x, sp_group, scatter_dim=1, gather_dim=2)++ # reshape outputs back to [B, N, C]+ x_output_shape = (B, N, C)+ x = x.reshape(x_output_shape)+ x = self.proj(x)+ x = self.proj_drop(x)+ return x+++class MultiHeadCrossAttention(nn.Module):+ def __init__(self, d_model, num_heads, attn_drop=0.0, proj_drop=0.0):+ super(MultiHeadCrossAttention, self).__init__()+ assert d_model % num_heads == 0, "d_model must be divisible by num_heads"++ self.d_model = d_model+ self.num_heads = num_heads+ self.head_dim = d_model // num_heads++ self.q_linear = nn.Linear(d_model, d_model)+ self.kv_linear = nn.Linear(d_model, d_model * 2)+ self.attn_drop = nn.Dropout(attn_drop)+ self.proj = nn.Linear(d_model, d_model)+ self.proj_drop = nn.Dropout(proj_drop)++ def forward(self, x, cond, mask=None):+ # query/value: img tokens; key: condition; mask: if padding tokens+ B, N, C = x.shape++ q = self.q_linear(x).view(1, -1, self.num_heads, self.head_dim)+ kv = self.kv_linear(cond).view(1, -1, 2, self.num_heads, self.head_dim)+ k, v = kv.unbind(2)++ attn_bias = None+ if mask is not None:+ attn_bias = xformers.ops.fmha.BlockDiagonalMask.from_seqlens([N] * B, mask)+ x = xformers.ops.memory_efficient_attention(q, k, v, p=self.attn_drop.p, attn_bias=attn_bias)++ x = x.view(B, -1, C)+ x = self.proj(x)+ x = self.proj_drop(x)+ return x+++class SeqParallelMultiHeadCrossAttention(MultiHeadCrossAttention):++ def __init__(+ self,+ d_model,+ num_heads,+ attn_drop=0.0,+ proj_drop=0.0,+ ):+ super().__init__(d_model=d_model, num_heads=num_heads, attn_drop=attn_drop, proj_drop=proj_drop)++ def forward(self, x, cond, mask=None):+ # query/value: img tokens; key: condition; mask: if padding tokens+ sp_group = get_sequence_parallel_group()+ sp_size = dist.get_world_size(sp_group)+ B, SUB_N, C = x.shape+ N = SUB_N * sp_size++ # shape:+ # q, k, v: [B, SUB_N, NUM_HEADS, HEAD_DIM]+ q = self.q_linear(x).view(B, -1, self.num_heads, self.head_dim)+ kv = self.kv_linear(cond).view(B, -1, 2, self.num_heads, self.head_dim)+ k, v = kv.unbind(2)++ # apply all_to_all to gather sequence and split attention heads+ q = all_to_all(q, sp_group, scatter_dim=2, gather_dim=1)+ k = all_to_all(k, sp_group, scatter_dim=2, gather_dim=1)+ v = all_to_all(v, sp_group, scatter_dim=2, gather_dim=1)++ q = q.view(1, -1, self.num_heads // sp_size, self.head_dim)+ k = k.view(1, -1, self.num_heads // sp_size, self.head_dim)+ v = v.view(1, -1, self.num_heads // sp_size, self.head_dim)++ # compute attention+ attn_bias = None+ if mask is not None:+ attn_bias = xformers.ops.fmha.BlockDiagonalMask.from_seqlens([N] * B, mask)+ x = xformers.ops.memory_efficient_attention(q, k, v, p=self.attn_drop.p, attn_bias=attn_bias)++ # apply all to all to gather back attention heads and scatter sequence+ x = x.view(B, -1, self.num_heads // sp_size, self.head_dim)+ x = all_to_all(x, sp_group, scatter_dim=1, gather_dim=2)++ # apply output projection+ x = x.view(B, -1, C)+ x = self.proj(x)+ x = self.proj_drop(x)+ return x+++class FinalLayer(nn.Module):+ """+ The final layer of DiT.+ """++ def __init__(self, hidden_size, num_patch, out_channels):+ super().__init__()+ self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)+ self.linear = nn.Linear(hidden_size, num_patch * out_channels, bias=True)+ self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True))++ def forward(self, x, c):+ shift, scale = self.adaLN_modulation(c).chunk(2, dim=1)+ x = modulate(self.norm_final, x, shift, scale)+ x = self.linear(x)+ return x+++class T2IFinalLayer(nn.Module):+ """+ The final layer of PixArt.+ """++ def __init__(self, hidden_size, num_patch, out_channels):+ super().__init__()+ self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)+ self.linear = nn.Linear(hidden_size, num_patch * out_channels, bias=True)+ self.scale_shift_table = nn.Parameter(torch.randn(2, hidden_size) / hidden_size**0.5)+ self.out_channels = out_channels++ def forward(self, x, t):+ shift, scale = (self.scale_shift_table[None] + t[:, None]).chunk(2, dim=1)+ x = t2i_modulate(self.norm_final(x), shift, scale)+ x = self.linear(x)+ return x+++# ===============================================+# Embedding Layers for Timesteps and Class Labels+# ===============================================+++class TimestepEmbedder(nn.Module):+ """+ Embeds scalar timesteps into vector representations.+ """++ def __init__(self, hidden_size, frequency_embedding_size=256):+ super().__init__()+ self.mlp = nn.Sequential(+ nn.Linear(frequency_embedding_size, hidden_size, bias=True),+ nn.SiLU(),+ nn.Linear(hidden_size, hidden_size, bias=True),+ )+ self.frequency_embedding_size = frequency_embedding_size++ @staticmethod+ def timestep_embedding(t, dim, max_period=10000):+ """+ Create sinusoidal timestep embeddings.+ :param t: a 1-D Tensor of N indices, one per batch element.+ These may be fractional.+ :param dim: the dimension of the output.+ :param max_period: controls the minimum frequency of the embeddings.+ :return: an (N, D) Tensor of positional embeddings.+ """+ # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py+ half = dim // 2+ freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half)+ freqs = freqs.to(device=t.device)+ args = t[:, None].float() * freqs[None]+ embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)+ if dim % 2:+ embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)+ return embedding++ def forward(self, t, dtype):+ t_freq = self.timestep_embedding(t, self.frequency_embedding_size)+ if t_freq.dtype != dtype:+ t_freq = t_freq.to(dtype)+ t_emb = self.mlp(t_freq)+ return t_emb+++class LabelEmbedder(nn.Module):+ """+ Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance.+ """++ def __init__(self, num_classes, hidden_size, dropout_prob):+ super().__init__()+ use_cfg_embedding = dropout_prob > 0+ self.embedding_table = nn.Embedding(num_classes + use_cfg_embedding, hidden_size)+ self.num_classes = num_classes+ self.dropout_prob = dropout_prob++ def token_drop(self, labels, force_drop_ids=None):+ """+ Drops labels to enable classifier-free guidance.+ """+ if force_drop_ids is None:+ drop_ids = torch.rand(labels.shape[0]).cuda() < self.dropout_prob+ else:+ drop_ids = force_drop_ids == 1+ labels = torch.where(drop_ids, self.num_classes, labels)+ return labels++ def forward(self, labels, train, force_drop_ids=None):+ use_dropout = self.dropout_prob > 0+ if (train and use_dropout) or (force_drop_ids is not None):+ labels = self.token_drop(labels, force_drop_ids)+ return self.embedding_table(labels)+++class SizeEmbedder(TimestepEmbedder):+ """+ Embeds scalar timesteps into vector representations.+ """++ def __init__(self, hidden_size, frequency_embedding_size=256):+ super().__init__(hidden_size=hidden_size, frequency_embedding_size=frequency_embedding_size)+ self.mlp = nn.Sequential(+ nn.Linear(frequency_embedding_size, hidden_size, bias=True),+ nn.SiLU(),+ nn.Linear(hidden_size, hidden_size, bias=True),+ )+ self.frequency_embedding_size = frequency_embedding_size+ self.outdim = hidden_size++ def forward(self, s, bs):+ if s.ndim == 1:+ s = s[:, None]+ assert s.ndim == 2+ if s.shape[0] != bs:+ s = s.repeat(bs // s.shape[0], 1)+ assert s.shape[0] == bs+ b, dims = s.shape[0], s.shape[1]+ s = rearrange(s, "b d -> (b d)")+ s_freq = self.timestep_embedding(s, self.frequency_embedding_size).to(self.dtype)+ s_emb = self.mlp(s_freq)+ s_emb = rearrange(s_emb, "(b d) d2 -> b (d d2)", b=b, d=dims, d2=self.outdim)+ return s_emb++ @property+ def dtype(self):+ return next(self.parameters()).dtype+++class CaptionEmbedder(nn.Module):+ """+ Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance.+ """++ def __init__(self, in_channels, hidden_size, uncond_prob, act_layer=nn.GELU(approximate="tanh"), token_num=120):+ super().__init__()+ self.y_proj = Mlp(+ in_features=in_channels, hidden_features=hidden_size, out_features=hidden_size, act_layer=act_layer, drop=0+ )+ self.register_buffer("y_embedding", nn.Parameter(torch.randn(token_num, in_channels) / in_channels**0.5))+ self.uncond_prob = uncond_prob++ def token_drop(self, caption, force_drop_ids=None):+ """+ Drops labels to enable classifier-free guidance.+ """+ if force_drop_ids is None:+ drop_ids = torch.rand(caption.shape[0]).cuda() < self.uncond_prob+ else:+ drop_ids = force_drop_ids == 1+ caption = torch.where(drop_ids[:, None, None, None], self.y_embedding, caption)+ return caption++ def forward(self, caption, train, force_drop_ids=None):+ if train:+ assert caption.shape[2:] == self.y_embedding.shape+ use_dropout = self.uncond_prob > 0+ if (train and use_dropout) or (force_drop_ids is not None):+ caption = self.token_drop(caption, force_drop_ids)+ caption = self.y_proj(caption)+ return caption+++# ===============================================+# Sine/Cosine Positional Embedding Functions+# ===============================================+# https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py+++def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0, scale=1.0, base_size=None):+ """+ grid_size: int of the grid height and width+ return:+ pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)+ """+ if not isinstance(grid_size, tuple):+ grid_size = (grid_size, grid_size)++ grid_h = np.arange(grid_size[0], dtype=np.float32) / scale+ grid_w = np.arange(grid_size[1], dtype=np.float32) / scale+ if base_size is not None:+ grid_h *= base_size / grid_size[0]+ grid_w *= base_size / grid_size[1]+ grid = np.meshgrid(grid_w, grid_h) # here w goes first+ grid = np.stack(grid, axis=0)++ grid = grid.reshape([2, 1, grid_size[1], grid_size[0]])+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)+ if cls_token and extra_tokens > 0:+ pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0)+ return pos_embed+++def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):+ assert embed_dim % 2 == 0++ # use half of dimensions to encode grid_h+ emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)+ emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)++ emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)+ return emb+++def get_1d_sincos_pos_embed(embed_dim, length, scale=1.0):+ pos = np.arange(0, length)[..., None] / scale+ return get_1d_sincos_pos_embed_from_grid(embed_dim, pos)+++def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):+ """+ embed_dim: output dimension for each position+ pos: a list of positions to be encoded: size (M,)+ out: (M, D)+ """+ assert embed_dim % 2 == 0+ omega = np.arange(embed_dim // 2, dtype=np.float64)+ omega /= embed_dim / 2.0+ omega = 1.0 / 10000**omega # (D/2,)++ pos = pos.reshape(-1) # (M,)+ out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product++ emb_sin = np.sin(out) # (M, D/2)+ emb_cos = np.cos(out) # (M, D/2)++ emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)+ return embdiff --git a/opensora/models/pixart/__init__.py b/opensora/models/pixart/__init__.pynew file mode 100644index 0000000..622ad39--- /dev/null+++ b/opensora/models/pixart/__init__.py@@ -0,0 +1 @@+from .pixart import PixArt, PixArt_XL_2diff --git a/opensora/models/pixart/pixart.py b/opensora/models/pixart/pixart.pynew file mode 100644index 0000000..daad7dc--- /dev/null+++ b/opensora/models/pixart/pixart.py@@ -0,0 +1,378 @@+# Adapted from PixArt+#+# This source code is licensed under the license found in the+# LICENSE file in the root directory of this source tree.+# --------------------------------------------------------+# References:+# PixArt: https://github.com/PixArt-alpha/PixArt-alpha+# DiT: https://github.com/facebookresearch/DiT/tree/main+# --------------------------------------------------------++import numpy as np+import torch+import torch.nn as nn+from einops import rearrange+from timm.models.layers import DropPath+from timm.models.vision_transformer import Mlp++# from .builder import MODELS+from torch.utils.checkpoint import checkpoint+from opensora.acceleration.checkpoint import auto_grad_checkpoint++from opensora.models.layers.blocks import (+ Attention,+ CaptionEmbedder,+ MultiHeadCrossAttention,+ PatchEmbed3D,+ SeqParallelAttention,+ SeqParallelMultiHeadCrossAttention,+ SizeEmbedder,+ T2IFinalLayer,+ TimestepEmbedder,+ approx_gelu,+ get_1d_sincos_pos_embed,+ get_2d_sincos_pos_embed,+ get_layernorm,+ t2i_modulate,+)+from opensora.registry import MODELS+from opensora.utils.ckpt_utils import load_checkpoint+++class PixArtBlock(nn.Module):+ """+ A PixArt block with adaptive layer norm (adaLN-single) conditioning.+ """++ def __init__(+ self,+ hidden_size,+ num_heads,+ mlp_ratio=4.0,+ drop_path=0.0,+ enable_flashattn=False,+ enable_layernorm_kernel=False,+ enable_sequence_parallelism=False,+ ):+ super().__init__()+ self.hidden_size = hidden_size+ self.enable_flashattn = enable_flashattn+ self._enable_sequence_parallelism = enable_sequence_parallelism++ if enable_sequence_parallelism:+ self.attn_cls = SeqParallelAttention+ self.mha_cls = SeqParallelMultiHeadCrossAttention+ else:+ self.attn_cls = Attention+ self.mha_cls = MultiHeadCrossAttention++ self.norm1 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)+ self.attn = self.attn_cls(+ hidden_size,+ num_heads=num_heads,+ qkv_bias=True,+ enable_flashattn=enable_flashattn,+ )+ self.cross_attn = self.mha_cls(hidden_size, num_heads)+ self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)+ self.mlp = Mlp(+ in_features=hidden_size, hidden_features=int(hidden_size * mlp_ratio), act_layer=approx_gelu, drop=0+ )+ self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()+ self.scale_shift_table = nn.Parameter(torch.randn(6, hidden_size) / hidden_size**0.5)++ def forward(self, x, y, t, mask=None):+ B, N, C = x.shape++ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (+ self.scale_shift_table[None] + t.reshape(B, 6, -1)+ ).chunk(6, dim=1)+ x = x + self.drop_path(gate_msa * self.attn(t2i_modulate(self.norm1(x), shift_msa, scale_msa)).reshape(B, N, C))+ x = x + self.cross_attn(x, y, mask)+ x = x + self.drop_path(gate_mlp * self.mlp(t2i_modulate(self.norm2(x), shift_mlp, scale_mlp)))++ return x++[email protected]_module()+class PixArt(nn.Module):+ """+ Diffusion model with a Transformer backbone.+ """++ def __init__(+ self,+ input_size=(1, 32, 32),+ in_channels=4,+ patch_size=(1, 2, 2),+ hidden_size=1152,+ depth=28,+ num_heads=16,+ mlp_ratio=4.0,+ class_dropout_prob=0.1,+ pred_sigma=True,+ drop_path: float = 0.0,+ no_temporal_pos_emb=False,+ caption_channels=4096,+ model_max_length=120,+ dtype=torch.float32,+ freeze=None,+ space_scale=1.0,+ time_scale=1.0,+ enable_flashattn=False,+ enable_layernorm_kernel=False,+ ):+ super().__init__()+ self.pred_sigma = pred_sigma+ self.in_channels = in_channels+ self.out_channels = in_channels * 2 if pred_sigma else in_channels+ self.hidden_size = hidden_size+ self.patch_size = patch_size+ self.input_size = input_size+ num_patches = np.prod([input_size[i] // patch_size[i] for i in range(3)])+ self.num_patches = num_patches+ self.num_temporal = input_size[0] // patch_size[0]+ self.num_spatial = num_patches // self.num_temporal+ self.base_size = int(np.sqrt(self.num_spatial))+ self.num_heads = num_heads+ self.dtype = dtype+ self.no_temporal_pos_emb = no_temporal_pos_emb+ self.depth = depth+ self.mlp_ratio = mlp_ratio+ self.enable_flashattn = enable_flashattn+ self.enable_layernorm_kernel = enable_layernorm_kernel+ self.space_scale = space_scale+ self.time_scale = time_scale++ self.x_embedder = PatchEmbed3D(patch_size, in_channels, hidden_size)+ self.t_embedder = TimestepEmbedder(hidden_size)+ self.t_block = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True))+ self.y_embedder = CaptionEmbedder(+ in_channels=caption_channels,+ hidden_size=hidden_size,+ uncond_prob=class_dropout_prob,+ act_layer=approx_gelu,+ token_num=model_max_length,+ )++ self.register_buffer("pos_embed", self.get_spatial_pos_embed())+ self.register_buffer("pos_embed_temporal", self.get_temporal_pos_embed())++ drop_path = [x.item() for x in torch.linspace(0, drop_path, depth)] # stochastic depth decay rule+ self.blocks = nn.ModuleList(+ [+ PixArtBlock(+ hidden_size,+ num_heads,+ mlp_ratio=mlp_ratio,+ drop_path=drop_path[i],+ enable_flashattn=enable_flashattn,+ enable_layernorm_kernel=enable_layernorm_kernel,+ )+ for i in range(depth)+ ]+ )+ self.final_layer = T2IFinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)++ self.initialize_weights()+ if freeze is not None:+ assert freeze in ["text"]+ if freeze == "text":+ self.freeze_text()++ def forward(self, x, timestep, y, mask=None):+ """+ Forward pass of PixArt.+ x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images)+ t: (N,) tensor of diffusion timesteps+ y: (N, 1, 120, C) tensor of class labels+ """+ x = x.to(self.dtype)+ timestep = timestep.to(self.dtype)+ y = y.to(self.dtype)++ # embedding+ x = self.x_embedder(x) # (B, N, D)+ x = rearrange(x, "b (t s) d -> b t s d", t=self.num_temporal, s=self.num_spatial)+ x = x + self.pos_embed+ if not self.no_temporal_pos_emb:+ x = rearrange(x, "b t s d -> b s t d")+ x = x + self.pos_embed_temporal+ x = rearrange(x, "b s t d -> b (t s) d")+ else:+ x = rearrange(x, "b t s d -> b (t s) d")++ t = self.t_embedder(timestep, dtype=x.dtype) # (N, D)+ t0 = self.t_block(t)+ y = self.y_embedder(y, self.training) # (N, 1, L, D)+ if mask is not None:+ if mask.shape[0] != y.shape[0]:+ mask = mask.repeat(y.shape[0] // mask.shape[0], 1)+ mask = mask.squeeze(1).squeeze(1)+ y = y.squeeze(1).masked_select(mask.unsqueeze(-1) != 0).view(1, -1, x.shape[-1])+ y_lens = mask.sum(dim=1).tolist()+ else:+ y_lens = [y.shape[2]] * y.shape[0]+ y = y.squeeze(1).view(1, -1, x.shape[-1])++ # blocks+ for block in self.blocks:+ x = auto_grad_checkpoint(block, x, y, t0, y_lens)++ # final process+ x = self.final_layer(x, t) # (N, T, patch_size ** 2 * out_channels)+ x = self.unpatchify(x) # (N, out_channels, H, W)++ # cast to float32 for better accuracy+ x = x.to(torch.float32)+ return x++ def unpatchify(self, x):+ c = self.out_channels+ t, h, w = [self.input_size[i] // self.patch_size[i] for i in range(3)]+ pt, ph, pw = self.patch_size++ x = x.reshape(shape=(x.shape[0], t, h, w, pt, ph, pw, c))+ x = rearrange(x, "n t h w r p q c -> n c t r h p w q")+ imgs = x.reshape(shape=(x.shape[0], c, t * pt, h * ph, w * pw))+ return imgs++ def get_spatial_pos_embed(self, grid_size=None):+ if grid_size is None:+ grid_size = self.input_size[1:]+ pos_embed = get_2d_sincos_pos_embed(+ self.hidden_size,+ (grid_size[0] // self.patch_size[1], grid_size[1] // self.patch_size[2]),+ scale=self.space_scale,+ base_size=self.base_size,+ )+ pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)+ return pos_embed++ def get_temporal_pos_embed(self):+ pos_embed = get_1d_sincos_pos_embed(+ self.hidden_size,+ self.input_size[0] // self.patch_size[0],+ scale=self.time_scale,+ )+ pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)+ return pos_embed++ def freeze_text(self):+ for n, p in self.named_parameters():+ if "cross_attn" in n:+ p.requires_grad = False++ def initialize_weights(self):+ # Initialize transformer layers:+ def _basic_init(module):+ if isinstance(module, nn.Linear):+ torch.nn.init.xavier_uniform_(module.weight)+ if module.bias is not None:+ nn.init.constant_(module.bias, 0)++ self.apply(_basic_init)++ # Initialize patch_embed like nn.Linear (instead of nn.Conv2d):+ w = self.x_embedder.proj.weight.data+ nn.init.xavier_uniform_(w.view([w.shape[0], -1]))++ # Initialize timestep embedding MLP:+ nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02)+ nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02)+ nn.init.normal_(self.t_block[1].weight, std=0.02)++ # Initialize caption embedding MLP:+ nn.init.normal_(self.y_embedder.y_proj.fc1.weight, std=0.02)+ nn.init.normal_(self.y_embedder.y_proj.fc2.weight, std=0.02)++ # Zero-out adaLN modulation layers in PixArt blocks:+ for block in self.blocks:+ nn.init.constant_(block.cross_attn.proj.weight, 0)+ nn.init.constant_(block.cross_attn.proj.bias, 0)++ # Zero-out output layers:+ nn.init.constant_(self.final_layer.linear.weight, 0)+ nn.init.constant_(self.final_layer.linear.bias, 0)++[email protected]_module()+class PixArtMS(PixArt):+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)++ assert self.hidden_size % 3 == 0, "hidden_size must be divisible by 3"+ self.csize_embedder = SizeEmbedder(self.hidden_size // 3)+ self.ar_embedder = SizeEmbedder(self.hidden_size // 3)++ def forward(self, x, timestep, y, mask=None, data_info=None):+ """+ Forward pass of PixArt.+ x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images)+ t: (N,) tensor of diffusion timesteps+ y: (N, 1, 120, C) tensor of class labels+ """+ x = x.to(self.dtype)+ timestep = timestep.to(self.dtype)+ y = y.to(self.dtype)++ c_size = data_info["hw"]+ ar = data_info["ar"]+ pos_embed = self.get_spatial_pos_embed((x.shape[-2], x.shape[-1])).to(x.dtype)++ # embedding+ x = self.x_embedder(x) # (B, N, D)+ x = rearrange(x, "b (t s) d -> b t s d", t=self.num_temporal, s=self.num_spatial)+ x = x + pos_embed.to(x.device)+ if not self.no_temporal_pos_emb:+ x = rearrange(x, "b t s d -> b s t d")+ x = x + self.pos_embed_temporal+ x = rearrange(x, "b s t d -> b (t s) d")+ else:+ x = rearrange(x, "b t s d -> b (t s) d")++ t = self.t_embedder(timestep, dtype=x.dtype) # (N, D)+ B = x.shape[0]+ csize = self.csize_embedder(c_size, B)+ ar = self.ar_embedder(ar, B)+ t = t + torch.cat([csize, ar], dim=1)++ t0 = self.t_block(t)+ y = self.y_embedder(y, self.training) # (N, 1, L, D)+ if mask is not None:+ if mask.shape[0] != y.shape[0]:+ mask = mask.repeat(y.shape[0] // mask.shape[0], 1)+ mask = mask.squeeze(1).squeeze(1)+ y = y.squeeze(1).masked_select(mask.unsqueeze(-1) != 0).view(1, -1, x.shape[-1])+ y_lens = mask.sum(dim=1).tolist()+ else:+ y_lens = [y.shape[2]] * y.shape[0]+ y = y.squeeze(1).view(1, -1, x.shape[-1])++ # blocks+ for block in self.blocks:+ x = block(x, y, t0, y_lens)++ # final process+ x = self.final_layer(x, t) # (N, T, patch_size ** 2 * out_channels)+ x = self.unpatchify(x) # (N, out_channels, H, W)++ # cast to float32 for better accuracy+ x = x.to(torch.float32)+ return x++[email protected]_module("PixArt-XL/2")+def PixArt_XL_2(from_pretrained=None, **kwargs):+ model = PixArt(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)+ return model++[email protected]_module("PixArtMS-XL/2")+def PixArtMS_XL_2(from_pretrained=None, **kwargs):+ model = PixArtMS(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)+ return modeldiff --git a/opensora/models/stdit/__init__.py b/opensora/models/stdit/__init__.pynew file mode 100644index 0000000..3b089af--- /dev/null+++ b/opensora/models/stdit/__init__.py@@ -0,0 +1 @@+from .stdit import STDiTdiff --git a/opensora/models/stdit/stdit.py b/opensora/models/stdit/stdit.pynew file mode 100644index 0000000..c8a2e95--- /dev/null+++ b/opensora/models/stdit/stdit.py@@ -0,0 +1,359 @@++import numpy as np+import torch+import torch.distributed as dist+import torch.nn as nn+from einops import rearrange+from timm.models.layers import DropPath+from timm.models.vision_transformer import Mlp++from opensora.acceleration.checkpoint import auto_grad_checkpoint+from opensora.acceleration.communications import gather_forward_split_backward, split_forward_gather_backward+from opensora.acceleration.parallel_states import get_sequence_parallel_group+from opensora.models.layers.blocks import (+ Attention,+ CaptionEmbedder,+ MultiHeadCrossAttention,+ PatchEmbed3D,+ SeqParallelAttention,+ SeqParallelMultiHeadCrossAttention,+ T2IFinalLayer,+ TimestepEmbedder,+ approx_gelu,+ get_1d_sincos_pos_embed,+ get_2d_sincos_pos_embed,+ get_layernorm,+ t2i_modulate,+)+from opensora.registry import MODELS+from opensora.utils.ckpt_utils import load_checkpoint+++class STDiTBlock(nn.Module):+ def __init__(+ self,+ hidden_size,+ num_heads,+ d_s=None,+ d_t=None,+ mlp_ratio=4.0,+ drop_path=0.0,+ enable_flashattn=False,+ enable_layernorm_kernel=False,+ enable_sequence_parallelism=False,+ ):++ super().__init__()+ self.hidden_size = hidden_size+ self.enable_flashattn = enable_flashattn+ self._enable_sequence_parallelism = enable_sequence_parallelism++ if enable_sequence_parallelism:+ self.attn_cls = SeqParallelAttention+ self.mha_cls = SeqParallelMultiHeadCrossAttention+ else:+ self.attn_cls = Attention+ self.mha_cls = MultiHeadCrossAttention++ self.norm1 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)+ self.attn = self.attn_cls(+ hidden_size,+ num_heads=num_heads,+ qkv_bias=True,+ enable_flashattn=enable_flashattn,+ )+ self.cross_attn = self.mha_cls(hidden_size, num_heads)+ self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)+ self.mlp = Mlp(+ in_features=hidden_size, hidden_features=int(hidden_size * mlp_ratio), act_layer=approx_gelu, drop=0+ )+ self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()+ self.scale_shift_table = nn.Parameter(torch.randn(6, hidden_size) / hidden_size**0.5)++ # temporal attention+ self.d_s = d_s+ self.d_t = d_t++ if self._enable_sequence_parallelism:+ sp_size = dist.get_world_size(get_sequence_parallel_group())+ # make sure d_t is divisible by sp_size+ assert d_t % sp_size == 0+ self.d_t = d_t // sp_size++ self.attn_temp = self.attn_cls(+ hidden_size,+ num_heads=num_heads,+ qkv_bias=True,+ enable_flashattn=self.enable_flashattn,+ )++ def forward(self, x, y, t, mask=None, tpe=None):+ B, N, C = x.shape++ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (+ self.scale_shift_table[None] + t.reshape(B, 6, -1)+ ).chunk(6, dim=1)+ x_m = t2i_modulate(self.norm1(x), shift_msa, scale_msa)++ # spatial branch+ x_s = rearrange(x_m, "b (t s) d -> (b t) s d", t=self.d_t, s=self.d_s)+ x_s = self.attn(x_s)+ x_s = rearrange(x_s, "(b t) s d -> b (t s) d", t=self.d_t, s=self.d_s)+ x = x + self.drop_path(gate_msa * x_s)++ # temporal branch+ x_t = rearrange(x, "b (t s) d -> (b s) t d", t=self.d_t, s=self.d_s)+ if tpe is not None:+ x_t = x_t + tpe+ x_t = self.attn_temp(x_t)+ x_t = rearrange(x_t, "(b s) t d -> b (t s) d", t=self.d_t, s=self.d_s)+ x = x + self.drop_path(gate_msa * x_t)++ # cross attn+ x = x + self.cross_attn(x, y, mask)++ # mlp+ x = x + self.drop_path(gate_mlp * self.mlp(t2i_modulate(self.norm2(x), shift_mlp, scale_mlp)))++ return x++[email protected]_module()+class STDiT(nn.Module):+ def __init__(+ self,+ input_size=(1, 32, 32),+ in_channels=4,+ patch_size=(1, 2, 2),+ hidden_size=1152,+ depth=28,+ num_heads=16,+ mlp_ratio=4.0,+ class_dropout_prob=0.1,+ pred_sigma=True,+ drop_path=0.0,+ no_temporal_pos_emb=False,+ caption_channels=4096,+ model_max_length=120,+ dtype=torch.float32,+ space_scale=1.0,+ time_scale=1.0,+ freeze=None,+ enable_flashattn=False,+ enable_layernorm_kernel=False,+ enable_sequence_parallelism=False,+ ):+ super().__init__()+ self.pred_sigma = pred_sigma+ self.in_channels = in_channels+ self.out_channels = in_channels * 2 if pred_sigma else in_channels+ self.hidden_size = hidden_size+ self.patch_size = patch_size+ self.input_size = input_size+ num_patches = np.prod([input_size[i] // patch_size[i] for i in range(3)])+ self.num_patches = num_patches+ self.num_temporal = input_size[0] // patch_size[0]+ self.num_spatial = num_patches // self.num_temporal+ self.num_heads = num_heads+ self.dtype = dtype+ self.no_temporal_pos_emb = no_temporal_pos_emb+ self.depth = depth+ self.mlp_ratio = mlp_ratio+ self.enable_flashattn = enable_flashattn+ self.enable_layernorm_kernel = enable_layernorm_kernel+ self.space_scale = space_scale+ self.time_scale = time_scale++ self.register_buffer("pos_embed", self.get_spatial_pos_embed())+ self.register_buffer("pos_embed_temporal", self.get_temporal_pos_embed())++ self.x_embedder = PatchEmbed3D(patch_size, in_channels, hidden_size)+ self.t_embedder = TimestepEmbedder(hidden_size)+ self.t_block = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True))+ self.y_embedder = CaptionEmbedder(+ in_channels=caption_channels,+ hidden_size=hidden_size,+ uncond_prob=class_dropout_prob,+ act_layer=approx_gelu,+ token_num=model_max_length,+ )++ drop_path = [x.item() for x in torch.linspace(0, drop_path, depth)]+ self.blocks = nn.ModuleList(+ [+ STDiTBlock(+ self.hidden_size,+ self.num_heads,+ mlp_ratio=self.mlp_ratio,+ drop_path=drop_path[i],+ enable_flashattn=self.enable_flashattn,+ enable_layernorm_kernel=self.enable_layernorm_kernel,+ enable_sequence_parallelism=enable_sequence_parallelism,+ d_t=self.num_temporal,+ d_s=self.num_spatial,+ )+ for i in range(self.depth)+ ]+ )+ self.final_layer = T2IFinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)++ # init model+ self.initialize_weights()+ self.initialize_temporal()+ if freeze is not None:+ assert freeze in ["not_temporal", "text"]+ if freeze == "not_temporal":+ self.freeze_not_temporal()+ elif freeze == "text":+ self.freeze_text()++ # sequence parallel related configs+ self.enable_sequence_parallelism = enable_sequence_parallelism+ if enable_sequence_parallelism:+ self.sp_rank = dist.get_rank(get_sequence_parallel_group())+ else:+ self.sp_rank = None++ def forward(self, x, timestep, y, mask=None):+ """+ Forward pass of PixArt.+ x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images)+ t: (N,) tensor of diffusion timesteps+ y: (N, 1, 120, C) tensor of class labels+ """+ x = x.to(self.dtype)+ timestep = timestep.to(self.dtype)+ y = y.to(self.dtype)++ # embedding+ x = self.x_embedder(x) # (B, N, D)+ x = rearrange(x, "b (t s) d -> b t s d", t=self.num_temporal, s=self.num_spatial)+ x = x + self.pos_embed+ x = rearrange(x, "b t s d -> b (t s) d")++ # shard over the sequence dim if sp is enabled+ if self.enable_sequence_parallelism:+ x = split_forward_gather_backward(x, get_sequence_parallel_group(), dim=1, grad_scale="down")++ t = self.t_embedder(timestep, dtype=x.dtype) # (N, D)+ t0 = self.t_block(t)+ y = self.y_embedder(y, self.training) # (N, 1, L, D)++ if mask is not None:+ if mask.shape[0] != y.shape[0]:+ mask = mask.repeat(y.shape[0] // mask.shape[0], 1)+ mask = mask.squeeze(1).squeeze(1)+ y = y.squeeze(1).masked_select(mask.unsqueeze(-1) != 0).view(1, -1, x.shape[-1])+ y_lens = mask.sum(dim=1).tolist()+ else:+ y_lens = [y.shape[2]] * y.shape[0]+ y = y.squeeze(1).view(1, -1, x.shape[-1])++ # blocks+ for i, block in enumerate(self.blocks):+ if i == 0:+ if self.enable_sequence_parallelism:+ tpe = torch.chunk(+ self.pos_embed_temporal, dist.get_world_size(get_sequence_parallel_group()), dim=1+ )[self.sp_rank].contiguous()+ else:+ tpe = self.pos_embed_temporal+ else:+ tpe = None+ x = auto_grad_checkpoint(block, x, y, t0, y_lens, tpe)++ if self.enable_sequence_parallelism:+ x = gather_forward_split_backward(x, get_sequence_parallel_group(), dim=1, grad_scale="up")++ # final process+ x = self.final_layer(x, t) # (N, T, patch_size ** 2 * out_channels)+ x = self.unpatchify(x) # (N, out_channels, H, W)++ # cast to float32 for better accuracy+ x = x.to(torch.float32)+ return x++ def unpatchify(self, x):+ c = self.out_channels+ t, h, w = [self.input_size[i] // self.patch_size[i] for i in range(3)]+ pt, ph, pw = self.patch_size++ x = x.reshape(shape=(x.shape[0], t, h, w, pt, ph, pw, c))+ x = rearrange(x, "n t h w r p q c -> n c t r h p w q")+ imgs = x.reshape(shape=(x.shape[0], c, t * pt, h * ph, w * pw))+ return imgs++ def get_spatial_pos_embed(self, grid_size=None):+ if grid_size is None:+ grid_size = self.input_size[1:]+ pos_embed = get_2d_sincos_pos_embed(+ self.hidden_size,+ (grid_size[0] // self.patch_size[1], grid_size[1] // self.patch_size[2]),+ scale=self.space_scale,+ )+ pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)+ return pos_embed++ def get_temporal_pos_embed(self):+ pos_embed = get_1d_sincos_pos_embed(+ self.hidden_size,+ self.input_size[0] // self.patch_size[0],+ scale=self.time_scale,+ )+ pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)+ return pos_embed++ def freeze_not_temporal(self):+ for n, p in self.named_parameters():+ if "attn_temp" not in n:+ p.requires_grad = False++ def freeze_text(self):+ for n, p in self.named_parameters():+ if "cross_attn" in n:+ p.requires_grad = False++ def initialize_temporal(self):+ for block in self.blocks:+ nn.init.constant_(block.attn_temp.proj.weight, 0)+ nn.init.constant_(block.attn_temp.proj.bias, 0)++ def initialize_weights(self):+ # Initialize transformer layers:+ def _basic_init(module):+ if isinstance(module, nn.Linear):+ torch.nn.init.xavier_uniform_(module.weight)+ if module.bias is not None:+ nn.init.constant_(module.bias, 0)++ self.apply(_basic_init)++ # Initialize patch_embed like nn.Linear (instead of nn.Conv2d):+ w = self.x_embedder.proj.weight.data+ nn.init.xavier_uniform_(w.view([w.shape[0], -1]))++ # Initialize timestep embedding MLP:+ nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02)+ nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02)+ nn.init.normal_(self.t_block[1].weight, std=0.02)++ # Initialize caption embedding MLP:+ nn.init.normal_(self.y_embedder.y_proj.fc1.weight, std=0.02)+ nn.init.normal_(self.y_embedder.y_proj.fc2.weight, std=0.02)++ # Zero-out adaLN modulation layers in PixArt blocks:+ for block in self.blocks:+ nn.init.constant_(block.cross_attn.proj.weight, 0)+ nn.init.constant_(block.cross_attn.proj.bias, 0)++ # Zero-out output layers:+ nn.init.constant_(self.final_layer.linear.weight, 0)+ nn.init.constant_(self.final_layer.linear.bias, 0)++[email protected]_module("STDiT-XL/2")+def STDiT_XL_2(from_pretrained=None, **kwargs):+ model = STDiT(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)+ if from_pretrained is not None:+ load_checkpoint(model, from_pretrained)+ return modeldiff --git a/opensora/models/text_encoder/__init__.py b/opensora/models/text_encoder/__init__.pynew file mode 100644index 0000000..fa494a2--- /dev/null+++ b/opensora/models/text_encoder/__init__.py@@ -0,0 +1,3 @@+from .t5 import T5Encoder+from .classes import ClassEncoder+from .clip import ClipEncoderdiff --git a/opensora/models/text_encoder/classes.py b/opensora/models/text_encoder/classes.pynew file mode 100644index 0000000..f02c9f2--- /dev/null+++ b/opensora/models/text_encoder/classes.py@@ -0,0 +1,20 @@+import torch++from opensora.registry import MODELS++[email protected]_module("classes")+class ClassEncoder:+ def __init__(self, num_classes, model_max_length=None, device="cuda", dtype=torch.float):+ self.num_classes = num_classes+ self.y_embedder = None++ self.model_max_length = model_max_length+ self.output_dim = None+ self.device = device++ def encode(self, text):+ return dict(y=torch.tensor([int(t) for t in text]).to(self.device))++ def null(self, n):+ return torch.tensor([self.num_classes] * n).to(self.device)diff --git a/opensora/models/text_encoder/clip.py b/opensora/models/text_encoder/clip.pynew file mode 100644index 0000000..cf3f414--- /dev/null+++ b/opensora/models/text_encoder/clip.py@@ -0,0 +1,100 @@+# Adapted from Latte++# This source code is licensed under the license found in the+# LICENSE file in the root directory of this source tree.+# --------------------------------------------------------+# References:+# Latte: https://github.com/Vchitect/Latte+# DiT: https://github.com/facebookresearch/DiT/tree/main+# --------------------------------------------------------+++import torch+import torch.nn as nn+import transformers+from transformers import CLIPTextModel, CLIPTokenizer++from opensora.registry import MODELS++transformers.logging.set_verbosity_error()+++class AbstractEncoder(nn.Module):+ def __init__(self):+ super().__init__()++ def encode(self, *args, **kwargs):+ raise NotImplementedError+++class FrozenCLIPEmbedder(AbstractEncoder):+ """Uses the CLIP transformer encoder for text (from Hugging Face)"""++ def __init__(self, path="openai/clip-vit-huge-patch14", device="cuda", max_length=77):+ super().__init__()+ self.tokenizer = CLIPTokenizer.from_pretrained(path)+ self.transformer = CLIPTextModel.from_pretrained(path)+ self.device = device+ self.max_length = max_length+ self._freeze()++ def _freeze(self):+ self.transformer = self.transformer.eval()+ for param in self.parameters():+ param.requires_grad = False++ def forward(self, text):+ batch_encoding = self.tokenizer(+ text,+ truncation=True,+ max_length=self.max_length,+ return_length=True,+ return_overflowing_tokens=False,+ padding="max_length",+ return_tensors="pt",+ )+ tokens = batch_encoding["input_ids"].to(self.device)+ outputs = self.transformer(input_ids=tokens)++ z = outputs.last_hidden_state+ pooled_z = outputs.pooler_output+ return z, pooled_z++ def encode(self, text):+ return self(text)++[email protected]_module("clip")+class ClipEncoder:+ """+ Embeds text prompt into vector representations. Also handles text dropout for classifier-free guidance.+ """++ def __init__(+ self,+ from_pretrained,+ model_max_length=77,+ device="cuda",+ dtype=torch.float,+ ):+ super().__init__()+ assert from_pretrained is not None, "Please specify the path to the T5 model"++ self.text_encoder = FrozenCLIPEmbedder(path=from_pretrained, max_length=model_max_length).to(device, dtype)+ self.y_embedder = None++ self.model_max_length = model_max_length+ self.output_dim = self.text_encoder.transformer.config.hidden_size++ def encode(self, text):+ _, pooled_embeddings = self.text_encoder.encode(text)+ y = pooled_embeddings.unsqueeze(1).unsqueeze(1)+ return dict(y=y)++ def null(self, n):+ null_y = self.y_embedder.y_embedding[None].repeat(n, 1, 1)[:, None]+ return null_y++ def to(self, dtype):+ self.text_encoder = self.text_encoder.to(dtype)+ return selfdiff --git a/opensora/models/text_encoder/t5.py b/opensora/models/text_encoder/t5.pynew file mode 100644index 0000000..672e6d9--- /dev/null+++ b/opensora/models/text_encoder/t5.py@@ -0,0 +1,345 @@+# Adapted from PixArt++# This source code is licensed under the license found in the+# LICENSE file in the root directory of this source tree.+# --------------------------------------------------------+# References:+# PixArt: https://github.com/PixArt-alpha/PixArt-alpha+# T5: https://github.com/google-research/text-to-text-transfer-transformer+# --------------------------------------------------------+++import html+import os+import re+import urllib.parse as ul++import ftfy+import torch+from bs4 import BeautifulSoup+from huggingface_hub import hf_hub_download+from transformers import AutoTokenizer, T5EncoderModel++from opensora.registry import MODELS+++class T5Embedder:+ available_models = ["t5-v1_1-xxl"]+ bad_punct_regex = re.compile(+ r"[" + "#®•©™&@·º½¾¿¡§~" + "\)" + "\(" + "\]" + "\[" + "\}" + "\{" + "\|" + "\\" + "\/" + "\*" + r"]{1,}"+ ) # noqa++ def __init__(+ self,+ device,+ dir_or_name="t5-v1_1-xxl",+ *,+ local_cache=False,+ cache_dir=None,+ hf_token=None,+ use_text_preprocessing=True,+ t5_model_kwargs=None,+ torch_dtype=None,+ use_offload_folder=None,+ model_max_length=120,+ ):+ self.device = torch.device(device)+ self.torch_dtype = torch_dtype or torch.bfloat16+ if t5_model_kwargs is None:+ t5_model_kwargs = {"low_cpu_mem_usage": True, "torch_dtype": self.torch_dtype}+ if use_offload_folder is not None:+ t5_model_kwargs["offload_folder"] = use_offload_folder+ t5_model_kwargs["device_map"] = {+ "shared": self.device,+ "encoder.embed_tokens": self.device,+ "encoder.block.0": self.device,+ "encoder.block.1": self.device,+ "encoder.block.2": self.device,+ "encoder.block.3": self.device,+ "encoder.block.4": self.device,+ "encoder.block.5": self.device,+ "encoder.block.6": self.device,+ "encoder.block.7": self.device,+ "encoder.block.8": self.device,+ "encoder.block.9": self.device,+ "encoder.block.10": self.device,+ "encoder.block.11": self.device,+ "encoder.block.12": "disk",+ "encoder.block.13": "disk",+ "encoder.block.14": "disk",+ "encoder.block.15": "disk",+ "encoder.block.16": "disk",+ "encoder.block.17": "disk",+ "encoder.block.18": "disk",+ "encoder.block.19": "disk",+ "encoder.block.20": "disk",+ "encoder.block.21": "disk",+ "encoder.block.22": "disk",+ "encoder.block.23": "disk",+ "encoder.final_layer_norm": "disk",+ "encoder.dropout": "disk",+ }+ else:+ t5_model_kwargs["device_map"] = {"shared": self.device, "encoder": self.device}++ self.use_text_preprocessing = use_text_preprocessing+ self.hf_token = hf_token+ self.cache_dir = cache_dir or os.path.expanduser("~/.cache/IF_")+ self.dir_or_name = dir_or_name+ tokenizer_path, path = dir_or_name, dir_or_name+ if local_cache:+ cache_dir = os.path.join(self.cache_dir, dir_or_name)+ tokenizer_path, path = cache_dir, cache_dir+ elif dir_or_name in self.available_models:+ cache_dir = os.path.join(self.cache_dir, dir_or_name)+ for filename in [+ "config.json",+ "special_tokens_map.json",+ "spiece.model",+ "tokenizer_config.json",+ "pytorch_model.bin.index.json",+ "pytorch_model-00001-of-00002.bin",+ "pytorch_model-00002-of-00002.bin",+ ]:+ hf_hub_download(+ repo_id=f"DeepFloyd/{dir_or_name}",+ filename=filename,+ cache_dir=cache_dir,+ force_filename=filename,+ token=self.hf_token,+ )+ tokenizer_path, path = cache_dir, cache_dir+ else:+ cache_dir = os.path.join(self.cache_dir, "t5-v1_1-xxl")+ for filename in [+ "config.json",+ "special_tokens_map.json",+ "spiece.model",+ "tokenizer_config.json",+ ]:+ hf_hub_download(+ repo_id="DeepFloyd/t5-v1_1-xxl",+ filename=filename,+ cache_dir=cache_dir,+ force_filename=filename,+ token=self.hf_token,+ )+ tokenizer_path = cache_dir++ print(tokenizer_path)+ self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)+ self.model = T5EncoderModel.from_pretrained(path, **t5_model_kwargs).eval()+ self.model_max_length = model_max_length++ def get_text_embeddings(self, texts):+ texts = [self.text_preprocessing(text) for text in texts]++ text_tokens_and_mask = self.tokenizer(+ texts,+ max_length=self.model_max_length,+ padding="max_length",+ truncation=True,+ return_attention_mask=True,+ add_special_tokens=True,+ return_tensors="pt",+ )++ text_tokens_and_mask["input_ids"] = text_tokens_and_mask["input_ids"]+ text_tokens_and_mask["attention_mask"] = text_tokens_and_mask["attention_mask"]++ with torch.no_grad():+ text_encoder_embs = self.model(+ input_ids=text_tokens_and_mask["input_ids"].to(self.device),+ attention_mask=text_tokens_and_mask["attention_mask"].to(self.device),+ )["last_hidden_state"].detach()+ return text_encoder_embs, text_tokens_and_mask["attention_mask"].to(self.device)++ def text_preprocessing(self, text):+ if self.use_text_preprocessing:+ # The exact text cleaning as was in the training stage:+ text = self.clean_caption(text)+ text = self.clean_caption(text)+ return text+ else:+ return text.lower().strip()++ @staticmethod+ def basic_clean(text):+ text = ftfy.fix_text(text)+ text = html.unescape(html.unescape(text))+ return text.strip()++ def clean_caption(self, caption):+ caption = str(caption)+ caption = ul.unquote_plus(caption)+ caption = caption.strip().lower()+ caption = re.sub("<person>", "person", caption)+ # urls:+ caption = re.sub(+ r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa+ "",+ caption,+ ) # regex for urls+ caption = re.sub(+ r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa+ "",+ caption,+ ) # regex for urls+ # html:+ caption = BeautifulSoup(caption, features="html.parser").text++ # @<nickname>+ caption = re.sub(r"@[\w\d]+\b", "", caption)++ # 31C0—31EF CJK Strokes+ # 31F0—31FF Katakana Phonetic Extensions+ # 3200—32FF Enclosed CJK Letters and Months+ # 3300—33FF CJK Compatibility+ # 3400—4DBF CJK Unified Ideographs Extension A+ # 4DC0—4DFF Yijing Hexagram Symbols+ # 4E00—9FFF CJK Unified Ideographs+ caption = re.sub(r"[\u31c0-\u31ef]+", "", caption)+ caption = re.sub(r"[\u31f0-\u31ff]+", "", caption)+ caption = re.sub(r"[\u3200-\u32ff]+", "", caption)+ caption = re.sub(r"[\u3300-\u33ff]+", "", caption)+ caption = re.sub(r"[\u3400-\u4dbf]+", "", caption)+ caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption)+ caption = re.sub(r"[\u4e00-\u9fff]+", "", caption)+ #######################################################++ # все виды тире / all types of dash --> "-"+ caption = re.sub(+ r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa+ "-",+ caption,+ )++ # кавычки к одному стандарту+ caption = re.sub(r"[`´«»“”¨]", '"', caption)+ caption = re.sub(r"[‘’]", "'", caption)++ # "+ caption = re.sub(r""?", "", caption)+ # &+ caption = re.sub(r"&", "", caption)++ # ip adresses:+ caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption)++ # article ids:+ caption = re.sub(r"\d:\d\d\s+$", "", caption)++ # \n+ caption = re.sub(r"\\n", " ", caption)++ # "#123"+ caption = re.sub(r"#\d{1,3}\b", "", caption)+ # "#12345.."+ caption = re.sub(r"#\d{5,}\b", "", caption)+ # "123456.."+ caption = re.sub(r"\b\d{6,}\b", "", caption)+ # filenames:+ caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption)++ #+ caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT"""+ caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT"""++ caption = re.sub(self.bad_punct_regex, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT+ caption = re.sub(r"\s+\.\s+", r" ", caption) # " . "++ # this-is-my-cute-cat / this_is_my_cute_cat+ regex2 = re.compile(r"(?:\-|\_)")+ if len(re.findall(regex2, caption)) > 3:+ caption = re.sub(regex2, " ", caption)++ caption = self.basic_clean(caption)++ caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640+ caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc+ caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231++ caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)+ caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)+ caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption)+ caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption)+ caption = re.sub(r"\bpage\s+\d+\b", "", caption)++ caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a...++ caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption)++ caption = re.sub(r"\b\s+\:\s+", r": ", caption)+ caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption)+ caption = re.sub(r"\s+", " ", caption)++ caption.strip()++ caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption)+ caption = re.sub(r"^[\'\_,\-\:;]", r"", caption)+ caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption)+ caption = re.sub(r"^\.\S+$", "", caption)++ return caption.strip()++[email protected]_module("t5")+class T5Encoder:+ def __init__(+ self,+ from_pretrained=None,+ model_max_length=120,+ device="cuda",+ dtype=torch.float,+ local_cache=True,+ shardformer=False,+ ):+ assert from_pretrained is not None, "Please specify the path to the T5 model"++ self.t5 = T5Embedder(+ device=device,+ torch_dtype=dtype,+ local_cache=local_cache,+ cache_dir=from_pretrained,+ model_max_length=model_max_length,+ )+ self.t5.model.to(dtype=dtype)+ self.y_embedder = None++ self.model_max_length = model_max_length+ self.output_dim = self.t5.model.config.d_model++ if shardformer:+ self.shardformer_t5()++ def shardformer_t5(self):+ from colossalai.shardformer import ShardConfig, ShardFormer++ from opensora.acceleration.shardformer.policy.t5_encoder import T5EncoderPolicy+ from opensora.utils.misc import requires_grad++ shard_config = ShardConfig(+ tensor_parallel_process_group=None,+ pipeline_stage_manager=None,+ enable_tensor_parallelism=False,+ enable_fused_normalization=False,+ enable_flash_attention=False,+ enable_jit_fused=True,+ enable_sequence_parallelism=False,+ enable_sequence_overlap=False,+ )+ shard_former = ShardFormer(shard_config=shard_config)+ optim_model, _ = shard_former.optimize(self.t5.model, policy=T5EncoderPolicy())+ self.t5.model = optim_model.half()++ # ensure the weights are frozen+ requires_grad(self.t5.model, False)++ def encode(self, text):+ caption_embs, emb_masks = self.t5.get_text_embeddings(text)+ caption_embs = caption_embs[:, None]+ return dict(y=caption_embs, mask=emb_masks)++ def null(self, n):+ null_y = self.y_embedder.y_embedding[None].repeat(n, 1, 1)[:, None]+ return null_ydiff --git a/opensora/models/vae/__init__.py b/opensora/models/vae/__init__.pynew file mode 100644index 0000000..04a09c9--- /dev/null+++ b/opensora/models/vae/__init__.py@@ -0,0 +1 @@+from .vae import VideoAutoencoderKL, VideoAutoencoderKLTemporalDecoderdiff --git a/opensora/models/vae/vae.py b/opensora/models/vae/vae.pynew file mode 100644index 0000000..3761f0a--- /dev/null+++ b/opensora/models/vae/vae.py@@ -0,0 +1,78 @@+import torch+import torch.nn as nn+from diffusers.models import AutoencoderKL, AutoencoderKLTemporalDecoder+from einops import rearrange++from opensora.registry import MODELS++[email protected]_module()+class VideoAutoencoderKL(nn.Module):+ def __init__(self, from_pretrained=None, split=None):+ super().__init__()+ self.module = AutoencoderKL.from_pretrained(from_pretrained)+ self.out_channels = self.module.config.latent_channels+ self.patch_size = (1, 8, 8)+ self.split = split++ def encode(self, x):+ # x: (B, C, T, H, W)+ B = x.shape[0]+ x = rearrange(x, "B C T H W -> (B T) C H W")++ if self.split is None:+ x = self.module.encode(x).latent_dist.sample().mul_(0.18215)+ else:+ bs = x.shape[0] // self.split+ x_out = []+ for i in range(self.split):+ x_out.append(self.module.encode(x[i * bs : (i + 1) * bs]).latent_dist.sample().mul_(0.18215))+ x = torch.cat(x_out, dim=0)+ x = rearrange(x, "(B T) C H W -> B C T H W", B=B)+ return x++ def decode(self, x):+ # x: (B, C, T, H, W)+ B = x.shape[0]+ x = rearrange(x, "B C T H W -> (B T) C H W")+ if self.split is None:+ x = self.module.decode(x / 0.18215).sample+ else:+ bs = x.shape[0] // self.split+ x_out = []+ for i in range(self.split):+ x_out.append(self.module.decode(x[i * bs : (i + 1) * bs] / 0.18215).sample)+ x = torch.cat(x_out, dim=0)+ x = rearrange(x, "(B T) C H W -> B C T H W", B=B)+ return x++ def get_latent_size(self, input_size):+ for i in range(3):+ assert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"+ input_size = [input_size[i] // self.patch_size[i] for i in range(3)]+ return input_size++[email protected]_module()+class VideoAutoencoderKLTemporalDecoder(nn.Module):+ def __init__(self, from_pretrained=None):+ super().__init__()+ self.module = AutoencoderKLTemporalDecoder.from_pretrained(from_pretrained)+ self.out_channels = self.module.config.latent_channels+ self.patch_size = (1, 8, 8)++ def encode(self, x):+ raise NotImplementedError++ def decode(self, x):+ B, _, T = x.shape[:3]+ x = rearrange(x, "B C T H W -> (B T) C H W")+ x = self.module.decode(x / 0.18215, num_frames=T).sample+ x = rearrange(x, "(B T) C H W -> B C T H W", B=B)+ return x++ def get_latent_size(self, input_size):+ for i in range(3):+ assert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"+ input_size = [input_size[i] // self.patch_size[i] for i in range(3)]+ return input_size