Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

added open_sora package (#66)

xyupeng9aab3ad343fe
diff --git a/.gitignore b/.gitignore
index 47e4823..b5fc089 100644
--- a/.gitignore
+++ b/.gitignore
@@ -157,7 +157,7 @@ cython_debug/
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
.vscode/
# misc files
diff --git a/opensora/acceleration/__init__.py b/opensora/acceleration/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/opensora/acceleration/checkpoint.py b/opensora/acceleration/checkpoint.py
new file mode 100644
index 0000000..d832a01
--- /dev/null
+++ b/opensora/acceleration/checkpoint.py
@@ -0,0 +1,24 @@
+from collections.abc import Iterable
+
+import torch.nn as nn
+from torch.utils.checkpoint import checkpoint, checkpoint_sequential
+
+
+def set_grad_checkpoint(model, use_fp32_attention=False, gc_step=1):
+ assert isinstance(model, nn.Module)
+
+ def set_attr(module):
+ module.grad_checkpointing = True
+ module.fp32_attention = use_fp32_attention
+ module.grad_checkpointing_step = gc_step
+
+ model.apply(set_attr)
+
+
+def auto_grad_checkpoint(module, *args, **kwargs):
+ if getattr(module, "grad_checkpointing", False):
+ if not isinstance(module, Iterable):
+ return checkpoint(module, *args, **kwargs)
+ gc_step = module[0].grad_checkpointing_step
+ return checkpoint_sequential(module, gc_step, *args, **kwargs)
+ return module(*args, **kwargs)
diff --git a/opensora/acceleration/communications.py b/opensora/acceleration/communications.py
new file mode 100644
index 0000000..b07d4c0
--- /dev/null
+++ b/opensora/acceleration/communications.py
@@ -0,0 +1,187 @@
+import torch
+import torch.distributed as dist
+
+
+# ====================
+# All-To-All
+# ====================
+def _all_to_all(
+ input_: torch.Tensor,
+ world_size: int,
+ group: dist.ProcessGroup,
+ scatter_dim: int,
+ gather_dim: int,
+):
+ input_list = [
+ t.contiguous() for t in torch.tensor_split(input_, world_size, scatter_dim)
+ ]
+ output_list = [torch.empty_like(input_list[0]) for _ in range(world_size)]
+ dist.all_to_all(output_list, input_list, group=group)
+ return torch.cat(output_list, dim=gather_dim).contiguous()
+
+class _AllToAll(torch.autograd.Function):
+ """All-to-all communication.
+
+ Args:
+ input_: input matrix
+ process_group: communication group
+ scatter_dim: scatter dimension
+ gather_dim: gather dimension
+ """
+
+ @staticmethod
+ def forward(ctx, input_, process_group, scatter_dim, gather_dim):
+ ctx.process_group = process_group
+ ctx.scatter_dim = scatter_dim
+ ctx.gather_dim = gather_dim
+ ctx.world_size = dist.get_world_size(process_group)
+ output = _all_to_all(
+ input_, ctx.world_size, process_group, scatter_dim, gather_dim
+ )
+ return output
+
+ @staticmethod
+ def backward(ctx, grad_output):
+ grad_output = _all_to_all(
+ grad_output,
+ ctx.world_size,
+ ctx.process_group,
+ ctx.gather_dim,
+ ctx.scatter_dim,
+ )
+ return (
+ grad_output,
+ None,
+ None,
+ None,
+ )
+
+
+def all_to_all(
+ input_: torch.Tensor,
+ process_group: dist.ProcessGroup,
+ scatter_dim: int = 2,
+ gather_dim: int = 1,
+):
+ return _AllToAll.apply(input_, process_group, scatter_dim, gather_dim)
+
+def _gather(
+ input_: torch.Tensor,
+ world_size: int,
+ group: dist.ProcessGroup,
+ gather_dim: int,
+):
+ if gather_list is None:
+ gather_list = [torch.empty_like(input_) for _ in range(world_size)]
+ dist.gather(input_, gather_list, group=group, gather_dim=gather_dim)
+ return gather_list
+
+# ====================
+# Gather-Split
+# ====================
+
+def _split(input_, pg: dist.ProcessGroup, dim=-1):
+ # skip if only one rank involved
+ world_size = dist.get_world_size(pg)
+ rank = dist.get_rank(pg)
+ if world_size == 1:
+ return input_
+
+ # Split along last dimension.
+ dim_size = input_.size(dim)
+ assert dim_size % world_size == 0, (
+ f"The dimension to split ({dim_size}) is not a multiple of world size ({world_size}), "
+ f"cannot split tensor evenly"
+ )
+
+ tensor_list = torch.split(input_, dim_size // world_size, dim=dim)
+ output = tensor_list[rank].contiguous()
+
+ return output
+
+
+def _gather(input_, pg: dist.ProcessGroup, dim=-1):
+ # skip if only one rank involved
+ input_ = input_.contiguous()
+ world_size = dist.get_world_size(pg)
+ rank = dist.get_rank(pg)
+
+ if world_size == 1:
+ return input_
+
+ # all gather
+ tensor_list = [torch.empty_like(input_) for _ in range(world_size)]
+ assert input_.device.type == "cuda"
+ torch.distributed.all_gather(tensor_list, input_, group=pg)
+
+ # concat
+ output = torch.cat(tensor_list, dim=dim).contiguous()
+
+ return output
+
+class _GatherForwardSplitBackward(torch.autograd.Function):
+ """Gather the input from model parallel region and concatenate.
+
+ Args:
+ input_: input matrix.
+ process_group: parallel mode.
+ dim: dimension
+ """
+
+ @staticmethod
+ def symbolic(graph, input_):
+
+ return _gather(input_)
+
+ @staticmethod
+ def forward(ctx, input_, process_group, dim, grad_scale):
+ ctx.mode = process_group
+ ctx.dim = dim
+ ctx.grad_scale = grad_scale
+ return _gather(input_, process_group, dim)
+
+ @staticmethod
+ def backward(ctx, grad_output):
+ if ctx.grad_scale == "up":
+ grad_output = grad_output * dist.get_world_size(ctx.mode)
+ elif ctx.grad_scale == "down":
+ grad_output = grad_output / dist.get_world_size(ctx.mode)
+
+ return _split(grad_output, ctx.mode, ctx.dim), None, None, None
+
+
+class _SplitForwardGatherBackward(torch.autograd.Function):
+ """
+ Split the input and keep only the corresponding chuck to the rank.
+
+ Args:
+ input_: input matrix.
+ process_group: parallel mode.
+ dim: dimension
+ """
+
+ @staticmethod
+ def symbolic(graph, input_):
+ return _split(input_)
+
+ @staticmethod
+ def forward(ctx, input_, process_group, dim, grad_scale):
+ ctx.mode = process_group
+ ctx.dim = dim
+ ctx.grad_scale = grad_scale
+ return _split(input_, process_group, dim)
+
+ @staticmethod
+ def backward(ctx, grad_output):
+ if ctx.grad_scale == "up":
+ grad_output = grad_output * dist.get_world_size(ctx.mode)
+ elif ctx.grad_scale == "down":
+ grad_output = grad_output / dist.get_world_size(ctx.mode)
+ return _gather(grad_output, ctx.mode, ctx.dim), None, None, None
+
+
+def split_forward_gather_backward(input_, process_group, dim, grad_scale=1.0):
+ return _SplitForwardGatherBackward.apply(input_, process_group, dim, grad_scale)
+
+def gather_forward_split_backward(input_, process_group, dim, grad_scale=None):
+ return _GatherForwardSplitBackward.apply(input_, process_group, dim, grad_scale)
diff --git a/opensora/acceleration/parallel_states.py b/opensora/acceleration/parallel_states.py
new file mode 100644
index 0000000..a8d71e8
--- /dev/null
+++ b/opensora/acceleration/parallel_states.py
@@ -0,0 +1,15 @@
+import torch.distributed as dist
+
+_GLOBAL_PARALLEL_GROUPS = dict()
+
+def set_data_parallel_group(group: dist.ProcessGroup):
+ _GLOBAL_PARALLEL_GROUPS["data"] = group
+
+def get_data_parallel_group():
+ return _GLOBAL_PARALLEL_GROUPS.get("data", None)
+
+def set_sequence_parallel_group(group: dist.ProcessGroup):
+ _GLOBAL_PARALLEL_GROUPS["sequence"] = group
+
+def get_sequence_parallel_group():
+ return _GLOBAL_PARALLEL_GROUPS.get("sequence", None)
\ No newline at end of file
diff --git a/opensora/acceleration/plugin.py b/opensora/acceleration/plugin.py
new file mode 100644
index 0000000..7c9be5d
--- /dev/null
+++ b/opensora/acceleration/plugin.py
@@ -0,0 +1,102 @@
+import random
+from typing import Optional
+
+import numpy as np
+import torch
+from colossalai.booster.plugin import LowLevelZeroPlugin
+from colossalai.cluster import ProcessGroupMesh
+from torch.utils.data import DataLoader
+from torch.utils.data.distributed import DistributedSampler
+
+DP_AXIS, SP_AXIS = 0, 1
+
+
+class ZeroSeqParallelPlugin(LowLevelZeroPlugin):
+ def __init__(
+ self,
+ sp_size: int = 1,
+ stage: int = 2,
+ precision: str = "fp16",
+ initial_scale: float = 2**32,
+ min_scale: float = 1,
+ growth_factor: float = 2,
+ backoff_factor: float = 0.5,
+ growth_interval: int = 1000,
+ hysteresis: int = 2,
+ max_scale: float = 2**32,
+ max_norm: float = 0.0,
+ norm_type: float = 2.0,
+ reduce_bucket_size_in_m: int = 12,
+ communication_dtype: Optional[torch.dtype] = None,
+ overlap_communication: bool = True,
+ cpu_offload: bool = False,
+ master_weights: bool = True,
+ verbose: bool = False,
+ ) -> None:
+ super().__init__(
+ stage=stage,
+ precision=precision,
+ initial_scale=initial_scale,
+ min_scale=min_scale,
+ growth_factor=growth_factor,
+ backoff_factor=backoff_factor,
+ growth_interval=growth_interval,
+ hysteresis=hysteresis,
+ max_scale=max_scale,
+ max_norm=max_norm,
+ norm_type=norm_type,
+ reduce_bucket_size_in_m=reduce_bucket_size_in_m,
+ communication_dtype=communication_dtype,
+ overlap_communication=overlap_communication,
+ cpu_offload=cpu_offload,
+ master_weights=master_weights,
+ verbose=verbose,
+ )
+ self.sp_size = sp_size
+ assert self.world_size % sp_size == 0, "world_size must be divisible by sp_size"
+ self.dp_size = self.world_size // sp_size
+ self.pg_mesh = ProcessGroupMesh(self.dp_size, self.sp_size)
+ self.dp_group = self.pg_mesh.get_group_along_axis(DP_AXIS)
+ self.sp_group = self.pg_mesh.get_group_along_axis(SP_AXIS)
+ self.dp_rank = self.pg_mesh.coordinate(DP_AXIS)
+ self.sp_rank = self.pg_mesh.coordinate(SP_AXIS)
+
+ def __del__(self):
+ """Destroy the prcess groups in ProcessGroupMesh"""
+ self.pg_mesh.destroy_mesh_process_groups()
+
+ def prepare_dataloader(
+ self,
+ dataset,
+ batch_size,
+ shuffle=False,
+ seed=1024,
+ drop_last=False,
+ pin_memory=False,
+ num_workers=0,
+ distributed_sampler_cls=None,
+ **kwargs
+ ):
+ _kwargs = kwargs.copy()
+ distributed_sampler_cls = distributed_sampler_cls or DistributedSampler
+ sampler = distributed_sampler_cls(
+ dataset, num_replicas=self.dp_size, rank=self.dp_rank, shuffle=shuffle
+ )
+
+ # Deterministic dataloader
+ def seed_worker(worker_id):
+ worker_seed = seed
+ np.random.seed(worker_seed)
+ torch.manual_seed(worker_seed)
+ random.seed(worker_seed)
+
+ return DataLoader(
+ dataset,
+ batch_size=batch_size,
+ sampler=sampler,
+ worker_init_fn=seed_worker,
+ drop_last=drop_last,
+ pin_memory=pin_memory,
+ num_workers=num_workers,
+ **_kwargs,
+ )
\ No newline at end of file
diff --git a/opensora/acceleration/shardformer/modeling/__init__.py b/opensora/acceleration/shardformer/modeling/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/opensora/acceleration/shardformer/modeling/t5.py b/opensora/acceleration/shardformer/modeling/t5.py
new file mode 100644
index 0000000..d494c1c
--- /dev/null
+++ b/opensora/acceleration/shardformer/modeling/t5.py
@@ -0,0 +1,38 @@
+import torch
+import torch.nn as nn
+
+
+class T5LayerNorm(nn.Module):
+ def __init__(self, hidden_size, eps=1e-6):
+ """
+ Construct a layernorm module in the T5 style. No bias and no subtraction of mean.
+ """
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(hidden_size))
+ self.variance_epsilon = eps
+
+ def forward(self, hidden_states):
+ # T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean
+ # Square Layer Normalization https://arxiv.org/abs/1910.07467 thus varience is calculated
+ # w/o mean and there is no bias. Additionally we want to make sure that the accumulation for
+ # half-precision inputs is done in fp32
+
+ variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+
+ # convert into half-precision if necessary
+ if self.weight.dtype in [torch.float16, torch.bfloat16]:
+ hidden_states = hidden_states.to(self.weight.dtype)
+
+ return self.weight * hidden_states
+
+ @staticmethod
+ def from_native_module(module, *args, **kwargs):
+ assert module.__class__.__name__ == "FusedRMSNorm", \
+ "Recovering T5LayerNorm requires the original layer to be apex's Fused RMS Norm." \
+ "Apex's fused norm is automatically used by Hugging Face Transformers https://github.com/huggingface/transformers/blob/main/src/transformers/models/t5/modeling_t5.py#L265C5-L265C48"
+
+ layer_norm = T5LayerNorm(module.normalized_shape, eps=module.eps)
+ layer_norm.weight.data.copy_(module.weight.data)
+ layer_norm = layer_norm.to(module.weight.device)
+ return layer_norm
diff --git a/opensora/acceleration/shardformer/policy/__init__.py b/opensora/acceleration/shardformer/policy/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/opensora/acceleration/shardformer/policy/t5_encoder.py b/opensora/acceleration/shardformer/policy/t5_encoder.py
new file mode 100644
index 0000000..e7ce3ac
--- /dev/null
+++ b/opensora/acceleration/shardformer/policy/t5_encoder.py
@@ -0,0 +1,81 @@
+from colossalai.shardformer.modeling.jit import get_jit_fused_dropout_add_func
+from colossalai.shardformer.modeling.t5 import (
+ get_jit_fused_T5_layer_ff_forward,
+ get_T5_layer_self_attention_forward,
+)
+from colossalai.shardformer.policies.base_policy import Policy, SubModuleReplacementDescription
+
+class T5EncoderPolicy(Policy):
+
+ def config_sanity_check(self):
+ assert not self.shard_config.enable_tensor_parallelism
+ assert not self.shard_config.enable_flash_attention
+
+ def preprocess(self):
+ return self.model
+
+ def module_policy(self):
+ from transformers.models.t5.modeling_t5 import (
+ T5LayerFF,
+ T5LayerSelfAttention,
+ T5Stack,
+ )
+
+ policy = {}
+
+ # check whether apex is installed
+ try:
+ from apex.normalization import FusedRMSNorm
+ from opensora.acceleration.shardformer.modeling.t5 import T5LayerNorm
+
+ # recover hf from fused rms norm to T5 norm which is faster
+ self.append_or_create_submodule_replacement(
+ description=SubModuleReplacementDescription(
+ suffix="layer_norm",
+ target_module=T5LayerNorm,
+ ),
+ policy=policy,
+ target_key=T5LayerFF,
+ )
+ self.append_or_create_submodule_replacement(
+ description=SubModuleReplacementDescription(
+ suffix="layer_norm",
+ target_module=T5LayerNorm
+ ),
+ policy=policy,
+ target_key=T5LayerSelfAttention,
+ )
+ self.append_or_create_submodule_replacement(
+ description=SubModuleReplacementDescription(
+ suffix="final_layer_norm",
+ target_module=T5LayerNorm
+ ),
+ policy=policy,
+ target_key=T5Stack,
+ )
+ except (ImportError, ModuleNotFoundError):
+ pass
+
+ # use jit operator
+ if self.shard_config.enable_jit_fused:
+ self.append_or_create_method_replacement(
+ description={
+ "forward": get_jit_fused_T5_layer_ff_forward(),
+ "dropout_add": get_jit_fused_dropout_add_func(),
+ },
+ policy=policy,
+ target_key=T5LayerFF,
+ )
+ self.append_or_create_method_replacement(
+ description={
+ "forward": get_T5_layer_self_attention_forward(),
+ "dropout_add": get_jit_fused_dropout_add_func(),
+ },
+ policy=policy,
+ target_key=T5LayerSelfAttention,
+ )
+
+ return policy
+
+ def postprocess(self):
+ return self.model
diff --git a/opensora/datasets/__init__.py b/opensora/datasets/__init__.py
new file mode 100644
index 0000000..c9b3395
--- /dev/null
+++ b/opensora/datasets/__init__.py
@@ -0,0 +1,2 @@
+from .datasets import DatasetFromCSV, get_transforms_image, get_transforms_video
+from .utils import prepare_dataloader, save_sample
diff --git a/opensora/datasets/datasets.py b/opensora/datasets/datasets.py
new file mode 100644
index 0000000..c709685
--- /dev/null
+++ b/opensora/datasets/datasets.py
@@ -0,0 +1,114 @@
+import csv
+import os
+
+import numpy as np
+import torch
+import torchvision
+import torchvision.transforms as transforms
+from torchvision.datasets.folder import IMG_EXTENSIONS, pil_loader
+
+from . import video_transforms
+from .utils import center_crop_arr
+
+
+def get_transforms_video(resolution=256):
+ transform_video = transforms.Compose(
+ [
+ video_transforms.ToTensorVideo(), # TCHW
+ video_transforms.RandomHorizontalFlipVideo(),
+ video_transforms.UCFCenterCropVideo(resolution),
+ transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True),
+ ]
+ )
+ return transform_video
+
+
+def get_transforms_image(image_size=256):
+ transform = transforms.Compose(
+ [
+ transforms.Lambda(lambda pil_image: center_crop_arr(pil_image, image_size)),
+ transforms.RandomHorizontalFlip(),
+ transforms.ToTensor(),
+ transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True),
+ ]
+ )
+ return transform
+
+
+class DatasetFromCSV(torch.utils.data.Dataset):
+ """load video according to the csv file.
+
+ Args:
+ target_video_len (int): the number of video frames will be load.
+ align_transform (callable): Align different videos in a specified size.
+ temporal_sample (callable): Sample the target length of a video.
+ """
+
+ def __init__(
+ self,
+ csv_path,
+ num_frames=16,
+ frame_interval=1,
+ transform=None,
+ root=None,
+ ):
+ self.csv_path = csv_path
+ with open(csv_path, "r") as f:
+ reader = csv.reader(f)
+ self.samples = list(reader)
+
+ ext = self.samples[0][0].split(".")[-1]
+ if ext.lower() in ["mp4", "avi", "mov", "mkv"]:
+ self.is_video = True
+ else:
+ assert f".{ext.lower()}" in IMG_EXTENSIONS
+ self.is_video = False
+
+ self.transform = transform
+
+ self.num_frames = num_frames
+ self.frame_interval = frame_interval
+ self.temporal_sample = video_transforms.TemporalRandomCrop(num_frames * frame_interval)
+ self.root = root
+
+ def getitem(self, index):
+ sample = self.samples[index]
+ path = sample[0]
+ if self.root:
+ path = os.path.join(self.root, path)
+ text = sample[1]
+
+ if self.is_video:
+ vframes, aframes, info = torchvision.io.read_video(filename=path, pts_unit="sec", output_format="TCHW")
+ total_frames = len(vframes)
+
+ # Sampling video frames
+ start_frame_ind, end_frame_ind = self.temporal_sample(total_frames)
+ assert (
+ end_frame_ind - start_frame_ind >= self.num_frames
+ ), f"{path} with index {index} has not enough frames."
+ frame_indice = np.linspace(start_frame_ind, end_frame_ind - 1, self.num_frames, dtype=int)
+
+ video = vframes[frame_indice]
+ video = self.transform(video) # T C H W
+ else:
+ image = pil_loader(path)
+ image = self.transform(image)
+ video = image.unsqueeze(0).repeat(self.num_frames, 1, 1, 1)
+
+ # TCHW -> CTHW
+ video = video.permute(1, 0, 2, 3)
+
+ return {"video": video, "text": text}
+
+ def __getitem__(self, index):
+ for _ in range(10):
+ try:
+ return self.getitem(index)
+ except Exception as e:
+ print(e)
+ index = np.random.randint(len(self))
+ raise RuntimeError("Too many bad data.")
+
+ def __len__(self):
+ return len(self.samples)
diff --git a/opensora/datasets/utils.py b/opensora/datasets/utils.py
new file mode 100644
index 0000000..00f8fe3
--- /dev/null
+++ b/opensora/datasets/utils.py
@@ -0,0 +1,134 @@
+import random
+from typing import Iterator, Optional
+
+import numpy as np
+import torch
+from PIL import Image
+from torch.distributed import ProcessGroup
+from torch.distributed.distributed_c10d import _get_default_group
+from torch.utils.data import DataLoader, Dataset
+from torch.utils.data.distributed import DistributedSampler
+from torchvision.io import write_video
+from torchvision.utils import save_image
+
+
+def save_sample(x, fps=8, save_path=None, normalize=True, value_range=(-1, 1)):
+ """
+ Args:
+ x (Tensor): shape [C, T, H, W]
+ """
+ assert x.ndim == 4
+
+ if x.shape[1] == 1: # T = 1: save as image
+ save_path += ".png"
+ x = x.squeeze(1)
+ save_image([x], save_path, normalize=normalize, value_range=value_range)
+ else:
+ save_path += ".mp4"
+ if normalize:
+ low, high = value_range
+ x.clamp_(min=low, max=high)
+ x.sub_(low).div_(max(high - low, 1e-5))
+
+ x = x.mul(255).add_(0.5).clamp_(0, 255).permute(1, 2, 3, 0).to("cpu", torch.uint8)
+ write_video(save_path, x, fps=fps, video_codec="h264")
+ print(f"Saved to {save_path}")
+
+
+class StatefulDistributedSampler(DistributedSampler):
+ def __init__(
+ self,
+ dataset: Dataset,
+ num_replicas: Optional[int] = None,
+ rank: Optional[int] = None,
+ shuffle: bool = True,
+ seed: int = 0,
+ drop_last: bool = False,
+ ) -> None:
+ super().__init__(dataset, num_replicas, rank, shuffle, seed, drop_last)
+ self.start_index: int = 0
+
+ def __iter__(self) -> Iterator:
+ iterator = super().__iter__()
+ indices = list(iterator)
+ indices = indices[self.start_index :]
+ return iter(indices)
+
+ def __len__(self) -> int:
+ return self.num_samples - self.start_index
+
+ def set_start_index(self, start_index: int) -> None:
+ self.start_index = start_index
+
+
+def prepare_dataloader(
+ dataset,
+ batch_size,
+ shuffle=False,
+ seed=1024,
+ drop_last=False,
+ pin_memory=False,
+ num_workers=0,
+ process_group: Optional[ProcessGroup] = None,
+ **kwargs,
+):
+ r"""
+ Prepare a dataloader for distributed training. The dataloader will be wrapped by
+ `torch.utils.data.DataLoader` and `StatefulDistributedSampler`.
+
+
+ Args:
+ dataset (`torch.utils.data.Dataset`): The dataset to be loaded.
+ shuffle (bool, optional): Whether to shuffle the dataset. Defaults to False.
+ seed (int, optional): Random worker seed for sampling, defaults to 1024.
+ add_sampler: Whether to add ``DistributedDataParallelSampler`` to the dataset. Defaults to True.
+ drop_last (bool, optional): Set to True to drop the last incomplete batch, if the dataset size
+ is not divisible by the batch size. If False and the size of dataset is not divisible by
+ the batch size, then the last batch will be smaller, defaults to False.
+ pin_memory (bool, optional): Whether to pin memory address in CPU memory. Defaults to False.
+ num_workers (int, optional): Number of worker threads for this dataloader. Defaults to 0.
+ kwargs (dict): optional parameters for ``torch.utils.data.DataLoader``, more details could be found in
+ `DataLoader <https://pytorch.org/docs/stable/_modules/torch/utils/data/dataloader.html#DataLoader>`_.
+
+ Returns:
+ :class:`torch.utils.data.DataLoader`: A DataLoader used for training or testing.
+ """
+ _kwargs = kwargs.copy()
+ process_group = process_group or _get_default_group()
+ sampler = StatefulDistributedSampler(
+ dataset, num_replicas=process_group.size(), rank=process_group.rank(), shuffle=shuffle
+ )
+
+ # Deterministic dataloader
+ def seed_worker(worker_id):
+ worker_seed = seed
+ np.random.seed(worker_seed)
+ torch.manual_seed(worker_seed)
+ random.seed(worker_seed)
+
+ return DataLoader(
+ dataset,
+ batch_size=batch_size,
+ sampler=sampler,
+ worker_init_fn=seed_worker,
+ drop_last=drop_last,
+ pin_memory=pin_memory,
+ num_workers=num_workers,
+ **_kwargs,
+ )
+
+def center_crop_arr(pil_image, image_size):
+ """
+ Center cropping implementation from ADM.
+ https://github.com/openai/guided-diffusion/blob/8fb3ad9197f16bbc40620447b2742e13458d2831/guided_diffusion/image_datasets.py#L126
+ """
+ while min(*pil_image.size) >= 2 * image_size:
+ pil_image = pil_image.resize(tuple(x // 2 for x in pil_image.size), resample=Image.BOX)
+
+ scale = image_size / min(*pil_image.size)
+ pil_image = pil_image.resize(tuple(round(x * scale) for x in pil_image.size), resample=Image.BICUBIC)
+
+ arr = np.array(pil_image)
+ crop_y = (arr.shape[0] - image_size) // 2
+ crop_x = (arr.shape[1] - image_size) // 2
+ return Image.fromarray(arr[crop_y : crop_y + image_size, crop_x : crop_x + image_size])
diff --git a/opensora/datasets/video_transforms.py b/opensora/datasets/video_transforms.py
new file mode 100644
index 0000000..21ee925
--- /dev/null
+++ b/opensora/datasets/video_transforms.py
@@ -0,0 +1,486 @@
+# Modified from Latte
+
+import numbers
+import random
+
+import torch
+
+
+def _is_tensor_video_clip(clip):
+ if not torch.is_tensor(clip):
+ raise TypeError("clip should be Tensor. Got %s" % type(clip))
+
+ if not clip.ndimension() == 4:
+ raise ValueError("clip should be 4D. Got %dD" % clip.dim())
+
+ return True
+
+
+def center_crop_arr(pil_image, image_size):
+ """
+ Center cropping implementation from ADM.
+ https://github.com/openai/guided-diffusion/blob/8fb3ad9197f16bbc40620447b2742e13458d2831/guided_diffusion/image_datasets.py#L126
+ """
+ while min(*pil_image.size) >= 2 * image_size:
+ pil_image = pil_image.resize(tuple(x // 2 for x in pil_image.size), resample=Image.BOX)
+
+ scale = image_size / min(*pil_image.size)
+ pil_image = pil_image.resize(tuple(round(x * scale) for x in pil_image.size), resample=Image.BICUBIC)
+
+ arr = np.array(pil_image)
+ crop_y = (arr.shape[0] - image_size) // 2
+ crop_x = (arr.shape[1] - image_size) // 2
+ return Image.fromarray(arr[crop_y : crop_y + image_size, crop_x : crop_x + image_size])
+
+
+def crop(clip, i, j, h, w):
+ """
+ Args:
+ clip (torch.tensor): Video clip to be cropped. Size is (T, C, H, W)
+ """
+ if len(clip.size()) != 4:
+ raise ValueError("clip should be a 4D tensor")
+ return clip[..., i : i + h, j : j + w]
+
+
+def resize(clip, target_size, interpolation_mode):
+ if len(target_size) != 2:
+ raise ValueError(f"target size should be tuple (height, width), instead got {target_size}")
+ return torch.nn.functional.interpolate(clip, size=target_size, mode=interpolation_mode, align_corners=False)
+
+
+def resize_scale(clip, target_size, interpolation_mode):
+ if len(target_size) != 2:
+ raise ValueError(f"target size should be tuple (height, width), instead got {target_size}")
+ H, W = clip.size(-2), clip.size(-1)
+ scale_ = target_size[0] / min(H, W)
+ return torch.nn.functional.interpolate(clip, scale_factor=scale_, mode=interpolation_mode, align_corners=False)
+
+
+def resized_crop(clip, i, j, h, w, size, interpolation_mode="bilinear"):
+ """
+ Do spatial cropping and resizing to the video clip
+ Args:
+ clip (torch.tensor): Video clip to be cropped. Size is (T, C, H, W)
+ i (int): i in (i,j) i.e coordinates of the upper left corner.
+ j (int): j in (i,j) i.e coordinates of the upper left corner.
+ h (int): Height of the cropped region.
+ w (int): Width of the cropped region.
+ size (tuple(int, int)): height and width of resized clip
+ Returns:
+ clip (torch.tensor): Resized and cropped clip. Size is (T, C, H, W)
+ """
+ if not _is_tensor_video_clip(clip):
+ raise ValueError("clip should be a 4D torch.tensor")
+ clip = crop(clip, i, j, h, w)
+ clip = resize(clip, size, interpolation_mode)
+ return clip
+
+
+def center_crop(clip, crop_size):
+ if not _is_tensor_video_clip(clip):
+ raise ValueError("clip should be a 4D torch.tensor")
+ h, w = clip.size(-2), clip.size(-1)
+ th, tw = crop_size
+ if h < th or w < tw:
+ raise ValueError("height and width must be no smaller than crop_size")
+
+ i = int(round((h - th) / 2.0))
+ j = int(round((w - tw) / 2.0))
+ return crop(clip, i, j, th, tw)
+
+
+def center_crop_using_short_edge(clip):
+ if not _is_tensor_video_clip(clip):
+ raise ValueError("clip should be a 4D torch.tensor")
+ h, w = clip.size(-2), clip.size(-1)
+ if h < w:
+ th, tw = h, h
+ i = 0
+ j = int(round((w - tw) / 2.0))
+ else:
+ th, tw = w, w
+ i = int(round((h - th) / 2.0))
+ j = 0
+ return crop(clip, i, j, th, tw)
+
+
+def random_shift_crop(clip):
+ """
+ Slide along the long edge, with the short edge as crop size
+ """
+ if not _is_tensor_video_clip(clip):
+ raise ValueError("clip should be a 4D torch.tensor")
+ h, w = clip.size(-2), clip.size(-1)
+
+ if h <= w:
+ short_edge = h
+ else:
+ short_edge = w
+
+ th, tw = short_edge, short_edge
+
+ i = torch.randint(0, h - th + 1, size=(1,)).item()
+ j = torch.randint(0, w - tw + 1, size=(1,)).item()
+ return crop(clip, i, j, th, tw)
+
+
+def to_tensor(clip):
+ """
+ Convert tensor data type from uint8 to float, divide value by 255.0 and
+ permute the dimensions of clip tensor
+ Args:
+ clip (torch.tensor, dtype=torch.uint8): Size is (T, C, H, W)
+ Return:
+ clip (torch.tensor, dtype=torch.float): Size is (T, C, H, W)
+ """
+ _is_tensor_video_clip(clip)
+ if not clip.dtype == torch.uint8:
+ raise TypeError("clip tensor should have data type uint8. Got %s" % str(clip.dtype))
+ # return clip.float().permute(3, 0, 1, 2) / 255.0
+ return clip.float() / 255.0
+
+
+def normalize(clip, mean, std, inplace=False):
+ """
+ Args:
+ clip (torch.tensor): Video clip to be normalized. Size is (T, C, H, W)
+ mean (tuple): pixel RGB mean. Size is (3)
+ std (tuple): pixel standard deviation. Size is (3)
+ Returns:
+ normalized clip (torch.tensor): Size is (T, C, H, W)
+ """
+ if not _is_tensor_video_clip(clip):
+ raise ValueError("clip should be a 4D torch.tensor")
+ if not inplace:
+ clip = clip.clone()
+ mean = torch.as_tensor(mean, dtype=clip.dtype, device=clip.device)
+ # print(mean)
+ std = torch.as_tensor(std, dtype=clip.dtype, device=clip.device)
+ clip.sub_(mean[:, None, None, None]).div_(std[:, None, None, None])
+ return clip
+
+
+def hflip(clip):
+ """
+ Args:
+ clip (torch.tensor): Video clip to be normalized. Size is (T, C, H, W)
+ Returns:
+ flipped clip (torch.tensor): Size is (T, C, H, W)
+ """
+ if not _is_tensor_video_clip(clip):
+ raise ValueError("clip should be a 4D torch.tensor")
+ return clip.flip(-1)
+
+
+class RandomCropVideo:
+ def __init__(self, size):
+ if isinstance(size, numbers.Number):
+ self.size = (int(size), int(size))
+ else:
+ self.size = size
+
+ def __call__(self, clip):
+ """
+ Args:
+ clip (torch.tensor): Video clip to be cropped. Size is (T, C, H, W)
+ Returns:
+ torch.tensor: randomly cropped video clip.
+ size is (T, C, OH, OW)
+ """
+ i, j, h, w = self.get_params(clip)
+ return crop(clip, i, j, h, w)
+
+ def get_params(self, clip):
+ h, w = clip.shape[-2:]
+ th, tw = self.size
+
+ if h < th or w < tw:
+ raise ValueError(f"Required crop size {(th, tw)} is larger than input image size {(h, w)}")
+
+ if w == tw and h == th:
+ return 0, 0, h, w
+
+ i = torch.randint(0, h - th + 1, size=(1,)).item()
+ j = torch.randint(0, w - tw + 1, size=(1,)).item()
+
+ return i, j, th, tw
+
+ def __repr__(self) -> str:
+ return f"{self.__class__.__name__}(size={self.size})"
+
+
+class CenterCropResizeVideo:
+ """
+ First use the short side for cropping length,
+ center crop video, then resize to the specified size
+ """
+
+ def __init__(
+ self,
+ size,
+ interpolation_mode="bilinear",
+ ):
+ if isinstance(size, tuple):
+ if len(size) != 2:
+ raise ValueError(f"size should be tuple (height, width), instead got {size}")
+ self.size = size
+ else:
+ self.size = (size, size)
+
+ self.interpolation_mode = interpolation_mode
+
+ def __call__(self, clip):
+ """
+ Args:
+ clip (torch.tensor): Video clip to be cropped. Size is (T, C, H, W)
+ Returns:
+ torch.tensor: scale resized / center cropped video clip.
+ size is (T, C, crop_size, crop_size)
+ """
+ clip_center_crop = center_crop_using_short_edge(clip)
+ clip_center_crop_resize = resize(
+ clip_center_crop, target_size=self.size, interpolation_mode=self.interpolation_mode
+ )
+ return clip_center_crop_resize
+
+ def __repr__(self) -> str:
+ return f"{self.__class__.__name__}(size={self.size}, interpolation_mode={self.interpolation_mode}"
+
+
+class UCFCenterCropVideo:
+ """
+ First scale to the specified size in equal proportion to the short edge,
+ then center cropping
+ """
+
+ def __init__(
+ self,
+ size,
+ interpolation_mode="bilinear",
+ ):
+ if isinstance(size, tuple):
+ if len(size) != 2:
+ raise ValueError(f"size should be tuple (height, width), instead got {size}")
+ self.size = size
+ else:
+ self.size = (size, size)
+
+ self.interpolation_mode = interpolation_mode
+
+ def __call__(self, clip):
+ """
+ Args:
+ clip (torch.tensor): Video clip to be cropped. Size is (T, C, H, W)
+ Returns:
+ torch.tensor: scale resized / center cropped video clip.
+ size is (T, C, crop_size, crop_size)
+ """
+ clip_resize = resize_scale(clip=clip, target_size=self.size, interpolation_mode=self.interpolation_mode)
+ clip_center_crop = center_crop(clip_resize, self.size)
+ return clip_center_crop
+
+ def __repr__(self) -> str:
+ return f"{self.__class__.__name__}(size={self.size}, interpolation_mode={self.interpolation_mode}"
+
+
+class KineticsRandomCropResizeVideo:
+ """
+ Slide along the long edge, with the short edge as crop size. And resie to the desired size.
+ """
+
+ def __init__(
+ self,
+ size,
+ interpolation_mode="bilinear",
+ ):
+ if isinstance(size, tuple):
+ if len(size) != 2:
+ raise ValueError(f"size should be tuple (height, width), instead got {size}")
+ self.size = size
+ else:
+ self.size = (size, size)
+
+ self.interpolation_mode = interpolation_mode
+
+ def __call__(self, clip):
+ clip_random_crop = random_shift_crop(clip)
+ clip_resize = resize(clip_random_crop, self.size, self.interpolation_mode)
+ return clip_resize
+
+
+class CenterCropVideo:
+ def __init__(
+ self,
+ size,
+ interpolation_mode="bilinear",
+ ):
+ if isinstance(size, tuple):
+ if len(size) != 2:
+ raise ValueError(f"size should be tuple (height, width), instead got {size}")
+ self.size = size
+ else:
+ self.size = (size, size)
+
+ self.interpolation_mode = interpolation_mode
+
+ def __call__(self, clip):
+ """
+ Args:
+ clip (torch.tensor): Video clip to be cropped. Size is (T, C, H, W)
+ Returns:
+ torch.tensor: center cropped video clip.
+ size is (T, C, crop_size, crop_size)
+ """
+ clip_center_crop = center_crop(clip, self.size)
+ return clip_center_crop
+
+ def __repr__(self) -> str:
+ return f"{self.__class__.__name__}(size={self.size}, interpolation_mode={self.interpolation_mode}"
+
+
+class NormalizeVideo:
+ """
+ Normalize the video clip by mean subtraction and division by standard deviation
+ Args:
+ mean (3-tuple): pixel RGB mean
+ std (3-tuple): pixel RGB standard deviation
+ inplace (boolean): whether do in-place normalization
+ """
+
+ def __init__(self, mean, std, inplace=False):
+ self.mean = mean
+ self.std = std
+ self.inplace = inplace
+
+ def __call__(self, clip):
+ """
+ Args:
+ clip (torch.tensor): video clip must be normalized. Size is (C, T, H, W)
+ """
+ return normalize(clip, self.mean, self.std, self.inplace)
+
+ def __repr__(self) -> str:
+ return f"{self.__class__.__name__}(mean={self.mean}, std={self.std}, inplace={self.inplace})"
+
+
+class ToTensorVideo:
+ """
+ Convert tensor data type from uint8 to float, divide value by 255.0 and
+ permute the dimensions of clip tensor
+ """
+
+ def __init__(self):
+ pass
+
+ def __call__(self, clip):
+ """
+ Args:
+ clip (torch.tensor, dtype=torch.uint8): Size is (T, C, H, W)
+ Return:
+ clip (torch.tensor, dtype=torch.float): Size is (T, C, H, W)
+ """
+ return to_tensor(clip)
+
+ def __repr__(self) -> str:
+ return self.__class__.__name__
+
+
+class RandomHorizontalFlipVideo:
+ """
+ Flip the video clip along the horizontal direction with a given probability
+ Args:
+ p (float): probability of the clip being flipped. Default value is 0.5
+ """
+
+ def __init__(self, p=0.5):
+ self.p = p
+
+ def __call__(self, clip):
+ """
+ Args:
+ clip (torch.tensor): Size is (T, C, H, W)
+ Return:
+ clip (torch.tensor): Size is (T, C, H, W)
+ """
+ if random.random() < self.p:
+ clip = hflip(clip)
+ return clip
+
+ def __repr__(self) -> str:
+ return f"{self.__class__.__name__}(p={self.p})"
+
+
+# ------------------------------------------------------------
+# --------------------- Sampling ---------------------------
+# ------------------------------------------------------------
+class TemporalRandomCrop(object):
+ """Temporally crop the given frame indices at a random location.
+
+ Args:
+ size (int): Desired length of frames will be seen in the model.
+ """
+
+ def __init__(self, size):
+ self.size = size
+
+ def __call__(self, total_frames):
+ rand_end = max(0, total_frames - self.size - 1)
+ begin_index = random.randint(0, rand_end)
+ end_index = min(begin_index + self.size, total_frames)
+ return begin_index, end_index
+
+
+if __name__ == "__main__":
+ import os
+
+ import numpy as np
+ import torchvision.io as io
+ from torchvision import transforms
+ from torchvision.utils import save_image
+
+ vframes, aframes, info = io.read_video(filename="./v_Archery_g01_c03.avi", pts_unit="sec", output_format="TCHW")
+
+ trans = transforms.Compose(
+ [
+ ToTensorVideo(),
+ RandomHorizontalFlipVideo(),
+ UCFCenterCropVideo(512),
+ # NormalizeVideo(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True),
+ transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True),
+ ]
+ )
+
+ target_video_len = 32
+ frame_interval = 1
+ total_frames = len(vframes)
+ print(total_frames)
+
+ temporal_sample = TemporalRandomCrop(target_video_len * frame_interval)
+
+ # Sampling video frames
+ start_frame_ind, end_frame_ind = temporal_sample(total_frames)
+ # print(start_frame_ind)
+ # print(end_frame_ind)
+ assert end_frame_ind - start_frame_ind >= target_video_len
+ frame_indice = np.linspace(start_frame_ind, end_frame_ind - 1, target_video_len, dtype=int)
+ print(frame_indice)
+
+ select_vframes = vframes[frame_indice]
+ print(select_vframes.shape)
+ print(select_vframes.dtype)
+
+ select_vframes_trans = trans(select_vframes)
+ print(select_vframes_trans.shape)
+ print(select_vframes_trans.dtype)
+
+ select_vframes_trans_int = ((select_vframes_trans * 0.5 + 0.5) * 255).to(dtype=torch.uint8)
+ print(select_vframes_trans_int.dtype)
+ print(select_vframes_trans_int.permute(0, 2, 3, 1).shape)
+
+ io.write_video("./test.avi", select_vframes_trans_int.permute(0, 2, 3, 1), fps=8)
+
+ for i in range(target_video_len):
+ save_image(
+ select_vframes_trans[i], os.path.join("./test000", "%04d.png" % i), normalize=True, value_range=(-1, 1)
+ )
diff --git a/opensora/models/__init__.py b/opensora/models/__init__.py
new file mode 100644
index 0000000..2aa793a
--- /dev/null
+++ b/opensora/models/__init__.py
@@ -0,0 +1,6 @@
+from .dit import *
+from .latte import *
+from .stdit import *
+from .pixart import *
+from .text_encoder import *
+from .vae import *
diff --git a/opensora/models/dit/__init__.py b/opensora/models/dit/__init__.py
new file mode 100644
index 0000000..2019831
--- /dev/null
+++ b/opensora/models/dit/__init__.py
@@ -0,0 +1 @@
+from .dit import DiT, DiT_XL_2, DiT_XL_2x2
diff --git a/opensora/models/dit/dit.py b/opensora/models/dit/dit.py
new file mode 100644
index 0000000..106f660
--- /dev/null
+++ b/opensora/models/dit/dit.py
@@ -0,0 +1,284 @@
+# Modified from Meta DiT
+
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+# --------------------------------------------------------
+# References:
+# DiT: https://github.com/facebookresearch/DiT/tree/main
+# GLIDE: https://github.com/openai/glide-text2im
+# MAE: https://github.com/facebookresearch/mae/blob/main/models_mae.py
+# --------------------------------------------------------
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.utils.checkpoint
+from einops import rearrange
+from timm.models.vision_transformer import Mlp
+
+from opensora.models.layers.blocks import (
+ Attention,
+ CaptionEmbedder,
+ FinalLayer,
+ LabelEmbedder,
+ PatchEmbed3D,
+ TimestepEmbedder,
+ approx_gelu,
+ get_1d_sincos_pos_embed,
+ get_2d_sincos_pos_embed,
+ get_layernorm,
+ modulate,
+)
+from opensora.registry import MODELS
+from opensora.utils.ckpt_utils import load_checkpoint
+from opensora.acceleration.checkpoint import auto_grad_checkpoint
+
+
+class DiTBlock(nn.Module):
+ """
+ A DiT block with adaptive layer norm zero (adaLN-Zero) conditioning.
+ """
+
+ def __init__(
+ self,
+ hidden_size,
+ num_heads,
+ mlp_ratio=4.0,
+ enable_flashattn=False,
+ enable_layernorm_kernel=False,
+ ):
+ super().__init__()
+ self.hidden_size = hidden_size
+ self.num_heads = num_heads
+ self.enable_flashattn = enable_flashattn
+ mlp_hidden_dim = int(hidden_size * mlp_ratio)
+
+ self.norm1 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)
+ self.attn = Attention(
+ hidden_size,
+ num_heads=num_heads,
+ qkv_bias=True,
+ enable_flashattn=enable_flashattn,
+ )
+ self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)
+ self.mlp = Mlp(in_features=hidden_size, hidden_features=mlp_hidden_dim, act_layer=approx_gelu, drop=0)
+ self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True))
+
+ def forward(self, x, c):
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN_modulation(c).chunk(6, dim=1)
+ x = x + gate_msa.unsqueeze(1) * self.attn(modulate(self.norm1, x, shift_msa, scale_msa))
+ x = x + gate_mlp.unsqueeze(1) * self.mlp(modulate(self.norm2, x, shift_mlp, scale_mlp))
+ return x
+
+
+class DiT(nn.Module):
+ """
+ Diffusion model with a Transformer backbone.
+ """
+
+ def __init__(
+ self,
+ input_size=(16, 32, 32),
+ in_channels=4,
+ patch_size=(1, 2, 2),
+ hidden_size=1152,
+ depth=28,
+ num_heads=16,
+ mlp_ratio=4.0,
+ class_dropout_prob=0.1,
+ learn_sigma=True,
+ condition="text",
+ no_temporal_pos_emb=False,
+ caption_channels=512,
+ model_max_length=77,
+ dtype=torch.float32,
+ enable_flashattn=False,
+ enable_layernorm_kernel=False,
+ ):
+ super().__init__()
+ self.learn_sigma = learn_sigma
+ self.in_channels = in_channels
+ self.out_channels = in_channels * 2 if learn_sigma else in_channels
+ self.hidden_size = hidden_size
+ self.patch_size = patch_size
+ self.input_size = input_size
+ num_patches = np.prod([input_size[i] // patch_size[i] for i in range(3)])
+ self.num_patches = num_patches
+ self.num_temporal = input_size[0] // patch_size[0]
+ self.num_spatial = num_patches // self.num_temporal
+ self.num_heads = num_heads
+ self.dtype = dtype
+ self.use_text_encoder = not condition.startswith("label")
+ if enable_flashattn:
+ assert dtype in [
+ torch.float16,
+ torch.bfloat16,
+ ], f"Flash attention only supports float16 and bfloat16, but got {self.dtype}"
+ self.no_temporal_pos_emb = no_temporal_pos_emb
+ self.mlp_ratio = mlp_ratio
+ self.depth = depth
+
+ self.register_buffer("pos_embed_spatial", self.get_spatial_pos_embed())
+ self.register_buffer("pos_embed_temporal", self.get_temporal_pos_embed())
+
+ self.x_embedder = PatchEmbed3D(patch_size, in_channels, embed_dim=hidden_size)
+ if not self.use_text_encoder:
+ num_classes = int(condition.split("_")[-1])
+ self.y_embedder = LabelEmbedder(num_classes, hidden_size, class_dropout_prob)
+ else:
+ self.y_embedder = CaptionEmbedder(
+ in_channels=caption_channels,
+ hidden_size=hidden_size,
+ uncond_prob=class_dropout_prob,
+ act_layer=approx_gelu,
+ token_num=1, # pooled token
+ )
+ self.t_embedder = TimestepEmbedder(hidden_size)
+ self.blocks = nn.ModuleList(
+ [
+ DiTBlock(
+ hidden_size,
+ num_heads,
+ mlp_ratio=mlp_ratio,
+ enable_flashattn=enable_flashattn,
+ enable_layernorm_kernel=enable_layernorm_kernel,
+ )
+ for _ in range(depth)
+ ]
+ )
+ self.final_layer = FinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)
+
+ self.initialize_weights()
+ self.enable_flashattn = enable_flashattn
+ self.enable_layernorm_kernel = enable_layernorm_kernel
+
+ def get_spatial_pos_embed(self):
+ pos_embed = get_2d_sincos_pos_embed(
+ self.hidden_size,
+ self.input_size[1] // self.patch_size[1],
+ )
+ pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)
+ return pos_embed
+
+ def get_temporal_pos_embed(self):
+ pos_embed = get_1d_sincos_pos_embed(
+ self.hidden_size,
+ self.input_size[0] // self.patch_size[0],
+ )
+ pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)
+ return pos_embed
+
+ def unpatchify(self, x):
+ c = self.out_channels
+ t, h, w = [self.input_size[i] // self.patch_size[i] for i in range(3)]
+ pt, ph, pw = self.patch_size
+
+ x = x.reshape(shape=(x.shape[0], t, h, w, pt, ph, pw, c))
+ x = rearrange(x, "n t h w r p q c -> n c t r h p w q")
+ imgs = x.reshape(shape=(x.shape[0], c, t * pt, h * ph, w * pw))
+ return imgs
+
+ def forward(self, x, t, y):
+ """
+ Forward pass of DiT.
+ x: (B, C, T, H, W) tensor of inputs
+ t: (B,) tensor of diffusion timesteps
+ y: list of text
+ """
+ # origin inputs should be float32, cast to specified dtype
+ x = x.to(self.dtype)
+
+ # embedding
+ x = self.x_embedder(x) # (B, N, D)
+ x = rearrange(x, "b (t s) d -> b t s d", t=self.num_temporal, s=self.num_spatial)
+ x = x + self.pos_embed_spatial
+ if not self.no_temporal_pos_emb:
+ x = rearrange(x, "b t s d -> b s t d")
+ x = x + self.pos_embed_temporal
+ x = rearrange(x, "b s t d -> b (t s) d")
+ else:
+ x = rearrange(x, "b t s d -> b (t s) d")
+
+ t = self.t_embedder(t, dtype=x.dtype) # (N, D)
+ y = self.y_embedder(y, self.training) # (N, D)
+ if self.use_text_encoder:
+ y = y.squeeze(1).squeeze(1)
+ condition = t + y
+
+ # blocks
+ for _, block in enumerate(self.blocks):
+ c = condition
+ x = auto_grad_checkpoint(block, x, c) # (B, N, D)
+
+ # final process
+ x = self.final_layer(x, condition) # (B, N, num_patches * out_channels)
+ x = self.unpatchify(x) # (B, out_channels, T, H, W)
+
+ # cast to float32 for better accuracy
+ x = x.to(torch.float32)
+ return x
+
+ def initialize_weights(self):
+ # Initialize transformer layers:
+ def _basic_init(module):
+ if isinstance(module, nn.Linear):
+ if module.weight.requires_grad_:
+ torch.nn.init.xavier_uniform_(module.weight)
+ if module.bias is not None:
+ nn.init.constant_(module.bias, 0)
+
+ self.apply(_basic_init)
+
+ # Initialize patch_embed like nn.Linear (instead of nn.Conv2d):
+ w = self.x_embedder.proj.weight.data
+ nn.init.xavier_uniform_(w.view([w.shape[0], -1]))
+ nn.init.constant_(self.x_embedder.proj.bias, 0)
+
+ # Initialize timestep embedding MLP:
+ nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02)
+ nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02)
+
+ # Zero-out adaLN modulation layers in DiT blocks:
+ for block in self.blocks:
+ nn.init.constant_(block.adaLN_modulation[-1].weight, 0)
+ nn.init.constant_(block.adaLN_modulation[-1].bias, 0)
+
+ # Zero-out output layers:
+ nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0)
+ nn.init.constant_(self.final_layer.adaLN_modulation[-1].bias, 0)
+ nn.init.constant_(self.final_layer.linear.weight, 0)
+ nn.init.constant_(self.final_layer.linear.bias, 0)
+
+ # Zero-out text embedding layers:
+ if self.use_text_encoder:
+ nn.init.normal_(self.y_embedder.y_proj.fc1.weight, std=0.02)
+ nn.init.normal_(self.y_embedder.y_proj.fc2.weight, std=0.02)
+
+
[email protected]_module("DiT-XL/2")
+def DiT_XL_2(from_pretrained=None, **kwargs):
+ model = DiT(
+ depth=28,
+ hidden_size=1152,
+ patch_size=(1, 2, 2),
+ num_heads=16,
+ **kwargs,
+ )
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
+
+
[email protected]_module("DiT-XL/2x2")
+def DiT_XL_2x2(from_pretrained=None, **kwargs):
+ model = DiT(
+ depth=28,
+ hidden_size=1152,
+ patch_size=(2, 2, 2),
+ num_heads=16,
+ **kwargs,
+ )
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
diff --git a/opensora/models/latte/__init__.py b/opensora/models/latte/__init__.py
new file mode 100644
index 0000000..3115f47
--- /dev/null
+++ b/opensora/models/latte/__init__.py
@@ -0,0 +1 @@
+from .latte import Latte, Latte_XL_2, Latte_XL_2x2
diff --git a/opensora/models/latte/latte.py b/opensora/models/latte/latte.py
new file mode 100644
index 0000000..11259de
--- /dev/null
+++ b/opensora/models/latte/latte.py
@@ -0,0 +1,100 @@
+# Adapted from Latte
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+# --------------------------------------------------------
+# References:
+# Latte: https://github.com/Vchitect/Latte
+# DiT: https://github.com/facebookresearch/DiT/tree/main
+# --------------------------------------------------------
+
+import torch
+from einops import rearrange, repeat
+
+from opensora.models.dit import DiT
+from opensora.registry import MODELS
+from opensora.utils.ckpt_utils import load_checkpoint
+from opensora.acceleration.checkpoint import auto_grad_checkpoint
+
+
+class Latte(DiT):
+ def forward(self, x, t, y):
+ """
+ Forward pass of DiT.
+ x: (B, C, T, H, W) tensor of inputs
+ t: (B,) tensor of diffusion timesteps
+ y: list of text
+ """
+ # origin inputs should be float32, cast to specified dtype
+ x = x.to(self.dtype)
+
+ # embedding
+ x = self.x_embedder(x) # (B, N, D)
+ x = rearrange(x, "b (t s) d -> b t s d", t=self.num_temporal, s=self.num_spatial)
+ x = x + self.pos_embed_spatial
+ x = rearrange(x, "b t s d -> b (t s) d")
+
+ t = self.t_embedder(t, dtype=x.dtype) # (N, D)
+ y = self.y_embedder(y, self.training) # (N, D)
+ if self.use_text_encoder:
+ y = y.squeeze(1).squeeze(1)
+ condition = t + y
+ condition_spatial = repeat(condition, "b d -> (b t) d", t=self.num_temporal)
+ condition_temporal = repeat(condition, "b d -> (b s) d", s=self.num_spatial)
+
+ # blocks
+ for i, block in enumerate(self.blocks):
+ if i % 2 == 0:
+ # spatial
+ x = rearrange(x, "b (t s) d -> (b t) s d", t=self.num_temporal, s=self.num_spatial)
+ c = condition_spatial
+ else:
+ # temporal
+ x = rearrange(x, "b (t s) d -> (b s) t d", t=self.num_temporal, s=self.num_spatial)
+ c = condition_temporal
+ if i == 1:
+ x = x + self.pos_embed_temporal
+
+ x = auto_grad_checkpoint(block, x, c) # (B, N, D)
+
+ if i % 2 == 0:
+ x = rearrange(x, "(b t) s d -> b (t s) d", t=self.num_temporal, s=self.num_spatial)
+ else:
+ x = rearrange(x, "(b s) t d -> b (t s) d", t=self.num_temporal, s=self.num_spatial)
+
+ # final process
+ x = self.final_layer(x, condition) # (B, N, num_patches * out_channels)
+ x = self.unpatchify(x) # (B, out_channels, T, H, W)
+
+ # cast to float32 for better accuracy
+ x = x.to(torch.float32)
+ return x
+
+
[email protected]_module("Latte-XL/2")
+def Latte_XL_2(from_pretrained=None, **kwargs):
+ model = Latte(
+ depth=28,
+ hidden_size=1152,
+ patch_size=(1, 2, 2),
+ num_heads=16,
+ **kwargs,
+ )
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
+
+
[email protected]_module("Latte-XL/2x2")
+def Latte_XL_2x2(from_pretrained=None, **kwargs):
+ model = Latte(
+ depth=28,
+ hidden_size=1152,
+ patch_size=(2, 2, 2),
+ num_heads=16,
+ **kwargs,
+ )
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
diff --git a/opensora/models/layers/__init__.py b/opensora/models/layers/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/opensora/models/layers/blocks.py b/opensora/models/layers/blocks.py
new file mode 100644
index 0000000..f06d696
--- /dev/null
+++ b/opensora/models/layers/blocks.py
@@ -0,0 +1,591 @@
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+# --------------------------------------------------------
+# References:
+# PixArt: https://github.com/PixArt-alpha/PixArt-alpha
+# Latte: https://github.com/Vchitect/Latte
+# DiT: https://github.com/facebookresearch/DiT/tree/main
+# GLIDE: https://github.com/openai/glide-text2im
+# MAE: https://github.com/facebookresearch/mae/blob/main/models_mae.py
+# --------------------------------------------------------
+
+import math
+
+import numpy as np
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.utils.checkpoint
+import xformers.ops
+from einops import rearrange
+from flash_attn import flash_attn_varlen_func
+from timm.models.vision_transformer import Mlp
+
+from opensora.acceleration.communications import all_to_all, split_forward_gather_backward
+from opensora.acceleration.parallel_states import get_sequence_parallel_group
+
+approx_gelu = lambda: nn.GELU(approximate="tanh")
+
+
+def get_layernorm(hidden_size: torch.Tensor, eps: float, affine: bool, use_kernel: bool):
+ if use_kernel:
+ try:
+ from apex.normalization import FusedLayerNorm
+
+ return FusedLayerNorm(hidden_size, elementwise_affine=affine, eps=eps)
+ except ImportError:
+ raise RuntimeError("FusedLayerNorm not available. Please install apex.")
+ else:
+ return nn.LayerNorm(hidden_size, eps, elementwise_affine=affine)
+
+
+def modulate(norm_func, x, shift, scale):
+ # Suppose x is (B, N, D), shift is (B, D), scale is (B, D)
+ dtype = x.dtype
+ x = norm_func(x.to(torch.float32)).to(dtype)
+ x = x * (scale.unsqueeze(1) + 1) + shift.unsqueeze(1)
+ x = x.to(dtype)
+ return x
+
+
+def t2i_modulate(x, shift, scale):
+ return x * (1 + scale) + shift
+
+
+# ===============================================
+# General-purpose Layers
+# ===============================================
+
+
+class PatchEmbed3D(nn.Module):
+ """Video to Patch Embedding.
+
+ Args:
+ patch_size (int): Patch token size. Default: (2,4,4).
+ in_chans (int): Number of input video channels. Default: 3.
+ embed_dim (int): Number of linear projection output channels. Default: 96.
+ norm_layer (nn.Module, optional): Normalization layer. Default: None
+ """
+
+ def __init__(
+ self,
+ patch_size=(2, 4, 4),
+ in_chans=3,
+ embed_dim=96,
+ norm_layer=None,
+ flatten=True,
+ ):
+ super().__init__()
+ self.patch_size = patch_size
+ self.flatten = flatten
+
+ self.in_chans = in_chans
+ self.embed_dim = embed_dim
+
+ self.proj = nn.Conv3d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+ if norm_layer is not None:
+ self.norm = norm_layer(embed_dim)
+ else:
+ self.norm = None
+
+ def forward(self, x):
+ """Forward function."""
+ # padding
+ _, _, D, H, W = x.size()
+ if W % self.patch_size[2] != 0:
+ x = F.pad(x, (0, self.patch_size[2] - W % self.patch_size[2]))
+ if H % self.patch_size[1] != 0:
+ x = F.pad(x, (0, 0, 0, self.patch_size[1] - H % self.patch_size[1]))
+ if D % self.patch_size[0] != 0:
+ x = F.pad(x, (0, 0, 0, 0, 0, self.patch_size[0] - D % self.patch_size[0]))
+
+ x = self.proj(x) # (B C T H W)
+ if self.norm is not None:
+ D, Wh, Ww = x.size(2), x.size(3), x.size(4)
+ x = x.flatten(2).transpose(1, 2)
+ x = self.norm(x)
+ x = x.transpose(1, 2).view(-1, self.embed_dim, D, Wh, Ww)
+ if self.flatten:
+ x = x.flatten(2).transpose(1, 2) # BCTHW -> BNC
+ return x
+
+
+class Attention(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ num_heads: int = 8,
+ qkv_bias: bool = False,
+ qk_norm: bool = False,
+ attn_drop: float = 0.0,
+ proj_drop: float = 0.0,
+ norm_layer: nn.Module = nn.LayerNorm,
+ enable_flashattn: bool = False,
+ ) -> None:
+ super().__init__()
+ assert dim % num_heads == 0, "dim should be divisible by num_heads"
+ self.dim = dim
+ self.num_heads = num_heads
+ self.head_dim = dim // num_heads
+ self.scale = self.head_dim**-0.5
+ self.enable_flashattn = enable_flashattn
+
+ self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+ self.q_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
+ self.k_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
+ self.attn_drop = nn.Dropout(attn_drop)
+ self.proj = nn.Linear(dim, dim)
+ self.proj_drop = nn.Dropout(proj_drop)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ B, N, C = x.shape
+ qkv = self.qkv(x)
+ qkv_shape = (B, N, 3, self.num_heads, self.head_dim)
+ if self.enable_flashattn:
+ qkv_permute_shape = (2, 0, 1, 3, 4)
+ else:
+ qkv_permute_shape = (2, 0, 3, 1, 4)
+ qkv = qkv.view(qkv_shape).permute(qkv_permute_shape)
+ q, k, v = qkv.unbind(0)
+ q, k = self.q_norm(q), self.k_norm(k)
+ if self.enable_flashattn:
+ from flash_attn import flash_attn_func
+
+ x = flash_attn_func(
+ q,
+ k,
+ v,
+ dropout_p=self.attn_drop.p if self.training else 0.0,
+ softmax_scale=self.scale,
+ )
+ else:
+ dtype = q.dtype
+ q = q * self.scale
+ attn = q @ k.transpose(-2, -1) # translate attn to float32
+ attn = attn.to(torch.float32)
+ attn = attn.softmax(dim=-1)
+ attn = attn.to(dtype) # cast back attn to original dtype
+ attn = self.attn_drop(attn)
+ x = attn @ v
+
+ x_output_shape = (B, N, C)
+ if not self.enable_flashattn:
+ x = x.transpose(1, 2)
+ x = x.reshape(x_output_shape)
+ x = self.proj(x)
+ x = self.proj_drop(x)
+ return x
+
+
+class SeqParallelAttention(Attention):
+
+ def __init__(
+ self,
+ dim: int,
+ num_heads: int = 8,
+ qkv_bias: bool = False,
+ qk_norm: bool = False,
+ attn_drop: float = 0.0,
+ proj_drop: float = 0.0,
+ norm_layer: nn.Module = nn.LayerNorm,
+ enable_flashattn: bool = False,
+ ) -> None:
+ super().__init__(
+ dim=dim,
+ num_heads=num_heads,
+ qkv_bias=qkv_bias,
+ qk_norm=qk_norm,
+ attn_drop=attn_drop,
+ proj_drop=proj_drop,
+ norm_layer=norm_layer,
+ enable_flashattn=enable_flashattn,
+ )
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ B, N, C = x.shape # for sequence parallel here, the N is a local sequence length
+ qkv = self.qkv(x)
+ qkv_shape = (B, N, 3, self.num_heads, self.head_dim)
+
+ qkv = qkv.view(qkv_shape)
+
+ sp_group = get_sequence_parallel_group()
+
+ # apply all_to_all to gather sequence and split attention heads
+ # [B, SUB_N, 3, NUM_HEAD, HEAD_DIM] -> [B, N, 3, NUM_HEAD_PER_DEVICE, HEAD_DIM]
+ qkv = all_to_all(qkv, sp_group, scatter_dim=3, gather_dim=1)
+
+ if self.enable_flashattn:
+ qkv_permute_shape = (2, 0, 1, 3, 4) # [3, B, N, NUM_HEAD_PER_DEVICE, HEAD_DIM]
+ else:
+ qkv_permute_shape = (2, 0, 3, 1, 4) # [3, B, NUM_HEAD_PER_DEVICE, N, HEAD_DIM]
+ qkv = qkv.permute(qkv_permute_shape)
+
+ q, k, v = qkv.unbind(0)
+ q, k = self.q_norm(q), self.k_norm(k)
+ if self.enable_flashattn:
+ from flash_attn import flash_attn_func
+
+ x = flash_attn_func(
+ q,
+ k,
+ v,
+ dropout_p=self.attn_drop.p if self.training else 0.0,
+ softmax_scale=self.scale,
+ )
+ else:
+ dtype = q.dtype
+ q = q * self.scale
+ attn = q @ k.transpose(-2, -1) # translate attn to float32
+ attn = attn.to(torch.float32)
+ attn = attn.softmax(dim=-1)
+ attn = attn.to(dtype) # cast back attn to original dtype
+ attn = self.attn_drop(attn)
+ x = attn @ v
+
+ if not self.enable_flashattn:
+ x = x.transpose(1, 2)
+
+ # apply all to all to gather back attention heads and split sequence
+ # [B, N, NUM_HEAD_PER_DEVICE, HEAD_DIM] -> [B, SUB_N, NUM_HEAD, HEAD_DIM]
+ x = all_to_all(x, sp_group, scatter_dim=1, gather_dim=2)
+
+ # reshape outputs back to [B, N, C]
+ x_output_shape = (B, N, C)
+ x = x.reshape(x_output_shape)
+ x = self.proj(x)
+ x = self.proj_drop(x)
+ return x
+
+
+class MultiHeadCrossAttention(nn.Module):
+ def __init__(self, d_model, num_heads, attn_drop=0.0, proj_drop=0.0):
+ super(MultiHeadCrossAttention, self).__init__()
+ assert d_model % num_heads == 0, "d_model must be divisible by num_heads"
+
+ self.d_model = d_model
+ self.num_heads = num_heads
+ self.head_dim = d_model // num_heads
+
+ self.q_linear = nn.Linear(d_model, d_model)
+ self.kv_linear = nn.Linear(d_model, d_model * 2)
+ self.attn_drop = nn.Dropout(attn_drop)
+ self.proj = nn.Linear(d_model, d_model)
+ self.proj_drop = nn.Dropout(proj_drop)
+
+ def forward(self, x, cond, mask=None):
+ # query/value: img tokens; key: condition; mask: if padding tokens
+ B, N, C = x.shape
+
+ q = self.q_linear(x).view(1, -1, self.num_heads, self.head_dim)
+ kv = self.kv_linear(cond).view(1, -1, 2, self.num_heads, self.head_dim)
+ k, v = kv.unbind(2)
+
+ attn_bias = None
+ if mask is not None:
+ attn_bias = xformers.ops.fmha.BlockDiagonalMask.from_seqlens([N] * B, mask)
+ x = xformers.ops.memory_efficient_attention(q, k, v, p=self.attn_drop.p, attn_bias=attn_bias)
+
+ x = x.view(B, -1, C)
+ x = self.proj(x)
+ x = self.proj_drop(x)
+ return x
+
+
+class SeqParallelMultiHeadCrossAttention(MultiHeadCrossAttention):
+
+ def __init__(
+ self,
+ d_model,
+ num_heads,
+ attn_drop=0.0,
+ proj_drop=0.0,
+ ):
+ super().__init__(d_model=d_model, num_heads=num_heads, attn_drop=attn_drop, proj_drop=proj_drop)
+
+ def forward(self, x, cond, mask=None):
+ # query/value: img tokens; key: condition; mask: if padding tokens
+ sp_group = get_sequence_parallel_group()
+ sp_size = dist.get_world_size(sp_group)
+ B, SUB_N, C = x.shape
+ N = SUB_N * sp_size
+
+ # shape:
+ # q, k, v: [B, SUB_N, NUM_HEADS, HEAD_DIM]
+ q = self.q_linear(x).view(B, -1, self.num_heads, self.head_dim)
+ kv = self.kv_linear(cond).view(B, -1, 2, self.num_heads, self.head_dim)
+ k, v = kv.unbind(2)
+
+ # apply all_to_all to gather sequence and split attention heads
+ q = all_to_all(q, sp_group, scatter_dim=2, gather_dim=1)
+ k = all_to_all(k, sp_group, scatter_dim=2, gather_dim=1)
+ v = all_to_all(v, sp_group, scatter_dim=2, gather_dim=1)
+
+ q = q.view(1, -1, self.num_heads // sp_size, self.head_dim)
+ k = k.view(1, -1, self.num_heads // sp_size, self.head_dim)
+ v = v.view(1, -1, self.num_heads // sp_size, self.head_dim)
+
+ # compute attention
+ attn_bias = None
+ if mask is not None:
+ attn_bias = xformers.ops.fmha.BlockDiagonalMask.from_seqlens([N] * B, mask)
+ x = xformers.ops.memory_efficient_attention(q, k, v, p=self.attn_drop.p, attn_bias=attn_bias)
+
+ # apply all to all to gather back attention heads and scatter sequence
+ x = x.view(B, -1, self.num_heads // sp_size, self.head_dim)
+ x = all_to_all(x, sp_group, scatter_dim=1, gather_dim=2)
+
+ # apply output projection
+ x = x.view(B, -1, C)
+ x = self.proj(x)
+ x = self.proj_drop(x)
+ return x
+
+
+class FinalLayer(nn.Module):
+ """
+ The final layer of DiT.
+ """
+
+ def __init__(self, hidden_size, num_patch, out_channels):
+ super().__init__()
+ self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+ self.linear = nn.Linear(hidden_size, num_patch * out_channels, bias=True)
+ self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True))
+
+ def forward(self, x, c):
+ shift, scale = self.adaLN_modulation(c).chunk(2, dim=1)
+ x = modulate(self.norm_final, x, shift, scale)
+ x = self.linear(x)
+ return x
+
+
+class T2IFinalLayer(nn.Module):
+ """
+ The final layer of PixArt.
+ """
+
+ def __init__(self, hidden_size, num_patch, out_channels):
+ super().__init__()
+ self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
+ self.linear = nn.Linear(hidden_size, num_patch * out_channels, bias=True)
+ self.scale_shift_table = nn.Parameter(torch.randn(2, hidden_size) / hidden_size**0.5)
+ self.out_channels = out_channels
+
+ def forward(self, x, t):
+ shift, scale = (self.scale_shift_table[None] + t[:, None]).chunk(2, dim=1)
+ x = t2i_modulate(self.norm_final(x), shift, scale)
+ x = self.linear(x)
+ return x
+
+
+# ===============================================
+# Embedding Layers for Timesteps and Class Labels
+# ===============================================
+
+
+class TimestepEmbedder(nn.Module):
+ """
+ Embeds scalar timesteps into vector representations.
+ """
+
+ def __init__(self, hidden_size, frequency_embedding_size=256):
+ super().__init__()
+ self.mlp = nn.Sequential(
+ nn.Linear(frequency_embedding_size, hidden_size, bias=True),
+ nn.SiLU(),
+ nn.Linear(hidden_size, hidden_size, bias=True),
+ )
+ self.frequency_embedding_size = frequency_embedding_size
+
+ @staticmethod
+ def timestep_embedding(t, dim, max_period=10000):
+ """
+ Create sinusoidal timestep embeddings.
+ :param t: a 1-D Tensor of N indices, one per batch element.
+ These may be fractional.
+ :param dim: the dimension of the output.
+ :param max_period: controls the minimum frequency of the embeddings.
+ :return: an (N, D) Tensor of positional embeddings.
+ """
+ # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py
+ half = dim // 2
+ freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half)
+ freqs = freqs.to(device=t.device)
+ args = t[:, None].float() * freqs[None]
+ embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
+ if dim % 2:
+ embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
+ return embedding
+
+ def forward(self, t, dtype):
+ t_freq = self.timestep_embedding(t, self.frequency_embedding_size)
+ if t_freq.dtype != dtype:
+ t_freq = t_freq.to(dtype)
+ t_emb = self.mlp(t_freq)
+ return t_emb
+
+
+class LabelEmbedder(nn.Module):
+ """
+ Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance.
+ """
+
+ def __init__(self, num_classes, hidden_size, dropout_prob):
+ super().__init__()
+ use_cfg_embedding = dropout_prob > 0
+ self.embedding_table = nn.Embedding(num_classes + use_cfg_embedding, hidden_size)
+ self.num_classes = num_classes
+ self.dropout_prob = dropout_prob
+
+ def token_drop(self, labels, force_drop_ids=None):
+ """
+ Drops labels to enable classifier-free guidance.
+ """
+ if force_drop_ids is None:
+ drop_ids = torch.rand(labels.shape[0]).cuda() < self.dropout_prob
+ else:
+ drop_ids = force_drop_ids == 1
+ labels = torch.where(drop_ids, self.num_classes, labels)
+ return labels
+
+ def forward(self, labels, train, force_drop_ids=None):
+ use_dropout = self.dropout_prob > 0
+ if (train and use_dropout) or (force_drop_ids is not None):
+ labels = self.token_drop(labels, force_drop_ids)
+ return self.embedding_table(labels)
+
+
+class SizeEmbedder(TimestepEmbedder):
+ """
+ Embeds scalar timesteps into vector representations.
+ """
+
+ def __init__(self, hidden_size, frequency_embedding_size=256):
+ super().__init__(hidden_size=hidden_size, frequency_embedding_size=frequency_embedding_size)
+ self.mlp = nn.Sequential(
+ nn.Linear(frequency_embedding_size, hidden_size, bias=True),
+ nn.SiLU(),
+ nn.Linear(hidden_size, hidden_size, bias=True),
+ )
+ self.frequency_embedding_size = frequency_embedding_size
+ self.outdim = hidden_size
+
+ def forward(self, s, bs):
+ if s.ndim == 1:
+ s = s[:, None]
+ assert s.ndim == 2
+ if s.shape[0] != bs:
+ s = s.repeat(bs // s.shape[0], 1)
+ assert s.shape[0] == bs
+ b, dims = s.shape[0], s.shape[1]
+ s = rearrange(s, "b d -> (b d)")
+ s_freq = self.timestep_embedding(s, self.frequency_embedding_size).to(self.dtype)
+ s_emb = self.mlp(s_freq)
+ s_emb = rearrange(s_emb, "(b d) d2 -> b (d d2)", b=b, d=dims, d2=self.outdim)
+ return s_emb
+
+ @property
+ def dtype(self):
+ return next(self.parameters()).dtype
+
+
+class CaptionEmbedder(nn.Module):
+ """
+ Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance.
+ """
+
+ def __init__(self, in_channels, hidden_size, uncond_prob, act_layer=nn.GELU(approximate="tanh"), token_num=120):
+ super().__init__()
+ self.y_proj = Mlp(
+ in_features=in_channels, hidden_features=hidden_size, out_features=hidden_size, act_layer=act_layer, drop=0
+ )
+ self.register_buffer("y_embedding", nn.Parameter(torch.randn(token_num, in_channels) / in_channels**0.5))
+ self.uncond_prob = uncond_prob
+
+ def token_drop(self, caption, force_drop_ids=None):
+ """
+ Drops labels to enable classifier-free guidance.
+ """
+ if force_drop_ids is None:
+ drop_ids = torch.rand(caption.shape[0]).cuda() < self.uncond_prob
+ else:
+ drop_ids = force_drop_ids == 1
+ caption = torch.where(drop_ids[:, None, None, None], self.y_embedding, caption)
+ return caption
+
+ def forward(self, caption, train, force_drop_ids=None):
+ if train:
+ assert caption.shape[2:] == self.y_embedding.shape
+ use_dropout = self.uncond_prob > 0
+ if (train and use_dropout) or (force_drop_ids is not None):
+ caption = self.token_drop(caption, force_drop_ids)
+ caption = self.y_proj(caption)
+ return caption
+
+
+# ===============================================
+# Sine/Cosine Positional Embedding Functions
+# ===============================================
+# https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py
+
+
+def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0, scale=1.0, base_size=None):
+ """
+ grid_size: int of the grid height and width
+ return:
+ pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
+ """
+ if not isinstance(grid_size, tuple):
+ grid_size = (grid_size, grid_size)
+
+ grid_h = np.arange(grid_size[0], dtype=np.float32) / scale
+ grid_w = np.arange(grid_size[1], dtype=np.float32) / scale
+ if base_size is not None:
+ grid_h *= base_size / grid_size[0]
+ grid_w *= base_size / grid_size[1]
+ grid = np.meshgrid(grid_w, grid_h) # here w goes first
+ grid = np.stack(grid, axis=0)
+
+ grid = grid.reshape([2, 1, grid_size[1], grid_size[0]])
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
+ if cls_token and extra_tokens > 0:
+ pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0)
+ return pos_embed
+
+
+def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
+ assert embed_dim % 2 == 0
+
+ # use half of dimensions to encode grid_h
+ emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
+ emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
+
+ emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
+ return emb
+
+
+def get_1d_sincos_pos_embed(embed_dim, length, scale=1.0):
+ pos = np.arange(0, length)[..., None] / scale
+ return get_1d_sincos_pos_embed_from_grid(embed_dim, pos)
+
+
+def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
+ """
+ embed_dim: output dimension for each position
+ pos: a list of positions to be encoded: size (M,)
+ out: (M, D)
+ """
+ assert embed_dim % 2 == 0
+ omega = np.arange(embed_dim // 2, dtype=np.float64)
+ omega /= embed_dim / 2.0
+ omega = 1.0 / 10000**omega # (D/2,)
+
+ pos = pos.reshape(-1) # (M,)
+ out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product
+
+ emb_sin = np.sin(out) # (M, D/2)
+ emb_cos = np.cos(out) # (M, D/2)
+
+ emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
+ return emb
diff --git a/opensora/models/pixart/__init__.py b/opensora/models/pixart/__init__.py
new file mode 100644
index 0000000..622ad39
--- /dev/null
+++ b/opensora/models/pixart/__init__.py
@@ -0,0 +1 @@
+from .pixart import PixArt, PixArt_XL_2
diff --git a/opensora/models/pixart/pixart.py b/opensora/models/pixart/pixart.py
new file mode 100644
index 0000000..daad7dc
--- /dev/null
+++ b/opensora/models/pixart/pixart.py
@@ -0,0 +1,378 @@
+# Adapted from PixArt
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+# --------------------------------------------------------
+# References:
+# PixArt: https://github.com/PixArt-alpha/PixArt-alpha
+# DiT: https://github.com/facebookresearch/DiT/tree/main
+# --------------------------------------------------------
+
+import numpy as np
+import torch
+import torch.nn as nn
+from einops import rearrange
+from timm.models.layers import DropPath
+from timm.models.vision_transformer import Mlp
+
+# from .builder import MODELS
+from torch.utils.checkpoint import checkpoint
+from opensora.acceleration.checkpoint import auto_grad_checkpoint
+
+from opensora.models.layers.blocks import (
+ Attention,
+ CaptionEmbedder,
+ MultiHeadCrossAttention,
+ PatchEmbed3D,
+ SeqParallelAttention,
+ SeqParallelMultiHeadCrossAttention,
+ SizeEmbedder,
+ T2IFinalLayer,
+ TimestepEmbedder,
+ approx_gelu,
+ get_1d_sincos_pos_embed,
+ get_2d_sincos_pos_embed,
+ get_layernorm,
+ t2i_modulate,
+)
+from opensora.registry import MODELS
+from opensora.utils.ckpt_utils import load_checkpoint
+
+
+class PixArtBlock(nn.Module):
+ """
+ A PixArt block with adaptive layer norm (adaLN-single) conditioning.
+ """
+
+ def __init__(
+ self,
+ hidden_size,
+ num_heads,
+ mlp_ratio=4.0,
+ drop_path=0.0,
+ enable_flashattn=False,
+ enable_layernorm_kernel=False,
+ enable_sequence_parallelism=False,
+ ):
+ super().__init__()
+ self.hidden_size = hidden_size
+ self.enable_flashattn = enable_flashattn
+ self._enable_sequence_parallelism = enable_sequence_parallelism
+
+ if enable_sequence_parallelism:
+ self.attn_cls = SeqParallelAttention
+ self.mha_cls = SeqParallelMultiHeadCrossAttention
+ else:
+ self.attn_cls = Attention
+ self.mha_cls = MultiHeadCrossAttention
+
+ self.norm1 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)
+ self.attn = self.attn_cls(
+ hidden_size,
+ num_heads=num_heads,
+ qkv_bias=True,
+ enable_flashattn=enable_flashattn,
+ )
+ self.cross_attn = self.mha_cls(hidden_size, num_heads)
+ self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)
+ self.mlp = Mlp(
+ in_features=hidden_size, hidden_features=int(hidden_size * mlp_ratio), act_layer=approx_gelu, drop=0
+ )
+ self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+ self.scale_shift_table = nn.Parameter(torch.randn(6, hidden_size) / hidden_size**0.5)
+
+ def forward(self, x, y, t, mask=None):
+ B, N, C = x.shape
+
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
+ self.scale_shift_table[None] + t.reshape(B, 6, -1)
+ ).chunk(6, dim=1)
+ x = x + self.drop_path(gate_msa * self.attn(t2i_modulate(self.norm1(x), shift_msa, scale_msa)).reshape(B, N, C))
+ x = x + self.cross_attn(x, y, mask)
+ x = x + self.drop_path(gate_mlp * self.mlp(t2i_modulate(self.norm2(x), shift_mlp, scale_mlp)))
+
+ return x
+
+
+class PixArt(nn.Module):
+ """
+ Diffusion model with a Transformer backbone.
+ """
+
+ def __init__(
+ self,
+ input_size=(1, 32, 32),
+ in_channels=4,
+ patch_size=(1, 2, 2),
+ hidden_size=1152,
+ depth=28,
+ num_heads=16,
+ mlp_ratio=4.0,
+ class_dropout_prob=0.1,
+ pred_sigma=True,
+ drop_path: float = 0.0,
+ no_temporal_pos_emb=False,
+ caption_channels=4096,
+ model_max_length=120,
+ dtype=torch.float32,
+ freeze=None,
+ space_scale=1.0,
+ time_scale=1.0,
+ enable_flashattn=False,
+ enable_layernorm_kernel=False,
+ ):
+ super().__init__()
+ self.pred_sigma = pred_sigma
+ self.in_channels = in_channels
+ self.out_channels = in_channels * 2 if pred_sigma else in_channels
+ self.hidden_size = hidden_size
+ self.patch_size = patch_size
+ self.input_size = input_size
+ num_patches = np.prod([input_size[i] // patch_size[i] for i in range(3)])
+ self.num_patches = num_patches
+ self.num_temporal = input_size[0] // patch_size[0]
+ self.num_spatial = num_patches // self.num_temporal
+ self.base_size = int(np.sqrt(self.num_spatial))
+ self.num_heads = num_heads
+ self.dtype = dtype
+ self.no_temporal_pos_emb = no_temporal_pos_emb
+ self.depth = depth
+ self.mlp_ratio = mlp_ratio
+ self.enable_flashattn = enable_flashattn
+ self.enable_layernorm_kernel = enable_layernorm_kernel
+ self.space_scale = space_scale
+ self.time_scale = time_scale
+
+ self.x_embedder = PatchEmbed3D(patch_size, in_channels, hidden_size)
+ self.t_embedder = TimestepEmbedder(hidden_size)
+ self.t_block = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True))
+ self.y_embedder = CaptionEmbedder(
+ in_channels=caption_channels,
+ hidden_size=hidden_size,
+ uncond_prob=class_dropout_prob,
+ act_layer=approx_gelu,
+ token_num=model_max_length,
+ )
+
+ self.register_buffer("pos_embed", self.get_spatial_pos_embed())
+ self.register_buffer("pos_embed_temporal", self.get_temporal_pos_embed())
+
+ drop_path = [x.item() for x in torch.linspace(0, drop_path, depth)] # stochastic depth decay rule
+ self.blocks = nn.ModuleList(
+ [
+ PixArtBlock(
+ hidden_size,
+ num_heads,
+ mlp_ratio=mlp_ratio,
+ drop_path=drop_path[i],
+ enable_flashattn=enable_flashattn,
+ enable_layernorm_kernel=enable_layernorm_kernel,
+ )
+ for i in range(depth)
+ ]
+ )
+ self.final_layer = T2IFinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)
+
+ self.initialize_weights()
+ if freeze is not None:
+ assert freeze in ["text"]
+ if freeze == "text":
+ self.freeze_text()
+
+ def forward(self, x, timestep, y, mask=None):
+ """
+ Forward pass of PixArt.
+ x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images)
+ t: (N,) tensor of diffusion timesteps
+ y: (N, 1, 120, C) tensor of class labels
+ """
+ x = x.to(self.dtype)
+ timestep = timestep.to(self.dtype)
+ y = y.to(self.dtype)
+
+ # embedding
+ x = self.x_embedder(x) # (B, N, D)
+ x = rearrange(x, "b (t s) d -> b t s d", t=self.num_temporal, s=self.num_spatial)
+ x = x + self.pos_embed
+ if not self.no_temporal_pos_emb:
+ x = rearrange(x, "b t s d -> b s t d")
+ x = x + self.pos_embed_temporal
+ x = rearrange(x, "b s t d -> b (t s) d")
+ else:
+ x = rearrange(x, "b t s d -> b (t s) d")
+
+ t = self.t_embedder(timestep, dtype=x.dtype) # (N, D)
+ t0 = self.t_block(t)
+ y = self.y_embedder(y, self.training) # (N, 1, L, D)
+ if mask is not None:
+ if mask.shape[0] != y.shape[0]:
+ mask = mask.repeat(y.shape[0] // mask.shape[0], 1)
+ mask = mask.squeeze(1).squeeze(1)
+ y = y.squeeze(1).masked_select(mask.unsqueeze(-1) != 0).view(1, -1, x.shape[-1])
+ y_lens = mask.sum(dim=1).tolist()
+ else:
+ y_lens = [y.shape[2]] * y.shape[0]
+ y = y.squeeze(1).view(1, -1, x.shape[-1])
+
+ # blocks
+ for block in self.blocks:
+ x = auto_grad_checkpoint(block, x, y, t0, y_lens)
+
+ # final process
+ x = self.final_layer(x, t) # (N, T, patch_size ** 2 * out_channels)
+ x = self.unpatchify(x) # (N, out_channels, H, W)
+
+ # cast to float32 for better accuracy
+ x = x.to(torch.float32)
+ return x
+
+ def unpatchify(self, x):
+ c = self.out_channels
+ t, h, w = [self.input_size[i] // self.patch_size[i] for i in range(3)]
+ pt, ph, pw = self.patch_size
+
+ x = x.reshape(shape=(x.shape[0], t, h, w, pt, ph, pw, c))
+ x = rearrange(x, "n t h w r p q c -> n c t r h p w q")
+ imgs = x.reshape(shape=(x.shape[0], c, t * pt, h * ph, w * pw))
+ return imgs
+
+ def get_spatial_pos_embed(self, grid_size=None):
+ if grid_size is None:
+ grid_size = self.input_size[1:]
+ pos_embed = get_2d_sincos_pos_embed(
+ self.hidden_size,
+ (grid_size[0] // self.patch_size[1], grid_size[1] // self.patch_size[2]),
+ scale=self.space_scale,
+ base_size=self.base_size,
+ )
+ pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)
+ return pos_embed
+
+ def get_temporal_pos_embed(self):
+ pos_embed = get_1d_sincos_pos_embed(
+ self.hidden_size,
+ self.input_size[0] // self.patch_size[0],
+ scale=self.time_scale,
+ )
+ pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)
+ return pos_embed
+
+ def freeze_text(self):
+ for n, p in self.named_parameters():
+ if "cross_attn" in n:
+ p.requires_grad = False
+
+ def initialize_weights(self):
+ # Initialize transformer layers:
+ def _basic_init(module):
+ if isinstance(module, nn.Linear):
+ torch.nn.init.xavier_uniform_(module.weight)
+ if module.bias is not None:
+ nn.init.constant_(module.bias, 0)
+
+ self.apply(_basic_init)
+
+ # Initialize patch_embed like nn.Linear (instead of nn.Conv2d):
+ w = self.x_embedder.proj.weight.data
+ nn.init.xavier_uniform_(w.view([w.shape[0], -1]))
+
+ # Initialize timestep embedding MLP:
+ nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02)
+ nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02)
+ nn.init.normal_(self.t_block[1].weight, std=0.02)
+
+ # Initialize caption embedding MLP:
+ nn.init.normal_(self.y_embedder.y_proj.fc1.weight, std=0.02)
+ nn.init.normal_(self.y_embedder.y_proj.fc2.weight, std=0.02)
+
+ # Zero-out adaLN modulation layers in PixArt blocks:
+ for block in self.blocks:
+ nn.init.constant_(block.cross_attn.proj.weight, 0)
+ nn.init.constant_(block.cross_attn.proj.bias, 0)
+
+ # Zero-out output layers:
+ nn.init.constant_(self.final_layer.linear.weight, 0)
+ nn.init.constant_(self.final_layer.linear.bias, 0)
+
+
+class PixArtMS(PixArt):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ assert self.hidden_size % 3 == 0, "hidden_size must be divisible by 3"
+ self.csize_embedder = SizeEmbedder(self.hidden_size // 3)
+ self.ar_embedder = SizeEmbedder(self.hidden_size // 3)
+
+ def forward(self, x, timestep, y, mask=None, data_info=None):
+ """
+ Forward pass of PixArt.
+ x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images)
+ t: (N,) tensor of diffusion timesteps
+ y: (N, 1, 120, C) tensor of class labels
+ """
+ x = x.to(self.dtype)
+ timestep = timestep.to(self.dtype)
+ y = y.to(self.dtype)
+
+ c_size = data_info["hw"]
+ ar = data_info["ar"]
+ pos_embed = self.get_spatial_pos_embed((x.shape[-2], x.shape[-1])).to(x.dtype)
+
+ # embedding
+ x = self.x_embedder(x) # (B, N, D)
+ x = rearrange(x, "b (t s) d -> b t s d", t=self.num_temporal, s=self.num_spatial)
+ x = x + pos_embed.to(x.device)
+ if not self.no_temporal_pos_emb:
+ x = rearrange(x, "b t s d -> b s t d")
+ x = x + self.pos_embed_temporal
+ x = rearrange(x, "b s t d -> b (t s) d")
+ else:
+ x = rearrange(x, "b t s d -> b (t s) d")
+
+ t = self.t_embedder(timestep, dtype=x.dtype) # (N, D)
+ B = x.shape[0]
+ csize = self.csize_embedder(c_size, B)
+ ar = self.ar_embedder(ar, B)
+ t = t + torch.cat([csize, ar], dim=1)
+
+ t0 = self.t_block(t)
+ y = self.y_embedder(y, self.training) # (N, 1, L, D)
+ if mask is not None:
+ if mask.shape[0] != y.shape[0]:
+ mask = mask.repeat(y.shape[0] // mask.shape[0], 1)
+ mask = mask.squeeze(1).squeeze(1)
+ y = y.squeeze(1).masked_select(mask.unsqueeze(-1) != 0).view(1, -1, x.shape[-1])
+ y_lens = mask.sum(dim=1).tolist()
+ else:
+ y_lens = [y.shape[2]] * y.shape[0]
+ y = y.squeeze(1).view(1, -1, x.shape[-1])
+
+ # blocks
+ for block in self.blocks:
+ x = block(x, y, t0, y_lens)
+
+ # final process
+ x = self.final_layer(x, t) # (N, T, patch_size ** 2 * out_channels)
+ x = self.unpatchify(x) # (N, out_channels, H, W)
+
+ # cast to float32 for better accuracy
+ x = x.to(torch.float32)
+ return x
+
+
[email protected]_module("PixArt-XL/2")
+def PixArt_XL_2(from_pretrained=None, **kwargs):
+ model = PixArt(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
+
+
[email protected]_module("PixArtMS-XL/2")
+def PixArtMS_XL_2(from_pretrained=None, **kwargs):
+ model = PixArtMS(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
diff --git a/opensora/models/stdit/__init__.py b/opensora/models/stdit/__init__.py
new file mode 100644
index 0000000..3b089af
--- /dev/null
+++ b/opensora/models/stdit/__init__.py
@@ -0,0 +1 @@
+from .stdit import STDiT
diff --git a/opensora/models/stdit/stdit.py b/opensora/models/stdit/stdit.py
new file mode 100644
index 0000000..c8a2e95
--- /dev/null
+++ b/opensora/models/stdit/stdit.py
@@ -0,0 +1,359 @@
+
+import numpy as np
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+from einops import rearrange
+from timm.models.layers import DropPath
+from timm.models.vision_transformer import Mlp
+
+from opensora.acceleration.checkpoint import auto_grad_checkpoint
+from opensora.acceleration.communications import gather_forward_split_backward, split_forward_gather_backward
+from opensora.acceleration.parallel_states import get_sequence_parallel_group
+from opensora.models.layers.blocks import (
+ Attention,
+ CaptionEmbedder,
+ MultiHeadCrossAttention,
+ PatchEmbed3D,
+ SeqParallelAttention,
+ SeqParallelMultiHeadCrossAttention,
+ T2IFinalLayer,
+ TimestepEmbedder,
+ approx_gelu,
+ get_1d_sincos_pos_embed,
+ get_2d_sincos_pos_embed,
+ get_layernorm,
+ t2i_modulate,
+)
+from opensora.registry import MODELS
+from opensora.utils.ckpt_utils import load_checkpoint
+
+
+class STDiTBlock(nn.Module):
+ def __init__(
+ self,
+ hidden_size,
+ num_heads,
+ d_s=None,
+ d_t=None,
+ mlp_ratio=4.0,
+ drop_path=0.0,
+ enable_flashattn=False,
+ enable_layernorm_kernel=False,
+ enable_sequence_parallelism=False,
+ ):
+
+ super().__init__()
+ self.hidden_size = hidden_size
+ self.enable_flashattn = enable_flashattn
+ self._enable_sequence_parallelism = enable_sequence_parallelism
+
+ if enable_sequence_parallelism:
+ self.attn_cls = SeqParallelAttention
+ self.mha_cls = SeqParallelMultiHeadCrossAttention
+ else:
+ self.attn_cls = Attention
+ self.mha_cls = MultiHeadCrossAttention
+
+ self.norm1 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)
+ self.attn = self.attn_cls(
+ hidden_size,
+ num_heads=num_heads,
+ qkv_bias=True,
+ enable_flashattn=enable_flashattn,
+ )
+ self.cross_attn = self.mha_cls(hidden_size, num_heads)
+ self.norm2 = get_layernorm(hidden_size, eps=1e-6, affine=False, use_kernel=enable_layernorm_kernel)
+ self.mlp = Mlp(
+ in_features=hidden_size, hidden_features=int(hidden_size * mlp_ratio), act_layer=approx_gelu, drop=0
+ )
+ self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+ self.scale_shift_table = nn.Parameter(torch.randn(6, hidden_size) / hidden_size**0.5)
+
+ # temporal attention
+ self.d_s = d_s
+ self.d_t = d_t
+
+ if self._enable_sequence_parallelism:
+ sp_size = dist.get_world_size(get_sequence_parallel_group())
+ # make sure d_t is divisible by sp_size
+ assert d_t % sp_size == 0
+ self.d_t = d_t // sp_size
+
+ self.attn_temp = self.attn_cls(
+ hidden_size,
+ num_heads=num_heads,
+ qkv_bias=True,
+ enable_flashattn=self.enable_flashattn,
+ )
+
+ def forward(self, x, y, t, mask=None, tpe=None):
+ B, N, C = x.shape
+
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (
+ self.scale_shift_table[None] + t.reshape(B, 6, -1)
+ ).chunk(6, dim=1)
+ x_m = t2i_modulate(self.norm1(x), shift_msa, scale_msa)
+
+ # spatial branch
+ x_s = rearrange(x_m, "b (t s) d -> (b t) s d", t=self.d_t, s=self.d_s)
+ x_s = self.attn(x_s)
+ x_s = rearrange(x_s, "(b t) s d -> b (t s) d", t=self.d_t, s=self.d_s)
+ x = x + self.drop_path(gate_msa * x_s)
+
+ # temporal branch
+ x_t = rearrange(x, "b (t s) d -> (b s) t d", t=self.d_t, s=self.d_s)
+ if tpe is not None:
+ x_t = x_t + tpe
+ x_t = self.attn_temp(x_t)
+ x_t = rearrange(x_t, "(b s) t d -> b (t s) d", t=self.d_t, s=self.d_s)
+ x = x + self.drop_path(gate_msa * x_t)
+
+ # cross attn
+ x = x + self.cross_attn(x, y, mask)
+
+ # mlp
+ x = x + self.drop_path(gate_mlp * self.mlp(t2i_modulate(self.norm2(x), shift_mlp, scale_mlp)))
+
+ return x
+
+
+class STDiT(nn.Module):
+ def __init__(
+ self,
+ input_size=(1, 32, 32),
+ in_channels=4,
+ patch_size=(1, 2, 2),
+ hidden_size=1152,
+ depth=28,
+ num_heads=16,
+ mlp_ratio=4.0,
+ class_dropout_prob=0.1,
+ pred_sigma=True,
+ drop_path=0.0,
+ no_temporal_pos_emb=False,
+ caption_channels=4096,
+ model_max_length=120,
+ dtype=torch.float32,
+ space_scale=1.0,
+ time_scale=1.0,
+ freeze=None,
+ enable_flashattn=False,
+ enable_layernorm_kernel=False,
+ enable_sequence_parallelism=False,
+ ):
+ super().__init__()
+ self.pred_sigma = pred_sigma
+ self.in_channels = in_channels
+ self.out_channels = in_channels * 2 if pred_sigma else in_channels
+ self.hidden_size = hidden_size
+ self.patch_size = patch_size
+ self.input_size = input_size
+ num_patches = np.prod([input_size[i] // patch_size[i] for i in range(3)])
+ self.num_patches = num_patches
+ self.num_temporal = input_size[0] // patch_size[0]
+ self.num_spatial = num_patches // self.num_temporal
+ self.num_heads = num_heads
+ self.dtype = dtype
+ self.no_temporal_pos_emb = no_temporal_pos_emb
+ self.depth = depth
+ self.mlp_ratio = mlp_ratio
+ self.enable_flashattn = enable_flashattn
+ self.enable_layernorm_kernel = enable_layernorm_kernel
+ self.space_scale = space_scale
+ self.time_scale = time_scale
+
+ self.register_buffer("pos_embed", self.get_spatial_pos_embed())
+ self.register_buffer("pos_embed_temporal", self.get_temporal_pos_embed())
+
+ self.x_embedder = PatchEmbed3D(patch_size, in_channels, hidden_size)
+ self.t_embedder = TimestepEmbedder(hidden_size)
+ self.t_block = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True))
+ self.y_embedder = CaptionEmbedder(
+ in_channels=caption_channels,
+ hidden_size=hidden_size,
+ uncond_prob=class_dropout_prob,
+ act_layer=approx_gelu,
+ token_num=model_max_length,
+ )
+
+ drop_path = [x.item() for x in torch.linspace(0, drop_path, depth)]
+ self.blocks = nn.ModuleList(
+ [
+ STDiTBlock(
+ self.hidden_size,
+ self.num_heads,
+ mlp_ratio=self.mlp_ratio,
+ drop_path=drop_path[i],
+ enable_flashattn=self.enable_flashattn,
+ enable_layernorm_kernel=self.enable_layernorm_kernel,
+ enable_sequence_parallelism=enable_sequence_parallelism,
+ d_t=self.num_temporal,
+ d_s=self.num_spatial,
+ )
+ for i in range(self.depth)
+ ]
+ )
+ self.final_layer = T2IFinalLayer(hidden_size, np.prod(self.patch_size), self.out_channels)
+
+ # init model
+ self.initialize_weights()
+ self.initialize_temporal()
+ if freeze is not None:
+ assert freeze in ["not_temporal", "text"]
+ if freeze == "not_temporal":
+ self.freeze_not_temporal()
+ elif freeze == "text":
+ self.freeze_text()
+
+ # sequence parallel related configs
+ self.enable_sequence_parallelism = enable_sequence_parallelism
+ if enable_sequence_parallelism:
+ self.sp_rank = dist.get_rank(get_sequence_parallel_group())
+ else:
+ self.sp_rank = None
+
+ def forward(self, x, timestep, y, mask=None):
+ """
+ Forward pass of PixArt.
+ x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images)
+ t: (N,) tensor of diffusion timesteps
+ y: (N, 1, 120, C) tensor of class labels
+ """
+ x = x.to(self.dtype)
+ timestep = timestep.to(self.dtype)
+ y = y.to(self.dtype)
+
+ # embedding
+ x = self.x_embedder(x) # (B, N, D)
+ x = rearrange(x, "b (t s) d -> b t s d", t=self.num_temporal, s=self.num_spatial)
+ x = x + self.pos_embed
+ x = rearrange(x, "b t s d -> b (t s) d")
+
+ # shard over the sequence dim if sp is enabled
+ if self.enable_sequence_parallelism:
+ x = split_forward_gather_backward(x, get_sequence_parallel_group(), dim=1, grad_scale="down")
+
+ t = self.t_embedder(timestep, dtype=x.dtype) # (N, D)
+ t0 = self.t_block(t)
+ y = self.y_embedder(y, self.training) # (N, 1, L, D)
+
+ if mask is not None:
+ if mask.shape[0] != y.shape[0]:
+ mask = mask.repeat(y.shape[0] // mask.shape[0], 1)
+ mask = mask.squeeze(1).squeeze(1)
+ y = y.squeeze(1).masked_select(mask.unsqueeze(-1) != 0).view(1, -1, x.shape[-1])
+ y_lens = mask.sum(dim=1).tolist()
+ else:
+ y_lens = [y.shape[2]] * y.shape[0]
+ y = y.squeeze(1).view(1, -1, x.shape[-1])
+
+ # blocks
+ for i, block in enumerate(self.blocks):
+ if i == 0:
+ if self.enable_sequence_parallelism:
+ tpe = torch.chunk(
+ self.pos_embed_temporal, dist.get_world_size(get_sequence_parallel_group()), dim=1
+ )[self.sp_rank].contiguous()
+ else:
+ tpe = self.pos_embed_temporal
+ else:
+ tpe = None
+ x = auto_grad_checkpoint(block, x, y, t0, y_lens, tpe)
+
+ if self.enable_sequence_parallelism:
+ x = gather_forward_split_backward(x, get_sequence_parallel_group(), dim=1, grad_scale="up")
+
+ # final process
+ x = self.final_layer(x, t) # (N, T, patch_size ** 2 * out_channels)
+ x = self.unpatchify(x) # (N, out_channels, H, W)
+
+ # cast to float32 for better accuracy
+ x = x.to(torch.float32)
+ return x
+
+ def unpatchify(self, x):
+ c = self.out_channels
+ t, h, w = [self.input_size[i] // self.patch_size[i] for i in range(3)]
+ pt, ph, pw = self.patch_size
+
+ x = x.reshape(shape=(x.shape[0], t, h, w, pt, ph, pw, c))
+ x = rearrange(x, "n t h w r p q c -> n c t r h p w q")
+ imgs = x.reshape(shape=(x.shape[0], c, t * pt, h * ph, w * pw))
+ return imgs
+
+ def get_spatial_pos_embed(self, grid_size=None):
+ if grid_size is None:
+ grid_size = self.input_size[1:]
+ pos_embed = get_2d_sincos_pos_embed(
+ self.hidden_size,
+ (grid_size[0] // self.patch_size[1], grid_size[1] // self.patch_size[2]),
+ scale=self.space_scale,
+ )
+ pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)
+ return pos_embed
+
+ def get_temporal_pos_embed(self):
+ pos_embed = get_1d_sincos_pos_embed(
+ self.hidden_size,
+ self.input_size[0] // self.patch_size[0],
+ scale=self.time_scale,
+ )
+ pos_embed = torch.from_numpy(pos_embed).float().unsqueeze(0).requires_grad_(False)
+ return pos_embed
+
+ def freeze_not_temporal(self):
+ for n, p in self.named_parameters():
+ if "attn_temp" not in n:
+ p.requires_grad = False
+
+ def freeze_text(self):
+ for n, p in self.named_parameters():
+ if "cross_attn" in n:
+ p.requires_grad = False
+
+ def initialize_temporal(self):
+ for block in self.blocks:
+ nn.init.constant_(block.attn_temp.proj.weight, 0)
+ nn.init.constant_(block.attn_temp.proj.bias, 0)
+
+ def initialize_weights(self):
+ # Initialize transformer layers:
+ def _basic_init(module):
+ if isinstance(module, nn.Linear):
+ torch.nn.init.xavier_uniform_(module.weight)
+ if module.bias is not None:
+ nn.init.constant_(module.bias, 0)
+
+ self.apply(_basic_init)
+
+ # Initialize patch_embed like nn.Linear (instead of nn.Conv2d):
+ w = self.x_embedder.proj.weight.data
+ nn.init.xavier_uniform_(w.view([w.shape[0], -1]))
+
+ # Initialize timestep embedding MLP:
+ nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02)
+ nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02)
+ nn.init.normal_(self.t_block[1].weight, std=0.02)
+
+ # Initialize caption embedding MLP:
+ nn.init.normal_(self.y_embedder.y_proj.fc1.weight, std=0.02)
+ nn.init.normal_(self.y_embedder.y_proj.fc2.weight, std=0.02)
+
+ # Zero-out adaLN modulation layers in PixArt blocks:
+ for block in self.blocks:
+ nn.init.constant_(block.cross_attn.proj.weight, 0)
+ nn.init.constant_(block.cross_attn.proj.bias, 0)
+
+ # Zero-out output layers:
+ nn.init.constant_(self.final_layer.linear.weight, 0)
+ nn.init.constant_(self.final_layer.linear.bias, 0)
+
+
[email protected]_module("STDiT-XL/2")
+def STDiT_XL_2(from_pretrained=None, **kwargs):
+ model = STDiT(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)
+ if from_pretrained is not None:
+ load_checkpoint(model, from_pretrained)
+ return model
diff --git a/opensora/models/text_encoder/__init__.py b/opensora/models/text_encoder/__init__.py
new file mode 100644
index 0000000..fa494a2
--- /dev/null
+++ b/opensora/models/text_encoder/__init__.py
@@ -0,0 +1,3 @@
+from .t5 import T5Encoder
+from .classes import ClassEncoder
+from .clip import ClipEncoder
diff --git a/opensora/models/text_encoder/classes.py b/opensora/models/text_encoder/classes.py
new file mode 100644
index 0000000..f02c9f2
--- /dev/null
+++ b/opensora/models/text_encoder/classes.py
@@ -0,0 +1,20 @@
+import torch
+
+from opensora.registry import MODELS
+
+
[email protected]_module("classes")
+class ClassEncoder:
+ def __init__(self, num_classes, model_max_length=None, device="cuda", dtype=torch.float):
+ self.num_classes = num_classes
+ self.y_embedder = None
+
+ self.model_max_length = model_max_length
+ self.output_dim = None
+ self.device = device
+
+ def encode(self, text):
+ return dict(y=torch.tensor([int(t) for t in text]).to(self.device))
+
+ def null(self, n):
+ return torch.tensor([self.num_classes] * n).to(self.device)
diff --git a/opensora/models/text_encoder/clip.py b/opensora/models/text_encoder/clip.py
new file mode 100644
index 0000000..cf3f414
--- /dev/null
+++ b/opensora/models/text_encoder/clip.py
@@ -0,0 +1,100 @@
+# Adapted from Latte
+
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+# --------------------------------------------------------
+# References:
+# Latte: https://github.com/Vchitect/Latte
+# DiT: https://github.com/facebookresearch/DiT/tree/main
+# --------------------------------------------------------
+
+
+import torch
+import torch.nn as nn
+import transformers
+from transformers import CLIPTextModel, CLIPTokenizer
+
+from opensora.registry import MODELS
+
+transformers.logging.set_verbosity_error()
+
+
+class AbstractEncoder(nn.Module):
+ def __init__(self):
+ super().__init__()
+
+ def encode(self, *args, **kwargs):
+ raise NotImplementedError
+
+
+class FrozenCLIPEmbedder(AbstractEncoder):
+ """Uses the CLIP transformer encoder for text (from Hugging Face)"""
+
+ def __init__(self, path="openai/clip-vit-huge-patch14", device="cuda", max_length=77):
+ super().__init__()
+ self.tokenizer = CLIPTokenizer.from_pretrained(path)
+ self.transformer = CLIPTextModel.from_pretrained(path)
+ self.device = device
+ self.max_length = max_length
+ self._freeze()
+
+ def _freeze(self):
+ self.transformer = self.transformer.eval()
+ for param in self.parameters():
+ param.requires_grad = False
+
+ def forward(self, text):
+ batch_encoding = self.tokenizer(
+ text,
+ truncation=True,
+ max_length=self.max_length,
+ return_length=True,
+ return_overflowing_tokens=False,
+ padding="max_length",
+ return_tensors="pt",
+ )
+ tokens = batch_encoding["input_ids"].to(self.device)
+ outputs = self.transformer(input_ids=tokens)
+
+ z = outputs.last_hidden_state
+ pooled_z = outputs.pooler_output
+ return z, pooled_z
+
+ def encode(self, text):
+ return self(text)
+
+
[email protected]_module("clip")
+class ClipEncoder:
+ """
+ Embeds text prompt into vector representations. Also handles text dropout for classifier-free guidance.
+ """
+
+ def __init__(
+ self,
+ from_pretrained,
+ model_max_length=77,
+ device="cuda",
+ dtype=torch.float,
+ ):
+ super().__init__()
+ assert from_pretrained is not None, "Please specify the path to the T5 model"
+
+ self.text_encoder = FrozenCLIPEmbedder(path=from_pretrained, max_length=model_max_length).to(device, dtype)
+ self.y_embedder = None
+
+ self.model_max_length = model_max_length
+ self.output_dim = self.text_encoder.transformer.config.hidden_size
+
+ def encode(self, text):
+ _, pooled_embeddings = self.text_encoder.encode(text)
+ y = pooled_embeddings.unsqueeze(1).unsqueeze(1)
+ return dict(y=y)
+
+ def null(self, n):
+ null_y = self.y_embedder.y_embedding[None].repeat(n, 1, 1)[:, None]
+ return null_y
+
+ def to(self, dtype):
+ self.text_encoder = self.text_encoder.to(dtype)
+ return self
diff --git a/opensora/models/text_encoder/t5.py b/opensora/models/text_encoder/t5.py
new file mode 100644
index 0000000..672e6d9
--- /dev/null
+++ b/opensora/models/text_encoder/t5.py
@@ -0,0 +1,345 @@
+# Adapted from PixArt
+
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+# --------------------------------------------------------
+# References:
+# PixArt: https://github.com/PixArt-alpha/PixArt-alpha
+# T5: https://github.com/google-research/text-to-text-transfer-transformer
+# --------------------------------------------------------
+
+
+import html
+import os
+import re
+import urllib.parse as ul
+
+import ftfy
+import torch
+from bs4 import BeautifulSoup
+from huggingface_hub import hf_hub_download
+from transformers import AutoTokenizer, T5EncoderModel
+
+from opensora.registry import MODELS
+
+
+class T5Embedder:
+ available_models = ["t5-v1_1-xxl"]
+ bad_punct_regex = re.compile(
+ r"[" + "#®•©™&@·º½¾¿¡§~" + "\)" + "\(" + "\]" + "\[" + "\}" + "\{" + "\|" + "\\" + "\/" + "\*" + r"]{1,}"
+ ) # noqa
+
+ def __init__(
+ self,
+ device,
+ dir_or_name="t5-v1_1-xxl",
+ *,
+ local_cache=False,
+ cache_dir=None,
+ hf_token=None,
+ use_text_preprocessing=True,
+ t5_model_kwargs=None,
+ torch_dtype=None,
+ use_offload_folder=None,
+ model_max_length=120,
+ ):
+ self.device = torch.device(device)
+ self.torch_dtype = torch_dtype or torch.bfloat16
+ if t5_model_kwargs is None:
+ t5_model_kwargs = {"low_cpu_mem_usage": True, "torch_dtype": self.torch_dtype}
+ if use_offload_folder is not None:
+ t5_model_kwargs["offload_folder"] = use_offload_folder
+ t5_model_kwargs["device_map"] = {
+ "shared": self.device,
+ "encoder.embed_tokens": self.device,
+ "encoder.block.0": self.device,
+ "encoder.block.1": self.device,
+ "encoder.block.2": self.device,
+ "encoder.block.3": self.device,
+ "encoder.block.4": self.device,
+ "encoder.block.5": self.device,
+ "encoder.block.6": self.device,
+ "encoder.block.7": self.device,
+ "encoder.block.8": self.device,
+ "encoder.block.9": self.device,
+ "encoder.block.10": self.device,
+ "encoder.block.11": self.device,
+ "encoder.block.12": "disk",
+ "encoder.block.13": "disk",
+ "encoder.block.14": "disk",
+ "encoder.block.15": "disk",
+ "encoder.block.16": "disk",
+ "encoder.block.17": "disk",
+ "encoder.block.18": "disk",
+ "encoder.block.19": "disk",
+ "encoder.block.20": "disk",
+ "encoder.block.21": "disk",
+ "encoder.block.22": "disk",
+ "encoder.block.23": "disk",
+ "encoder.final_layer_norm": "disk",
+ "encoder.dropout": "disk",
+ }
+ else:
+ t5_model_kwargs["device_map"] = {"shared": self.device, "encoder": self.device}
+
+ self.use_text_preprocessing = use_text_preprocessing
+ self.hf_token = hf_token
+ self.cache_dir = cache_dir or os.path.expanduser("~/.cache/IF_")
+ self.dir_or_name = dir_or_name
+ tokenizer_path, path = dir_or_name, dir_or_name
+ if local_cache:
+ cache_dir = os.path.join(self.cache_dir, dir_or_name)
+ tokenizer_path, path = cache_dir, cache_dir
+ elif dir_or_name in self.available_models:
+ cache_dir = os.path.join(self.cache_dir, dir_or_name)
+ for filename in [
+ "config.json",
+ "special_tokens_map.json",
+ "spiece.model",
+ "tokenizer_config.json",
+ "pytorch_model.bin.index.json",
+ "pytorch_model-00001-of-00002.bin",
+ "pytorch_model-00002-of-00002.bin",
+ ]:
+ hf_hub_download(
+ repo_id=f"DeepFloyd/{dir_or_name}",
+ filename=filename,
+ cache_dir=cache_dir,
+ force_filename=filename,
+ token=self.hf_token,
+ )
+ tokenizer_path, path = cache_dir, cache_dir
+ else:
+ cache_dir = os.path.join(self.cache_dir, "t5-v1_1-xxl")
+ for filename in [
+ "config.json",
+ "special_tokens_map.json",
+ "spiece.model",
+ "tokenizer_config.json",
+ ]:
+ hf_hub_download(
+ repo_id="DeepFloyd/t5-v1_1-xxl",
+ filename=filename,
+ cache_dir=cache_dir,
+ force_filename=filename,
+ token=self.hf_token,
+ )
+ tokenizer_path = cache_dir
+
+ print(tokenizer_path)
+ self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
+ self.model = T5EncoderModel.from_pretrained(path, **t5_model_kwargs).eval()
+ self.model_max_length = model_max_length
+
+ def get_text_embeddings(self, texts):
+ texts = [self.text_preprocessing(text) for text in texts]
+
+ text_tokens_and_mask = self.tokenizer(
+ texts,
+ max_length=self.model_max_length,
+ padding="max_length",
+ truncation=True,
+ return_attention_mask=True,
+ add_special_tokens=True,
+ return_tensors="pt",
+ )
+
+ text_tokens_and_mask["input_ids"] = text_tokens_and_mask["input_ids"]
+ text_tokens_and_mask["attention_mask"] = text_tokens_and_mask["attention_mask"]
+
+ with torch.no_grad():
+ text_encoder_embs = self.model(
+ input_ids=text_tokens_and_mask["input_ids"].to(self.device),
+ attention_mask=text_tokens_and_mask["attention_mask"].to(self.device),
+ )["last_hidden_state"].detach()
+ return text_encoder_embs, text_tokens_and_mask["attention_mask"].to(self.device)
+
+ def text_preprocessing(self, text):
+ if self.use_text_preprocessing:
+ # The exact text cleaning as was in the training stage:
+ text = self.clean_caption(text)
+ text = self.clean_caption(text)
+ return text
+ else:
+ return text.lower().strip()
+
+ @staticmethod
+ def basic_clean(text):
+ text = ftfy.fix_text(text)
+ text = html.unescape(html.unescape(text))
+ return text.strip()
+
+ def clean_caption(self, caption):
+ caption = str(caption)
+ caption = ul.unquote_plus(caption)
+ caption = caption.strip().lower()
+ caption = re.sub("<person>", "person", caption)
+ # urls:
+ caption = re.sub(
+ r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa
+ "",
+ caption,
+ ) # regex for urls
+ caption = re.sub(
+ r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa
+ "",
+ caption,
+ ) # regex for urls
+ # html:
+ caption = BeautifulSoup(caption, features="html.parser").text
+
+ # @<nickname>
+ caption = re.sub(r"@[\w\d]+\b", "", caption)
+
+ # 31C0—31EF CJK Strokes
+ # 31F0—31FF Katakana Phonetic Extensions
+ # 3200—32FF Enclosed CJK Letters and Months
+ # 3300—33FF CJK Compatibility
+ # 3400—4DBF CJK Unified Ideographs Extension A
+ # 4DC0—4DFF Yijing Hexagram Symbols
+ # 4E00—9FFF CJK Unified Ideographs
+ caption = re.sub(r"[\u31c0-\u31ef]+", "", caption)
+ caption = re.sub(r"[\u31f0-\u31ff]+", "", caption)
+ caption = re.sub(r"[\u3200-\u32ff]+", "", caption)
+ caption = re.sub(r"[\u3300-\u33ff]+", "", caption)
+ caption = re.sub(r"[\u3400-\u4dbf]+", "", caption)
+ caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption)
+ caption = re.sub(r"[\u4e00-\u9fff]+", "", caption)
+ #######################################################
+
+ # все виды тире / all types of dash --> "-"
+ caption = re.sub(
+ r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa
+ "-",
+ caption,
+ )
+
+ # кавычки к одному стандарту
+ caption = re.sub(r"[`´«»“”¨]", '"', caption)
+ caption = re.sub(r"[‘’]", "'", caption)
+
+ # &quot;
+ caption = re.sub(r"&quot;?", "", caption)
+ # &amp
+ caption = re.sub(r"&amp", "", caption)
+
+ # ip adresses:
+ caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption)
+
+ # article ids:
+ caption = re.sub(r"\d:\d\d\s+$", "", caption)
+
+ # \n
+ caption = re.sub(r"\\n", " ", caption)
+
+ # "#123"
+ caption = re.sub(r"#\d{1,3}\b", "", caption)
+ # "#12345.."
+ caption = re.sub(r"#\d{5,}\b", "", caption)
+ # "123456.."
+ caption = re.sub(r"\b\d{6,}\b", "", caption)
+ # filenames:
+ caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption)
+
+ #
+ caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT"""
+ caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT"""
+
+ caption = re.sub(self.bad_punct_regex, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT
+ caption = re.sub(r"\s+\.\s+", r" ", caption) # " . "
+
+ # this-is-my-cute-cat / this_is_my_cute_cat
+ regex2 = re.compile(r"(?:\-|\_)")
+ if len(re.findall(regex2, caption)) > 3:
+ caption = re.sub(regex2, " ", caption)
+
+ caption = self.basic_clean(caption)
+
+ caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640
+ caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc
+ caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231
+
+ caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)
+ caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)
+ caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption)
+ caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption)
+ caption = re.sub(r"\bpage\s+\d+\b", "", caption)
+
+ caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a...
+
+ caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption)
+
+ caption = re.sub(r"\b\s+\:\s+", r": ", caption)
+ caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption)
+ caption = re.sub(r"\s+", " ", caption)
+
+ caption.strip()
+
+ caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption)
+ caption = re.sub(r"^[\'\_,\-\:;]", r"", caption)
+ caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption)
+ caption = re.sub(r"^\.\S+$", "", caption)
+
+ return caption.strip()
+
+
[email protected]_module("t5")
+class T5Encoder:
+ def __init__(
+ self,
+ from_pretrained=None,
+ model_max_length=120,
+ device="cuda",
+ dtype=torch.float,
+ local_cache=True,
+ shardformer=False,
+ ):
+ assert from_pretrained is not None, "Please specify the path to the T5 model"
+
+ self.t5 = T5Embedder(
+ device=device,
+ torch_dtype=dtype,
+ local_cache=local_cache,
+ cache_dir=from_pretrained,
+ model_max_length=model_max_length,
+ )
+ self.t5.model.to(dtype=dtype)
+ self.y_embedder = None
+
+ self.model_max_length = model_max_length
+ self.output_dim = self.t5.model.config.d_model
+
+ if shardformer:
+ self.shardformer_t5()
+
+ def shardformer_t5(self):
+ from colossalai.shardformer import ShardConfig, ShardFormer
+
+ from opensora.acceleration.shardformer.policy.t5_encoder import T5EncoderPolicy
+ from opensora.utils.misc import requires_grad
+
+ shard_config = ShardConfig(
+ tensor_parallel_process_group=None,
+ pipeline_stage_manager=None,
+ enable_tensor_parallelism=False,
+ enable_fused_normalization=False,
+ enable_flash_attention=False,
+ enable_jit_fused=True,
+ enable_sequence_parallelism=False,
+ enable_sequence_overlap=False,
+ )
+ shard_former = ShardFormer(shard_config=shard_config)
+ optim_model, _ = shard_former.optimize(self.t5.model, policy=T5EncoderPolicy())
+ self.t5.model = optim_model.half()
+
+ # ensure the weights are frozen
+ requires_grad(self.t5.model, False)
+
+ def encode(self, text):
+ caption_embs, emb_masks = self.t5.get_text_embeddings(text)
+ caption_embs = caption_embs[:, None]
+ return dict(y=caption_embs, mask=emb_masks)
+
+ def null(self, n):
+ null_y = self.y_embedder.y_embedding[None].repeat(n, 1, 1)[:, None]
+ return null_y
diff --git a/opensora/models/vae/__init__.py b/opensora/models/vae/__init__.py
new file mode 100644
index 0000000..04a09c9
--- /dev/null
+++ b/opensora/models/vae/__init__.py
@@ -0,0 +1 @@
+from .vae import VideoAutoencoderKL, VideoAutoencoderKLTemporalDecoder
diff --git a/opensora/models/vae/vae.py b/opensora/models/vae/vae.py
new file mode 100644
index 0000000..3761f0a
--- /dev/null
+++ b/opensora/models/vae/vae.py
@@ -0,0 +1,78 @@
+import torch
+import torch.nn as nn
+from diffusers.models import AutoencoderKL, AutoencoderKLTemporalDecoder
+from einops import rearrange
+
+from opensora.registry import MODELS
+
+
+class VideoAutoencoderKL(nn.Module):
+ def __init__(self, from_pretrained=None, split=None):
+ super().__init__()
+ self.module = AutoencoderKL.from_pretrained(from_pretrained)
+ self.out_channels = self.module.config.latent_channels
+ self.patch_size = (1, 8, 8)
+ self.split = split
+
+ def encode(self, x):
+ # x: (B, C, T, H, W)
+ B = x.shape[0]
+ x = rearrange(x, "B C T H W -> (B T) C H W")
+
+ if self.split is None:
+ x = self.module.encode(x).latent_dist.sample().mul_(0.18215)
+ else:
+ bs = x.shape[0] // self.split
+ x_out = []
+ for i in range(self.split):
+ x_out.append(self.module.encode(x[i * bs : (i + 1) * bs]).latent_dist.sample().mul_(0.18215))
+ x = torch.cat(x_out, dim=0)
+ x = rearrange(x, "(B T) C H W -> B C T H W", B=B)
+ return x
+
+ def decode(self, x):
+ # x: (B, C, T, H, W)
+ B = x.shape[0]
+ x = rearrange(x, "B C T H W -> (B T) C H W")
+ if self.split is None:
+ x = self.module.decode(x / 0.18215).sample
+ else:
+ bs = x.shape[0] // self.split
+ x_out = []
+ for i in range(self.split):
+ x_out.append(self.module.decode(x[i * bs : (i + 1) * bs] / 0.18215).sample)
+ x = torch.cat(x_out, dim=0)
+ x = rearrange(x, "(B T) C H W -> B C T H W", B=B)
+ return x
+
+ def get_latent_size(self, input_size):
+ for i in range(3):
+ assert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"
+ input_size = [input_size[i] // self.patch_size[i] for i in range(3)]
+ return input_size
+
+
+class VideoAutoencoderKLTemporalDecoder(nn.Module):
+ def __init__(self, from_pretrained=None):
+ super().__init__()
+ self.module = AutoencoderKLTemporalDecoder.from_pretrained(from_pretrained)
+ self.out_channels = self.module.config.latent_channels
+ self.patch_size = (1, 8, 8)
+
+ def encode(self, x):
+ raise NotImplementedError
+
+ def decode(self, x):
+ B, _, T = x.shape[:3]
+ x = rearrange(x, "B C T H W -> (B T) C H W")
+ x = self.module.decode(x / 0.18215, num_frames=T).sample
+ x = rearrange(x, "(B T) C H W -> B C T H W", B=B)
+ return x
+
+ def get_latent_size(self, input_size):
+ for i in range(3):
+ assert input_size[i] % self.patch_size[i] == 0, "Input size must be divisible by patch size"
+ input_size = [input_size[i] // self.patch_size[i] for i in range(3)]
+ return input_size