Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

[feat] support for stdit2 sampling

Zangwei Zhenga01f6da20e13
diff --git a/configs/opensora-v1-1/inference/16x256x256.py b/configs/opensora-v1-1/inference/Vx1024.py
similarity index 88%
rename from configs/opensora-v1-1/inference/16x256x256.py
rename to configs/opensora-v1-1/inference/Vx1024.py
index b68602f..649d9e5 100644
--- a/configs/opensora-v1-1/inference/16x256x256.py
+++ b/configs/opensora-v1-1/inference/Vx1024.py
@@ -1,13 +1,13 @@
-num_frames = 16
+num_frames = 4
fps = 24 // 3
-image_size = (256, 256)
+image_size = (704, 1472)
+multi_resolution = "STDiT2"
# Define model
model = dict(
type="STDiT2-XL/2",
- space_scale=0.5,
- time_scale=1.0,
from_pretrained="PixArt-XL-2-1024-MS.pth",
+ input_sq_size=512,
enable_flashattn=True,
enable_layernorm_kernel=True,
)
diff --git a/configs/opensora-v1-1/inference/Vx256x256.py b/configs/opensora-v1-1/inference/Vx256x256.py
deleted file mode 100644
index f609c37..0000000
--- a/configs/opensora-v1-1/inference/Vx256x256.py
+++ /dev/null
@@ -1,38 +0,0 @@
-num_frames = None
-fps = 24 // 3
-image_size = (256, 256)
-
-# Define model
-model = dict(
- type="STDiT2-XL/2",
- space_scale=0.5,
- from_pretrained="PixArt-XL-2-1024-MS.pth",
- enable_flashattn=True,
- enable_layernorm_kernel=True,
-)
-vae = dict(
- type="VideoAutoencoderKL",
- from_pretrained="stabilityai/sd-vae-ft-ema",
- micro_batch_size=4,
-)
-text_encoder = dict(
- type="t5",
- from_pretrained="DeepFloyd/t5-v1_1-xxl",
- model_max_length=120,
-)
-scheduler = dict(
- type="iddpm",
- num_sampling_steps=100,
- cfg_scale=7.0,
- cfg_channel=3, # or None
-)
-dtype = "fp16"
-
-# Condition
-prompt_path = "./assets/texts/t2v_samples.txt"
-prompt = None # prompt has higher priority than prompt_path
-
-# Others
-batch_size = 1
-seed = 42
-save_dir = "./outputs/samples/"
diff --git a/configs/opensora-v1-1/train/Vx360p.py b/configs/opensora-v1-1/train/Vx360p.py
index 8d831a6..2d8305d 100644
--- a/configs/opensora-v1-1/train/Vx360p.py
+++ b/configs/opensora-v1-1/train/Vx360p.py
@@ -4,7 +4,7 @@ dataset = dict(
data_path=None,
num_frames=None,
frame_interval=3,
- image_size=(512, 512), # pretrained model is trained on 512x512
+ image_size=(None, None, None),
transform_name="resize_crop",
)
bucket_config = {
@@ -25,6 +25,7 @@ sp_size = 1
model = dict(
type="STDiT2-XL/2",
from_pretrained="PixArt-XL-2-1024-MS.pth",
+ input_sq_size=512, # pretrained model is trained on 512x512
enable_flashattn=True,
enable_layernorm_kernel=True,
)
diff --git a/configs/pixart/inference/1x1024MS.py b/configs/pixart/inference/1x1024MS.py
index 3b34ebf..90687e5 100644
--- a/configs/pixart/inference/1x1024MS.py
+++ b/configs/pixart/inference/1x1024MS.py
@@ -1,7 +1,7 @@
num_frames = 1
fps = 1
image_size = (1920, 512)
-multi_resolution = True
+multi_resolution = "PixArtMS"
# Define model
model = dict(
diff --git a/opensora/datasets/utils.py b/opensora/datasets/utils.py
index 82166da..7cf7d32 100644
--- a/opensora/datasets/utils.py
+++ b/opensora/datasets/utils.py
@@ -1,9 +1,10 @@
+import numbers
+
import numpy as np
import torch
import torchvision
import torchvision.transforms as transforms
from PIL import Image
-
from torchvision.datasets.folder import IMG_EXTENSIONS, pil_loader
from torchvision.io import write_video
from torchvision.utils import save_image
@@ -63,7 +64,13 @@ def get_transforms_image(name="center", image_size=(256, 256)):
]
)
elif name == "resize_crop":
- transform = None
+ transform = transforms.Compose(
+ [
+ transforms.Lambda(lambda pil_image: resize_crop_to_fill(pil_image, image_size)),
+ transforms.ToTensor(),
+ transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True),
+ ]
+ )
else:
raise NotImplementedError(f"Transform {name} not implemented")
return transform
@@ -136,3 +143,21 @@ def center_crop_arr(pil_image, image_size):
crop_y = (arr.shape[0] - image_size) // 2
crop_x = (arr.shape[1] - image_size) // 2
return Image.fromarray(arr[crop_y : crop_y + image_size, crop_x : crop_x + image_size])
+
+
+def resize_crop_to_fill(pil_image, image_size):
+ w, h = pil_image.size # PIL is (W, H)
+ th, tw = image_size
+ rh, rw = th / h, tw / w
+ if rh > rw:
+ sh, sw = th, int(w * rh)
+ image = pil_image.resize((sw, sh), Image.BICUBIC)
+ i = 0
+ j = int(round((sw - tw) / 2.0))
+ else:
+ sh, sw = int(h * rw), tw
+ image = pil_image.resize((sw, sh), Image.BICUBIC)
+ i = int(round((sh - th) / 2.0))
+ j = 0
+ arr = np.array(image)
+ return Image.fromarray(arr[i : i + th, j : j + tw])
diff --git a/opensora/models/layers/blocks.py b/opensora/models/layers/blocks.py
index b8824ce..4df738c 100644
--- a/opensora/models/layers/blocks.py
+++ b/opensora/models/layers/blocks.py
@@ -29,9 +29,7 @@ from opensora.acceleration.parallel_states import get_sequence_parallel_group
approx_gelu = lambda: nn.GELU(approximate="tanh")
-def get_layernorm(
- hidden_size: torch.Tensor, eps: float, affine: bool, use_kernel: bool
-):
+def get_layernorm(hidden_size: torch.Tensor, eps: float, affine: bool, use_kernel: bool):
if use_kernel:
try:
from apex.normalization import FusedLayerNorm
@@ -86,9 +84,7 @@ class PatchEmbed3D(nn.Module):
self.in_chans = in_chans
self.embed_dim = embed_dim
- self.proj = nn.Conv3d(
- in_chans, embed_dim, kernel_size=patch_size, stride=patch_size
- )
+ self.proj = nn.Conv3d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
if norm_layer is not None:
self.norm = norm_layer(embed_dim)
else:
@@ -221,9 +217,7 @@ class SeqParallelAttention(Attention):
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
- B, N, C = (
- x.shape
- ) # for sequence parallel here, the N is a local sequence length
+ B, N, C = x.shape # for sequence parallel here, the N is a local sequence length
qkv = self.qkv(x)
qkv_shape = (B, N, 3, self.num_heads, self.head_dim)
@@ -316,9 +310,7 @@ class MultiHeadCrossAttention(nn.Module):
attn_bias = None
if mask is not None:
attn_bias = xformers.ops.fmha.BlockDiagonalMask.from_seqlens([N] * B, mask)
- x = xformers.ops.memory_efficient_attention(
- q, k, v, p=self.attn_drop.p, attn_bias=attn_bias
- )
+ x = xformers.ops.memory_efficient_attention(q, k, v, p=self.attn_drop.p, attn_bias=attn_bias)
x = x.view(B, -1, C)
x = self.proj(x)
@@ -357,12 +349,8 @@ class SeqParallelMultiHeadCrossAttention(MultiHeadCrossAttention):
# apply all_to_all to gather sequence and split attention heads
q = all_to_all(q, sp_group, scatter_dim=2, gather_dim=1)
- k = split_forward_gather_backward(
- k, get_sequence_parallel_group(), dim=2, grad_scale="down"
- )
- v = split_forward_gather_backward(
- v, get_sequence_parallel_group(), dim=2, grad_scale="down"
- )
+ k = split_forward_gather_backward(k, get_sequence_parallel_group(), dim=2, grad_scale="down")
+ v = split_forward_gather_backward(v, get_sequence_parallel_group(), dim=2, grad_scale="down")
q = q.view(1, -1, self.num_heads // sp_size, self.head_dim)
k = k.view(1, -1, self.num_heads // sp_size, self.head_dim)
@@ -372,9 +360,7 @@ class SeqParallelMultiHeadCrossAttention(MultiHeadCrossAttention):
attn_bias = None
if mask is not None:
attn_bias = xformers.ops.fmha.BlockDiagonalMask.from_seqlens([N] * B, mask)
- x = xformers.ops.memory_efficient_attention(
- q, k, v, p=self.attn_drop.p, attn_bias=attn_bias
- )
+ x = xformers.ops.memory_efficient_attention(q, k, v, p=self.attn_drop.p, attn_bias=attn_bias)
# apply all to all to gather back attention heads and scatter sequence
x = x.view(B, -1, self.num_heads // sp_size, self.head_dim)
@@ -396,9 +382,7 @@ class FinalLayer(nn.Module):
super().__init__()
self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
self.linear = nn.Linear(hidden_size, num_patch * out_channels, bias=True)
- self.adaLN_modulation = nn.Sequential(
- nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True)
- )
+ self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True))
def forward(self, x, c):
shift, scale = self.adaLN_modulation(c).chunk(2, dim=1)
@@ -416,9 +400,7 @@ class T2IFinalLayer(nn.Module):
super().__init__()
self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
self.linear = nn.Linear(hidden_size, num_patch * out_channels, bias=True)
- self.scale_shift_table = nn.Parameter(
- torch.randn(2, hidden_size) / hidden_size**0.5
- )
+ self.scale_shift_table = nn.Parameter(torch.randn(2, hidden_size) / hidden_size**0.5)
self.out_channels = out_channels
self.d_t = d_t
self.d_s = d_s
@@ -441,9 +423,7 @@ class T2IFinalLayer(nn.Module):
shift, scale = (self.scale_shift_table[None] + t[:, None]).chunk(2, dim=1)
x = t2i_modulate(self.norm_final(x), shift, scale)
if x_mask is not None:
- shift_zero, scale_zero = (self.scale_shift_table[None] + t0[:, None]).chunk(
- 2, dim=1
- )
+ shift_zero, scale_zero = (self.scale_shift_table[None] + t0[:, None]).chunk(2, dim=1)
x_zero = t2i_modulate(self.norm_final(x), shift_zero, scale_zero)
x = self.t_mask_select(x_mask, x, x_zero, T, S)
x = self.linear(x)
@@ -481,18 +461,12 @@ class TimestepEmbedder(nn.Module):
"""
# https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py
half = dim // 2
- freqs = torch.exp(
- -math.log(max_period)
- * torch.arange(start=0, end=half, dtype=torch.float32)
- / half
- )
+ freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half)
freqs = freqs.to(device=t.device)
args = t[:, None].float() * freqs[None]
embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
if dim % 2:
- embedding = torch.cat(
- [embedding, torch.zeros_like(embedding[:, :1])], dim=-1
- )
+ embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
return embedding
def forward(self, t, dtype):
@@ -511,9 +485,7 @@ class LabelEmbedder(nn.Module):
def __init__(self, num_classes, hidden_size, dropout_prob):
super().__init__()
use_cfg_embedding = dropout_prob > 0
- self.embedding_table = nn.Embedding(
- num_classes + use_cfg_embedding, hidden_size
- )
+ self.embedding_table = nn.Embedding(num_classes + use_cfg_embedding, hidden_size)
self.num_classes = num_classes
self.dropout_prob = dropout_prob
@@ -541,9 +513,7 @@ class SizeEmbedder(TimestepEmbedder):
"""
def __init__(self, hidden_size, frequency_embedding_size=256):
- super().__init__(
- hidden_size=hidden_size, frequency_embedding_size=frequency_embedding_size
- )
+ super().__init__(hidden_size=hidden_size, frequency_embedding_size=frequency_embedding_size)
self.mlp = nn.Sequential(
nn.Linear(frequency_embedding_size, hidden_size, bias=True),
nn.SiLU(),
@@ -561,9 +531,7 @@ class SizeEmbedder(TimestepEmbedder):
assert s.shape[0] == bs
b, dims = s.shape[0], s.shape[1]
s = rearrange(s, "b d -> (b d)")
- s_freq = self.timestep_embedding(s, self.frequency_embedding_size).to(
- self.dtype
- )
+ s_freq = self.timestep_embedding(s, self.frequency_embedding_size).to(self.dtype)
s_emb = self.mlp(s_freq)
s_emb = rearrange(s_emb, "(b d) d2 -> b (d d2)", b=b, d=dims, d2=self.outdim)
return s_emb
@@ -622,13 +590,9 @@ class CaptionEmbedder(nn.Module):
class PositionEmbedding2D(nn.Module):
- def __init__(
- self, dim: int, max_position_embeddings: int = 512, scale: float = 1.0
- ) -> None:
+ def __init__(self, dim: int) -> None:
super().__init__()
self.dim = dim
- self.max_position_embeddings = max_position_embeddings
- self.scale = scale
assert dim % 4 == 0, "dim must be divisible by 4"
half_dim = dim // 2
inv_freq = 1.0 / (10000 ** (torch.arange(0, half_dim, 2).float() / half_dim))
@@ -640,17 +604,18 @@ class PositionEmbedding2D(nn.Module):
emb_sin = torch.sin(out)
return torch.cat((emb_sin, emb_cos), dim=-1)
- @functools.lru_cache(maxsize=128)
+ @functools.lru_cache(maxsize=512)
def _get_cached_emb(
self,
device: torch.device,
dtype: torch.dtype,
h: int,
w: int,
+ scale: float = 1.0,
base_size: Optional[int] = None,
):
- grid_h = torch.arange(h, device=device) / self.scale
- grid_w = torch.arange(w, device=device) / self.scale
+ grid_h = torch.arange(h, device=device) / scale
+ grid_w = torch.arange(w, device=device) / scale
if base_size is not None:
grid_h *= base_size / h
grid_w *= base_size / w
@@ -666,9 +631,9 @@ class PositionEmbedding2D(nn.Module):
return torch.concat([emb_h, emb_w], dim=-1).unsqueeze(0).to(dtype)
def forward(
- self, x: torch.Tensor, h: int, w: int, base_size: Optional[int] = None
+ self, x: torch.Tensor, h: int, w: int, scale: Optional[float] = 1.0, base_size: Optional[int] = None
) -> torch.Tensor:
- return self._get_cached_emb(x.device, x.dtype, h, w, base_size)
+ return self._get_cached_emb(x.device, x.dtype, h, w, scale, base_size)
# ===============================================
@@ -677,9 +642,7 @@ class PositionEmbedding2D(nn.Module):
# https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py
-def get_2d_sincos_pos_embed(
- embed_dim, grid_size, cls_token=False, extra_tokens=0, scale=1.0, base_size=None
-):
+def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0, scale=1.0, base_size=None):
"""
grid_size: int of the grid height and width
return:
@@ -699,9 +662,7 @@ def get_2d_sincos_pos_embed(
grid = grid.reshape([2, 1, grid_size[1], grid_size[0]])
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
if cls_token and extra_tokens > 0:
- pos_embed = np.concatenate(
- [np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0
- )
+ pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0)
return pos_embed
diff --git a/opensora/models/stdit/stdit2.py b/opensora/models/stdit/stdit2.py
index 25973fd..e494209 100644
--- a/opensora/models/stdit/stdit2.py
+++ b/opensora/models/stdit/stdit2.py
@@ -15,13 +15,13 @@ from opensora.models.layers.blocks import (
CaptionEmbedder,
MultiHeadCrossAttention,
PatchEmbed3D,
+ PositionEmbedding2D,
SeqParallelAttention,
SeqParallelMultiHeadCrossAttention,
SizeEmbedder,
T2IFinalLayer,
TimestepEmbedder,
approx_gelu,
- get_1d_sincos_pos_embed,
get_2d_sincos_pos_embed,
get_layernorm,
t2i_modulate,
@@ -175,7 +175,8 @@ class STDiT2Block(nn.Module):
class STDiT2(nn.Module):
def __init__(
self,
- input_size=(None, 32, 32),
+ input_size=(None, None, None),
+ input_sq_size=32,
in_channels=4,
patch_size=(1, 2, 2),
hidden_size=1152,
@@ -210,7 +211,8 @@ class STDiT2(nn.Module):
# support dynamic input
self.patch_size = patch_size
self.input_size = input_size
- self.input_sq_size = int((input_size[1] * input_size[2]) ** 0.5)
+ self.input_sq_size = input_sq_size
+ self.pos_embed = PositionEmbedding2D(hidden_size)
self.x_embedder = PatchEmbed3D(patch_size, in_channels, hidden_size)
self.t_embedder = TimestepEmbedder(hidden_size)
@@ -298,23 +300,16 @@ class STDiT2(nn.Module):
# === process data info ===
# 1. get dynamic size
- if height is None or width is None:
- hw = torch.tensor([self.input_size[1], self.input_size[2]], device=x.device, dtype=x.dtype).repeat(B, 1)
- else:
- hw = torch.cat([height[:, None], width[:, None]], dim=1)
+ hw = torch.cat([height[:, None], width[:, None]], dim=1)
+ rs = (height[0].item() * width[0].item()) ** 0.5
csize = self.csize_embedder(hw, B)
# 2. get aspect ratio
- if ar is None:
- ar = torch.tensor([[self.input_size[1] / self.input_size[2]]], device=x.device, dtype=x.dtype).repeat(B, 1)
- else:
- ar = ar.unsqueeze(1)
+ ar = ar.unsqueeze(1)
ar = self.ar_embedder(ar, B)
data_info = torch.cat([csize, ar], dim=1)
# 3. get number of frames
- if num_frames is None:
- num_frames = torch.tensor([x.shape[2]], device=x.device, dtype=x.dtype)
fl = num_frames.unsqueeze(1)
fl = self.fl_embedder(fl, B)
@@ -322,9 +317,9 @@ class STDiT2(nn.Module):
_, _, Tx, Hx, Wx = x.size()
T, H, W = self.get_dynamic_size(x)
S = H * W
- rs = (height[0] * width[0]).sqrt().item()
scale = rs / self.input_sq_size
- pos_emb = self.get_spatial_pos_embed(H, W, scale=scale, base_size=round(S**0.5)).to(x.device, x.dtype)
+ base_size = round(S**0.5)
+ pos_emb = self.pos_embed(x, H, W, scale=scale, base_size=base_size)
# embedding
x = self.x_embedder(x) # [B, N, C]
diff --git a/scripts/inference-long.py b/scripts/inference-long.py
index f6e0a71..0784aef 100644
--- a/scripts/inference-long.py
+++ b/scripts/inference-long.py
@@ -131,11 +131,21 @@ def main():
# 3.4. support for multi-resolution
model_args = dict()
- if cfg.multi_resolution:
+ if cfg.multi_resolution == "PixArtMS":
image_size = cfg.image_size
hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
model_args["data_info"] = dict(ar=ar, hw=hw)
+ elif cfg.multi_resolution == "STDiT2":
+ image_size = cfg.image_size
+ height = torch.tensor([image_size[0]], device=device, dtype=dtype).repeat(cfg.batch_size)
+ width = torch.tensor([image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)
+ num_frames = torch.tensor([cfg.num_frames], device=device, dtype=dtype).repeat(cfg.batch_size)
+ ar = torch.tensor([image_size[0] / image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)
+ model_args["height"] = height
+ model_args["width"] = width
+ model_args["num_frames"] = num_frames
+ model_args["ar"] = ar
# 3.5 reference
if cfg.reference_path is not None:
diff --git a/scripts/inference.py b/scripts/inference.py
index f1323d2..e83ff5a 100644
--- a/scripts/inference.py
+++ b/scripts/inference.py
@@ -70,11 +70,21 @@ def main():
# 3.4. support for multi-resolution
model_args = dict()
- if cfg.multi_resolution:
+ if cfg.multi_resolution == "PixArtMS":
image_size = cfg.image_size
hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(cfg.batch_size, 1)
model_args["data_info"] = dict(ar=ar, hw=hw)
+ elif cfg.multi_resolution == "STDiT2":
+ image_size = cfg.image_size
+ height = torch.tensor([image_size[0]], device=device, dtype=dtype).repeat(cfg.batch_size)
+ width = torch.tensor([image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)
+ num_frames = torch.tensor([cfg.num_frames], device=device, dtype=dtype).repeat(cfg.batch_size)
+ ar = torch.tensor([image_size[0] / image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)
+ model_args["height"] = height
+ model_args["width"] = width
+ model_args["num_frames"] = num_frames
+ model_args["ar"] = ar
# ======================================================
# 4. inference
diff --git a/tools/datasets/csvutil.py b/tools/datasets/csvutil.py
index 7017943..dd76482 100644
--- a/tools/datasets/csvutil.py
+++ b/tools/datasets/csvutil.py
@@ -23,17 +23,26 @@ def apply(df, func):
return df.progress_apply(func)
+IMG_EXTENSIONS = (".jpg", ".jpeg", ".png", ".ppm", ".bmp", ".pgm", ".tif", ".tiff", ".webp")
+
+
def get_video_info(path):
import cv2
- cap = cv2.VideoCapture(path)
- num_frames, height, width, fps = (
- int(cap.get(cv2.CAP_PROP_FRAME_COUNT)),
- int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)),
- int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
- float(cap.get(cv2.CAP_PROP_FPS)),
- )
- aspect_ratio = height / width if width > 0 else np.nan
+ ext = os.path.splitext(path)[1].lower()
+ if ext in IMG_EXTENSIONS:
+ im = cv2.imread(path)
+ height, width = im.shape[:2]
+ num_frames, fps = 1, np.nan
+ else:
+ cap = cv2.VideoCapture(path)
+ num_frames, height, width, fps = (
+ int(cap.get(cv2.CAP_PROP_FRAME_COUNT)),
+ int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)),
+ int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
+ float(cap.get(cv2.CAP_PROP_FPS)),
+ )
+ aspect_ratio = height / width if width > 0 else np.nan
return num_frames, height, width, aspect_ratio, fps