Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

refactored code into a package (#13)

Frank Lee0c05cd2e9d6c
diff --git a/open_sora/__init__.py b/open_sora/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/diffusion/__init__.py b/open_sora/diffusion/__init__.py
similarity index 100%
rename from diffusion/__init__.py
rename to open_sora/diffusion/__init__.py
diff --git a/diffusion/diffusion_utils.py b/open_sora/diffusion/diffusion_utils.py
similarity index 100%
rename from diffusion/diffusion_utils.py
rename to open_sora/diffusion/diffusion_utils.py
diff --git a/diffusion/gaussian_diffusion.py b/open_sora/diffusion/gaussian_diffusion.py
similarity index 89%
rename from diffusion/gaussian_diffusion.py
rename to open_sora/diffusion/gaussian_diffusion.py
index 6c0b0de..fe94622 100644
--- a/diffusion/gaussian_diffusion.py
+++ b/open_sora/diffusion/gaussian_diffusion.py
@@ -47,9 +47,7 @@ class ModelVarType(enum.Enum):
class LossType(enum.Enum):
MSE = enum.auto() # use raw MSE loss (and KL when learning variances)
- RESCALED_MSE = (
- enum.auto()
- ) # use raw MSE loss (with RESCALED_KL when learning variances)
+ RESCALED_MSE = enum.auto() # use raw MSE loss (with RESCALED_KL when learning variances)
KL = enum.auto() # use the variational lower-bound
RESCALED_KL = enum.auto() # like KL, but rescale to estimate the full VLB
@@ -60,9 +58,7 @@ class LossType(enum.Enum):
def _warmup_beta(beta_start, beta_end, num_diffusion_timesteps, warmup_frac):
betas = beta_end * np.ones(num_diffusion_timesteps, dtype=np.float64)
warmup_time = int(num_diffusion_timesteps * warmup_frac)
- betas[:warmup_time] = np.linspace(
- beta_start, beta_end, warmup_time, dtype=np.float64
- )
+ betas[:warmup_time] = np.linspace(beta_start, beta_end, warmup_time, dtype=np.float64)
return betas
@@ -82,9 +78,7 @@ def get_beta_schedule(beta_schedule, *, beta_start, beta_end, num_diffusion_time
** 2
)
elif beta_schedule == "linear":
- betas = np.linspace(
- beta_start, beta_end, num_diffusion_timesteps, dtype=np.float64
- )
+ betas = np.linspace(beta_start, beta_end, num_diffusion_timesteps, dtype=np.float64)
elif beta_schedule == "warmup10":
betas = _warmup_beta(beta_start, beta_end, num_diffusion_timesteps, 0.1)
elif beta_schedule == "warmup50":
@@ -92,9 +86,7 @@ def get_beta_schedule(beta_schedule, *, beta_start, beta_end, num_diffusion_time
elif beta_schedule == "const":
betas = beta_end * np.ones(num_diffusion_timesteps, dtype=np.float64)
elif beta_schedule == "jsd": # 1/T, 1/(T-1), 1/(T-2), ..., 1
- betas = 1.0 / np.linspace(
- num_diffusion_timesteps, 1, num_diffusion_timesteps, dtype=np.float64
- )
+ betas = 1.0 / np.linspace(num_diffusion_timesteps, 1, num_diffusion_timesteps, dtype=np.float64)
else:
raise NotImplementedError(beta_schedule)
assert betas.shape == (num_diffusion_timesteps,)
@@ -183,9 +175,7 @@ class GaussianDiffusion:
self.sqrt_recipm1_alphas_cumprod = np.sqrt(1.0 / self.alphas_cumprod - 1)
# calculations for posterior q(x_{t-1} | x_t, x_0)
- self.posterior_variance = (
- betas * (1.0 - self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod)
- )
+ self.posterior_variance = betas * (1.0 - self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod)
# below: log calculation clipped because the posterior variance is 0 at the beginning of the diffusion chain
self.posterior_log_variance_clipped = (
np.log(np.append(self.posterior_variance[1], self.posterior_variance[1:]))
@@ -193,14 +183,8 @@ class GaussianDiffusion:
else np.array([])
)
- self.posterior_mean_coef1 = (
- betas * np.sqrt(self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod)
- )
- self.posterior_mean_coef2 = (
- (1.0 - self.alphas_cumprod_prev)
- * np.sqrt(alphas)
- / (1.0 - self.alphas_cumprod)
- )
+ self.posterior_mean_coef1 = betas * np.sqrt(self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod)
+ self.posterior_mean_coef2 = (1.0 - self.alphas_cumprod_prev) * np.sqrt(alphas) / (1.0 - self.alphas_cumprod)
def q_mean_variance(self, x_start, t):
"""
@@ -209,13 +193,9 @@ class GaussianDiffusion:
:param t: the number of diffusion steps (minus 1). Here, 0 means one step.
:return: A tuple (mean, variance, log_variance), all of x_start's shape.
"""
- mean = (
- _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start
- )
+ mean = _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start
variance = _extract_into_tensor(1.0 - self.alphas_cumprod, t, x_start.shape)
- log_variance = _extract_into_tensor(
- self.log_one_minus_alphas_cumprod, t, x_start.shape
- )
+ log_variance = _extract_into_tensor(self.log_one_minus_alphas_cumprod, t, x_start.shape)
return mean, variance, log_variance
def q_sample(self, x_start, t, noise=None):
@@ -232,8 +212,7 @@ class GaussianDiffusion:
assert noise.shape == x_start.shape
return (
_extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start
- + _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape)
- * noise
+ + _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise
)
def q_posterior_mean_variance(self, x_start, x_t, t):
@@ -247,9 +226,7 @@ class GaussianDiffusion:
+ _extract_into_tensor(self.posterior_mean_coef2, t, x_t.shape) * x_t
)
posterior_variance = _extract_into_tensor(self.posterior_variance, t, x_t.shape)
- posterior_log_variance_clipped = _extract_into_tensor(
- self.posterior_log_variance_clipped, t, x_t.shape
- )
+ posterior_log_variance_clipped = _extract_into_tensor(self.posterior_log_variance_clipped, t, x_t.shape)
assert (
posterior_mean.shape[0]
== posterior_variance.shape[0]
@@ -258,9 +235,7 @@ class GaussianDiffusion:
)
return posterior_mean, posterior_variance, posterior_log_variance_clipped
- def p_mean_variance(
- self, model, x, t, clip_denoised=True, denoised_fn=None, model_kwargs=None
- ):
+ def p_mean_variance(self, model, x, t, clip_denoised=True, denoised_fn=None, model_kwargs=None):
"""
Apply the model to get p(x_{t-1} | x_t), as well as a prediction of
the initial x, x_0.
@@ -294,9 +269,7 @@ class GaussianDiffusion:
if self.model_var_type in [ModelVarType.LEARNED, ModelVarType.LEARNED_RANGE]:
assert model_output.shape == (B, S, C * 2, *x.shape[3:])
model_output, model_var_values = th.split(model_output, C, dim=2)
- min_log = _extract_into_tensor(
- self.posterior_log_variance_clipped, t, x.shape
- )
+ min_log = _extract_into_tensor(self.posterior_log_variance_clipped, t, x.shape)
max_log = _extract_into_tensor(np.log(self.betas), t, x.shape)
# The model_var_values is [-1, 1] for [min_var, max_var].
frac = (model_var_values + 1) / 2
@@ -328,16 +301,10 @@ class GaussianDiffusion:
if self.model_mean_type == ModelMeanType.START_X:
pred_xstart = process_xstart(model_output)
else:
- pred_xstart = process_xstart(
- self._predict_xstart_from_eps(x_t=x, t=t, eps=model_output)
- )
- model_mean, _, _ = self.q_posterior_mean_variance(
- x_start=pred_xstart, x_t=x, t=t
- )
+ pred_xstart = process_xstart(self._predict_xstart_from_eps(x_t=x, t=t, eps=model_output))
+ model_mean, _, _ = self.q_posterior_mean_variance(x_start=pred_xstart, x_t=x, t=t)
- assert (
- model_mean.shape == model_log_variance.shape == pred_xstart.shape == x.shape
- )
+ assert model_mean.shape == model_log_variance.shape == pred_xstart.shape == x.shape
return {
"mean": model_mean,
"variance": model_variance,
@@ -355,8 +322,7 @@ class GaussianDiffusion:
def _predict_eps_from_xstart(self, x_t, t, pred_xstart):
return (
- _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t
- - pred_xstart
+ _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - pred_xstart
) / _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape)
def condition_mean(self, cond_fn, p_mean_var, x, t, model_kwargs=None):
@@ -368,9 +334,7 @@ class GaussianDiffusion:
This uses the conditioning strategy from Sohl-Dickstein et al. (2015).
"""
gradient = cond_fn(x, t, **model_kwargs)
- new_mean = (
- p_mean_var["mean"].float() + p_mean_var["variance"] * gradient.float()
- )
+ new_mean = p_mean_var["mean"].float() + p_mean_var["variance"] * gradient.float()
return new_mean
def condition_score(self, cond_fn, p_mean_var, x, t, model_kwargs=None):
@@ -388,9 +352,7 @@ class GaussianDiffusion:
out = p_mean_var.copy()
out["pred_xstart"] = self._predict_xstart_from_eps(x, t, eps)
- out["mean"], _, _ = self.q_posterior_mean_variance(
- x_start=out["pred_xstart"], x_t=x, t=t
- )
+ out["mean"], _, _ = self.q_posterior_mean_variance(x_start=out["pred_xstart"], x_t=x, t=t)
return out
def p_sample(
@@ -428,13 +390,9 @@ class GaussianDiffusion:
model_kwargs=model_kwargs,
)
noise = th.randn_like(x)
- nonzero_mask = (
- (t != 0).float().view(-1, *([1] * (len(x.shape) - 1)))
- ) # no noise when t == 0
+ nonzero_mask = (t != 0).float().view(-1, *([1] * (len(x.shape) - 1))) # no noise when t == 0
if cond_fn is not None:
- out["mean"] = self.condition_mean(
- cond_fn, out, x, t, model_kwargs=model_kwargs
- )
+ out["mean"] = self.condition_mean(cond_fn, out, x, t, model_kwargs=model_kwargs)
sample = out["mean"] + nonzero_mask * th.exp(0.5 * out["log_variance"]) * noise
return {"sample": sample, "pred_xstart": out["pred_xstart"]}
@@ -564,20 +522,11 @@ class GaussianDiffusion:
alpha_bar = _extract_into_tensor(self.alphas_cumprod, t, x.shape)
alpha_bar_prev = _extract_into_tensor(self.alphas_cumprod_prev, t, x.shape)
- sigma = (
- eta
- * th.sqrt((1 - alpha_bar_prev) / (1 - alpha_bar))
- * th.sqrt(1 - alpha_bar / alpha_bar_prev)
- )
+ sigma = eta * th.sqrt((1 - alpha_bar_prev) / (1 - alpha_bar)) * th.sqrt(1 - alpha_bar / alpha_bar_prev)
# Equation 12.
noise = th.randn_like(x)
- mean_pred = (
- out["pred_xstart"] * th.sqrt(alpha_bar_prev)
- + th.sqrt(1 - alpha_bar_prev - sigma**2) * eps
- )
- nonzero_mask = (
- (t != 0).float().view(-1, *([1] * (len(x.shape) - 1)))
- ) # no noise when t == 0
+ mean_pred = out["pred_xstart"] * th.sqrt(alpha_bar_prev) + th.sqrt(1 - alpha_bar_prev - sigma**2) * eps
+ nonzero_mask = (t != 0).float().view(-1, *([1] * (len(x.shape) - 1))) # no noise when t == 0
sample = mean_pred + nonzero_mask * sigma * noise
return {"sample": sample, "pred_xstart": out["pred_xstart"]}
@@ -609,16 +558,12 @@ class GaussianDiffusion:
# Usually our model outputs epsilon, but we re-derive it
# in case we used x_start or x_prev prediction.
eps = (
- _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x.shape) * x
- - out["pred_xstart"]
+ _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x.shape) * x - out["pred_xstart"]
) / _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x.shape)
alpha_bar_next = _extract_into_tensor(self.alphas_cumprod_next, t, x.shape)
# Equation 12. reversed
- mean_pred = (
- out["pred_xstart"] * th.sqrt(alpha_bar_next)
- + th.sqrt(1 - alpha_bar_next) * eps
- )
+ mean_pred = out["pred_xstart"] * th.sqrt(alpha_bar_next) + th.sqrt(1 - alpha_bar_next) * eps
return {"sample": mean_pred, "pred_xstart": out["pred_xstart"]}
@@ -704,9 +649,7 @@ class GaussianDiffusion:
yield out
img = out["sample"]
- def _vb_terms_bpd(
- self, model, x_start, x_t, t, clip_denoised=True, model_kwargs=None, mask=None
- ):
+ def _vb_terms_bpd(self, model, x_start, x_t, t, clip_denoised=True, model_kwargs=None, mask=None):
"""
Get a term for the variational lower-bound.
The resulting units are bits (rather than nats, as one might expect).
@@ -715,15 +658,9 @@ class GaussianDiffusion:
- 'output': a shape [N] tensor of NLLs or KLs.
- 'pred_xstart': the x_0 predictions.
"""
- true_mean, _, true_log_variance_clipped = self.q_posterior_mean_variance(
- x_start=x_start, x_t=x_t, t=t
- )
- out = self.p_mean_variance(
- model, x_t, t, clip_denoised=clip_denoised, model_kwargs=model_kwargs
- )
- kl = normal_kl(
- true_mean, true_log_variance_clipped, out["mean"], out["log_variance"]
- )
+ true_mean, _, true_log_variance_clipped = self.q_posterior_mean_variance(x_start=x_start, x_t=x_t, t=t)
+ out = self.p_mean_variance(model, x_t, t, clip_denoised=clip_denoised, model_kwargs=model_kwargs)
+ kl = normal_kl(true_mean, true_log_variance_clipped, out["mean"], out["log_variance"])
kl = mean_flat(kl, mask=mask) / np.log(2.0)
decoder_nll = -discretized_gaussian_log_likelihood(
@@ -743,9 +680,7 @@ class GaussianDiffusion:
mask = mask.view(*mask.shape, *([1] * (ndim - mask.ndim)))
return mask
- def training_losses(
- self, model, x_start, t, model_kwargs=None, noise=None, mask=None
- ):
+ def training_losses(self, model, x_start, t, model_kwargs=None, noise=None, mask=None):
"""
Compute training losses for a single timestep.
:param model: the model to evaluate loss on.
@@ -807,9 +742,7 @@ class GaussianDiffusion:
terms["vb"] *= self.num_timesteps / 1000.0
target = {
- ModelMeanType.PREVIOUS_X: self.q_posterior_mean_variance(
- x_start=x_start, x_t=x_t, t=t
- )[0],
+ ModelMeanType.PREVIOUS_X: self.q_posterior_mean_variance(x_start=x_start, x_t=x_t, t=t)[0],
ModelMeanType.START_X: x_start,
ModelMeanType.EPSILON: noise,
}[self.model_mean_type]
@@ -835,9 +768,7 @@ class GaussianDiffusion:
batch_size = x_start.shape[0]
t = th.tensor([self.num_timesteps - 1] * batch_size, device=x_start.device)
qt_mean, _, qt_log_variance = self.q_mean_variance(x_start, t)
- kl_prior = normal_kl(
- mean1=qt_mean, logvar1=qt_log_variance, mean2=0.0, logvar2=0.0
- )
+ kl_prior = normal_kl(mean1=qt_mean, logvar1=qt_log_variance, mean2=0.0, logvar2=0.0)
return mean_flat(kl_prior) / np.log(2.0)
def calc_bpd_loop(self, model, x_start, clip_denoised=True, model_kwargs=None):
diff --git a/diffusion/respace.py b/open_sora/diffusion/respace.py
similarity index 100%
rename from diffusion/respace.py
rename to open_sora/diffusion/respace.py
diff --git a/diffusion/timestep_sampler.py b/open_sora/diffusion/timestep_sampler.py
similarity index 100%
rename from diffusion/timestep_sampler.py
rename to open_sora/diffusion/timestep_sampler.py
diff --git a/open_sora/modeling/__init__.py b/open_sora/modeling/__init__.py
new file mode 100644
index 0000000..72ac366
--- /dev/null
+++ b/open_sora/modeling/__init__.py
@@ -0,0 +1,3 @@
+from .dit import DiT, DiT_models
+
+__all__ = ["DiT_models", "DiT"]
diff --git a/models.py b/open_sora/modeling/dit.py
similarity index 87%
rename from models.py
rename to open_sora/modeling/dit.py
index 45dea99..0ec1736 100644
--- a/models.py
+++ b/open_sora/modeling/dit.py
@@ -46,9 +46,7 @@ class CrossAttention(nn.Module):
):
super().__init__()
self.hidden_size = head_dim * num_heads
- cross_attention_dim = (
- cross_attention_dim if cross_attention_dim is not None else query_dim
- )
+ cross_attention_dim = cross_attention_dim if cross_attention_dim is not None else query_dim
self.scale = head_dim**-0.5
self.num_heads = num_heads
@@ -59,9 +57,7 @@ class CrossAttention(nn.Module):
self.to_k = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)
self.to_v = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)
- self.to_out = nn.Sequential(
- nn.Linear(self.hidden_size, query_dim), nn.Dropout(dropout)
- )
+ self.to_out = nn.Sequential(nn.Linear(self.hidden_size, query_dim), nn.Dropout(dropout))
def forward(self, hidden_states, context=None, mask=None):
bsz, q_len, _ = hidden_states.shape
@@ -75,24 +71,18 @@ class CrossAttention(nn.Module):
# [B, S, H, D]
query = query.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
key = key.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(1, 2)
- value = value.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(
- 1, 2
- )
+ value = value.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(1, 2)
if mask is not None:
assert mask.shape == (bsz, 1, q_len, kv_seq_len)
if self.sdpa:
- attn_output = F.scaled_dot_product_attention(
- query, key, value, attn_mask=mask, scale=self.scale
- )
+ attn_output = F.scaled_dot_product_attention(query, key, value, attn_mask=mask, scale=self.scale)
else:
attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scale
assert attn_weights.shape == (bsz, self.num_heads, q_len, kv_seq_len)
if mask is not None:
attn_weights = attn_weights + mask
- attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
- query.dtype
- )
+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
attn_output = torch.matmul(attn_weights, value)
assert attn_output.shape == (bsz, self.num_heads, q_len, self.head_dim)
attn_output = attn_output.transpose(1, 2).contiguous()
@@ -136,23 +126,17 @@ class TimestepEmbedder(nn.Module):
"""
# https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py
half = dim // 2
- freqs = torch.exp(
- -math.log(max_period)
- * torch.arange(start=0, end=half, dtype=torch.float32)
- / half
- ).to(device=t.device)
+ freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to(
+ device=t.device
+ )
args = t[:, None].float() * freqs[None]
embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
if dim % 2:
- embedding = torch.cat(
- [embedding, torch.zeros_like(embedding[:, :1])], dim=-1
- )
+ embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
return embedding
def forward(self, t):
- t_freq = self.timestep_embedding(t, self.frequency_embedding_size).to(
- self.mlp[0].weight.dtype
- )
+ t_freq = self.timestep_embedding(t, self.frequency_embedding_size).to(self.mlp[0].weight.dtype)
t_emb = self.mlp(t_freq)
return t_emb
@@ -165,9 +149,7 @@ class LabelEmbedder(nn.Module):
def __init__(self, num_classes, hidden_size, dropout_prob):
super().__init__()
use_cfg_embedding = dropout_prob > 0
- self.embedding_table = nn.Embedding(
- num_classes + use_cfg_embedding, hidden_size
- )
+ self.embedding_table = nn.Embedding(num_classes + use_cfg_embedding, hidden_size)
self.num_classes = num_classes
self.dropout_prob = dropout_prob
@@ -176,9 +158,7 @@ class LabelEmbedder(nn.Module):
Drops labels to enable classifier-free guidance.
"""
if force_drop_ids is None:
- drop_ids = (
- torch.rand(labels.shape[0], device=labels.device) < self.dropout_prob
- )
+ drop_ids = torch.rand(labels.shape[0], device=labels.device) < self.dropout_prob
else:
drop_ids = force_drop_ids == 1
labels = torch.where(drop_ids, self.num_classes, labels)
@@ -205,27 +185,21 @@ class PatchEmbedder(nn.Module):
) -> None:
super().__init__()
self.patch_size = patch_size
- self.proj = nn.Conv2d(
- in_chans, embed_dim, kernel_size=patch_size, stride=patch_size, bias=bias
- )
+ self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size, bias=bias)
self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
def forward(self, x: torch.Tensor) -> torch.Tensor:
# [B, S, C, P, P] -> [B, S, C*P*P]
# FIXME: hack diffusion and use view
x = x.view(*x.shape[:2], -1)
- out = F.linear(
- x, self.proj.weight.view(self.proj.weight.shape[0], -1), self.proj.bias
- )
+ out = F.linear(x, self.proj.weight.view(self.proj.weight.shape[0], -1), self.proj.bias)
out = self.norm(out)
# [B, S, H]
return out
class TextEmbedder(nn.Module):
- def __init__(
- self, in_features: int, embed_dim: int = 768, bias: bool = True
- ) -> None:
+ def __init__(self, in_features: int, embed_dim: int = 768, bias: bool = True) -> None:
super().__init__()
self.proj = nn.Linear(in_features, embed_dim, bias=bias)
@@ -267,9 +241,7 @@ class DiTBlock(nn.Module):
A DiT block with adaptive layer norm zero (adaLN-Zero) conditioning.
"""
- def __init__(
- self, hidden_size, num_heads, cross_attention_dim, mlp_ratio=4.0, **block_kwargs
- ):
+ def __init__(self, hidden_size, num_heads, cross_attention_dim, mlp_ratio=4.0, **block_kwargs):
super().__init__()
self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
self.attn = CrossAttention(
@@ -305,9 +277,7 @@ class FinalLayer(nn.Module):
def __init__(self, hidden_size, patch_size, out_channels):
super().__init__()
- self.linear = nn.Linear(
- hidden_size, patch_size * patch_size * out_channels, bias=True
- )
+ self.linear = nn.Linear(hidden_size, patch_size * patch_size * out_channels, bias=True)
self.patch_size = patch_size
def unpatchify(self, x):
@@ -345,17 +315,12 @@ class DiT(nn.Module):
self.patch_size = patch_size
self.num_heads = num_heads
- self.video_embedder = PatchEmbedder(
- patch_size, in_channels, hidden_size, bias=True
- )
+ self.video_embedder = PatchEmbedder(patch_size, in_channels, hidden_size, bias=True)
self.t_embedder = TimestepEmbedder(text_embed_dim)
self.pos_embed = PositionEmbedding(hidden_size, max_num_embeddings)
self.blocks = nn.ModuleList(
- [
- DiTBlock(hidden_size, num_heads, text_embed_dim, mlp_ratio=mlp_ratio)
- for _ in range(depth)
- ]
+ [DiTBlock(hidden_size, num_heads, text_embed_dim, mlp_ratio=mlp_ratio) for _ in range(depth)]
)
self.final_layer = FinalLayer(hidden_size, patch_size, self.out_channels)
self.initialize_weights()
@@ -384,9 +349,7 @@ class DiT(nn.Module):
assert attention_mask.ndim == 4
attention_mask = attention_mask.to(dtype)
inverted_mask = 1.0 - attention_mask
- return inverted_mask.masked_fill(
- inverted_mask.to(torch.bool), torch.finfo(dtype).min
- )
+ return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)
return attention_mask
def enable_gradient_checkpointing(self):
@@ -417,24 +380,18 @@ class DiT(nn.Module):
block, video_latent_states, text_latent_states, attention_mask
)
else:
- video_latent_states = block(
- video_latent_states, text_latent_states, attention_mask
- )
+ video_latent_states = block(video_latent_states, text_latent_states, attention_mask)
video_latent_states = self.final_layer(video_latent_states)
return video_latent_states
- def forward_with_cfg(
- self, x, t, text_latent_states, cfg_scale, attention_mask=None
- ):
+ def forward_with_cfg(self, x, t, text_latent_states, cfg_scale, attention_mask=None):
"""
Forward pass of DiT, but also batches the unconditional forward pass for classifier-free guidance.
"""
# https://github.com/openai/glide-text2im/blob/main/notebooks/text2im.ipynb
half = x[: len(x) // 2]
combined = torch.cat([half, half], dim=0)
- model_out = self.forward(
- combined, t, text_latent_states, attention_mask=attention_mask
- )
+ model_out = self.forward(combined, t, text_latent_states, attention_mask=attention_mask)
# For exact reproducibility reasons, we apply classifier-free guidance on only
# three channels by default. The standard approach to cfg applies it to all channels.
# This can be done by uncommenting the following line and commenting-out the line following that.
@@ -468,9 +425,7 @@ def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=
grid = grid.reshape([2, 1, grid_size, grid_size])
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
if cls_token and extra_tokens > 0:
- pos_embed = np.concatenate(
- [np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0
- )
+ pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0)
return pos_embed
diff --git a/open_sora/utils/__init__.py b/open_sora/utils/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/data_utils.py b/open_sora/utils/data.py
similarity index 87%
rename from data_utils.py
rename to open_sora/utils/data.py
index 35f67aa..8ef6ca4 100644
--- a/data_utils.py
+++ b/open_sora/utils/data.py
@@ -37,9 +37,7 @@ def video2col(video_4d: torch.Tensor, patch_size: int) -> torch.Tensor:
return torch.stack(out, dim=1).view(-1, c, patch_size, patch_size)
-def col2video(
- patches: torch.Tensor, video_shape: Tuple[int, int, int, int]
-) -> torch.Tensor:
+def col2video(patches: torch.Tensor, video_shape: Tuple[int, int, int, int]) -> torch.Tensor:
"""
Convert a 2D tensor of patches to a 4D video tensor.
@@ -76,10 +74,7 @@ def pad_sequences(sequences: List[torch.Tensor]) -> Tuple[torch.Tensor, torch.Te
"""
max_len = max([sequence.shape[0] for sequence in sequences])
padded_sequences = [
- F.pad(
- sequence, [0] * (sequence.ndim - 1) * 2 + [0, max_len - sequence.shape[0]]
- )
- for sequence in sequences
+ F.pad(sequence, [0] * (sequence.ndim - 1) * 2 + [0, max_len - sequence.shape[0]]) for sequence in sequences
]
padded_sequences = torch.stack(padded_sequences, dim=0)
padding_mask = torch.zeros(
@@ -93,9 +88,7 @@ def pad_sequences(sequences: List[torch.Tensor]) -> Tuple[torch.Tensor, torch.Te
return padded_sequences, padding_mask
-def patchify_batch(
- videos: List[torch.Tensor], patch_size: int
-) -> Tuple[torch.Tensor, torch.Tensor]:
+def patchify_batch(videos: List[torch.Tensor], patch_size: int) -> Tuple[torch.Tensor, torch.Tensor]:
"""Patchify a batch of videos.
Args:
@@ -134,10 +127,7 @@ def make_batch(samples: List[dict], video_dir: str) -> dict:
Returns:
dict: A batch of samples.
"""
- videos = [
- read_video(os.path.join(video_dir, sample["video_file"]), pts_unit="sec")[0]
- for sample in samples
- ]
+ videos = [read_video(os.path.join(video_dir, sample["video_file"]), pts_unit="sec")[0] for sample in samples]
texts = [sample["text_latent_states"] for sample in samples]
texts, text_padding_mask = pad_sequences(texts)
return {
@@ -156,9 +146,7 @@ def unnormalize_video(video: torch.Tensor) -> torch.Tensor:
@torch.no_grad()
-def preprocess_batch(
- batch: dict, patch_size: int, vqvae: Optional[nn.Module] = None, device=None
-) -> dict:
+def preprocess_batch(batch: dict, patch_size: int, vqvae: Optional[nn.Module] = None, device=None) -> dict:
if device is None:
device = get_current_device()
videos = []
@@ -186,18 +174,14 @@ def preprocess_batch(
return batch
-def load_datasets(
- dataset_paths: Union[PathType, List[PathType]], mode: str = "train"
-) -> Optional[DatasetType]:
+def load_datasets(dataset_paths: Union[PathType, List[PathType]], mode: str = "train") -> Optional[DatasetType]:
"""
Load pre-tokenized dataset.
Each instance of dataset is a dictionary with
`{'input_ids': List[int], 'labels': List[int], sequence: str}` format.
"""
mode_map = {"train": "train", "dev": "validation", "test": "test"}
- assert mode in tuple(
- mode_map
- ), f"Unsupported mode {mode}, it must be in {tuple(mode_map)}"
+ assert mode in tuple(mode_map), f"Unsupported mode {mode}, it must be in {tuple(mode_map)}"
if isinstance(dataset_paths, (str, os.PathLike)):
dataset_paths = [dataset_paths]
@@ -206,9 +190,7 @@ def load_datasets(
for ds_path in dataset_paths:
ds_path = os.path.abspath(ds_path)
assert os.path.exists(ds_path), f"Not existed file path {ds_path}"
- ds_dict = load_from_disk(
- dataset_path=ds_path, keep_in_memory=False
- ).with_format("torch")
+ ds_dict = load_from_disk(dataset_path=ds_path, keep_in_memory=False).with_format("torch")
if isinstance(ds_dict, HFDataset):
datasets.append(ds_dict)
else:
diff --git a/requirements.txt b/requirements.txt
index 638876b..6e8108d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,3 +4,4 @@ datasets
transformers
av
tensorboard
+colossalai
diff --git a/sample.py b/sample.py
index 176de3a..b297f88 100644
--- a/sample.py
+++ b/sample.py
@@ -17,9 +17,9 @@ from colossalai.utils import get_current_device
from torchvision.io import write_video
from transformers import AutoModel, AutoTokenizer, CLIPTextModel
-from data_utils import col2video
-from diffusion import create_diffusion
-from models import DiT_models
+from open_sora.diffusion import create_diffusion
+from open_sora.modeling import DiT_models
+from open_sora.utils.data import col2video
def main(args):
@@ -28,11 +28,7 @@ def main(args):
torch.set_grad_enabled(False)
device = get_current_device()
if len(args.vqvae) > 0:
- vqvae = (
- AutoModel.from_pretrained(args.vqvae, trust_remote_code=True)
- .to(device)
- .eval()
- )
+ vqvae = AutoModel.from_pretrained(args.vqvae, trust_remote_code=True).to(device).eval()
in_channels = vqvae.embedding_dim
else:
# disable VQ-VAE if not provided, just use raw video frames
@@ -54,9 +50,7 @@ def main(args):
num_frames = args.fps * args.sec
z = torch.randn(
1,
- (args.height // patch_size // 4)
- * (args.width // patch_size // 4)
- * (num_frames // 2),
+ (args.height // patch_size // 4) * (args.width // patch_size // 4) * (num_frames // 2),
in_channels,
patch_size,
patch_size,
@@ -66,9 +60,7 @@ def main(args):
# Setup classifier-free guidance:
model_kwargs = {}
z = torch.cat([z, z], 0)
- model_kwargs["text_latent_states"] = torch.cat(
- [text_latent_states, text_latent_states], 0
- )
+ model_kwargs["text_latent_states"] = torch.cat([text_latent_states, text_latent_states], 0)
model_kwargs["cfg_scale"] = args.cfg_scale
model_kwargs["attention_mask"] = torch.ones(
2, 1, z.shape[1], text_latent_states.shape[1], device=device, dtype=torch.int
@@ -104,9 +96,7 @@ def main(args):
if __name__ == "__main__":
parser = argparse.ArgumentParser()
- parser.add_argument(
- "--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8"
- )
+ parser.add_argument("--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8")
parser.add_argument(
"--text",
type=str,
@@ -122,9 +112,7 @@ if __name__ == "__main__":
help="Optional path to a DiT checkpoint (default: auto-download a pre-trained DiT-XL/2 model).",
)
parser.add_argument("--vqvae", default="hpcai-tech/vqvae")
- parser.add_argument(
- "--text_model", type=str, default="openai/clip-vit-base-patch32"
- )
+ parser.add_argument("--text_model", type=str, default="openai/clip-vit-base-patch32")
parser.add_argument("--width", type=int, default=480)
parser.add_argument("--height", type=int, default=320)
parser.add_argument("--fps", type=int, default=15)
diff --git a/scripts/train.sh b/scripts/train.sh
index bd02dbd..754fce7 100644
--- a/scripts/train.sh
+++ b/scripts/train.sh
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
# get args
-GPUS=${1:8}
+GPUS=${1:-8}
# get root dir
FOLDER_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
@@ -25,6 +25,9 @@ PROCESSED_DATASET=(
./dataset/MSRVTT-processed/val/part-00009
)
+# create timestamp to differentiate between runs
+timestamp=$(date +%Y-%m-%d-%H-%M)
+
# run single node training
torchrun --standalone \
--nproc_per_node $GPUS \
@@ -37,5 +40,5 @@ torchrun --standalone \
--dataset $PROCESSED_DATASET \
--video_dir $COLLATED_VIDEO_DIR \
--save_interval 224 \
- --checkpoint_dir ./checkpoints \
- --tensorboard_dir ./runs
+ --checkpoint_dir ./checkpoints/$timestamp \
+ --tensorboard_dir ./runs/$timestamp
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..582beef
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,82 @@
+import os
+from typing import List
+
+from setuptools import find_packages, setup
+
+
+def fetch_requirements(path) -> List[str]:
+ """
+ This function reads the requirements file.
+
+ Args:
+ path (str): the path to the requirements file.
+
+ Returns:
+ The lines in the requirements file.
+ """
+ with open(path, "r") as fd:
+ return [r.strip() for r in fd.readlines()]
+
+
+def fetch_readme() -> str:
+ """
+ This function reads the README.md file in the current directory.
+
+ Returns:
+ The lines in the README file.
+ """
+ with open("README.md", encoding="utf-8") as f:
+ return f.read()
+
+
+def get_version() -> str:
+ """
+ This function reads the version.txt and generates the colossalai/version.py file.
+
+ Returns:
+ The library version stored in version.txt.
+ """
+
+ setup_file_path = os.path.abspath(__file__)
+ project_path = os.path.dirname(setup_file_path)
+ version_txt_path = os.path.join(project_path, "version.txt")
+
+ with open(version_txt_path) as f:
+ version = f.read().strip()
+ return version
+
+
+setup(
+ name="open-sora",
+ version=get_version(),
+ packages=find_packages(
+ exclude=(
+ "docker",
+ "tests",
+ "docs",
+ "examples",
+ "tests",
+ "scripts",
+ "requirements",
+ "extensions",
+ "*.egg-info",
+ ),
+ ),
+ description="Unofficial implementation of OpenAI's Sora by the Colossal-AI Team",
+ long_description=fetch_readme(),
+ long_description_content_type="text/markdown",
+ license="Apache Software License 2.0",
+ url="https://www.colossalai.org",
+ project_urls={
+ "Github": "https://github.com/hpcaitech/Open-Sora",
+ },
+ install_requires=fetch_requirements("requirements.txt"),
+ python_requires=">=3.6",
+ classifiers=[
+ "Programming Language :: Python :: 3",
+ "License :: OSI Approved :: Apache Software License",
+ "Environment :: GPU :: NVIDIA CUDA",
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
+ "Topic :: System :: Distributed Computing",
+ ],
+)
diff --git a/train.py b/train.py
index 821ae8b..3a353c2 100644
--- a/train.py
+++ b/train.py
@@ -7,16 +7,12 @@
"""
A minimal training script for DiT using PyTorch DDP.
"""
-import torch
-
-# the first flag below was False when we tested this script but True makes A100 training a lot faster:
-torch.backends.cuda.matmul.allow_tf32 = True
-torch.backends.cudnn.allow_tf32 = True
import argparse
import os
from copy import deepcopy
from functools import partial
+import torch
import torch.distributed as dist
from colossalai import launch_from_torch
from colossalai.accelerator import get_accelerator
@@ -29,15 +25,21 @@ from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm
from transformers import AutoModel
-from data_utils import load_datasets, make_batch, preprocess_batch
-from diffusion import create_diffusion
-from models import DiT_models
+from open_sora.diffusion import create_diffusion
+from open_sora.modeling import DiT_models
+from open_sora.utils.data import load_datasets, make_batch, preprocess_batch
#################################################################################
# Training Helper Functions #
#################################################################################
+def configure_backends():
+ # the first flag below was False when we tested this script but True makes A100 training a lot faster:
+ torch.backends.cuda.matmul.allow_tf32 = True
+ torch.backends.cudnn.allow_tf32 = True
+
+
@torch.no_grad()
def update_ema(ema_model, model, decay=0.9999):
"""
@@ -71,9 +73,12 @@ def main(args):
"""
Trains a new DiT model.
"""
+ # Step 1: init distributed environment
launch_from_torch({})
coordinator = DistCoordinator()
logger = get_dist_logger()
+
+ # Step 2: set up acceleration plugins
plugin = LowLevelZeroPlugin(stage=2, precision="fp16")
booster = Booster(plugin=plugin)
@@ -82,35 +87,35 @@ def main(args):
os.makedirs(args.tensorboard_dir, exist_ok=True)
writer = SummaryWriter(args.tensorboard_dir)
- # Setup model
+ # Step 3: Create VQ-VAE
if len(args.vqvae) > 0:
- vqvae = (
- AutoModel.from_pretrained(args.vqvae, trust_remote_code=True)
- .to(get_current_device())
- .eval()
- )
+ vqvae = AutoModel.from_pretrained(args.vqvae, trust_remote_code=True).to(get_current_device()).eval()
model_kwargs = {"in_channels": vqvae.embedding_dim}
else:
# disable VQ-VAE if not provided, just use raw video frames
vqvae = None
model_kwargs = {}
+
+ # Step 4: Create DiT and EMA
model = DiT_models[args.model](**model_kwargs).to(get_current_device())
patch_size = model.patch_size
ema = deepcopy(model)
requires_grad(ema, False)
+
model.train() # important! This enables embedding dropout for classifier-free guidance
ema.eval() # EMA model should always be in eval mode
+
+ # configure gradient checkpointing
if args.grad_checkpoint:
model.enable_gradient_checkpointing()
- diffusion = create_diffusion(
- timestep_respacing=""
- ) # default: 1000 steps, linear noise schedule
+ # Step 5: create diffusion pipeline
+ diffusion = create_diffusion(timestep_respacing="") # default: 1000 steps, linear noise schedule
- # Setup optimizer (we used default Adam betas=(0.9, 0.999) and a constant learning rate of 1e-4 in our paper):
+ # Step 6: setup optimizer (we used default Adam betas=(0.9, 0.999) and a constant learning rate of 1e-4 in our paper):
opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=0)
- # Setup dataloader
+ # Step 7: Setup dataloader
dataset = load_datasets(args.dataset)
dataloader = plugin.prepare_dataloader(
dataset,
@@ -121,13 +126,14 @@ def main(args):
)
logger.info(f"Dataset contains {len(dataset)} samples", ranks=[0])
- # Setup booster
+ # Step 8: setup booster
model, opt, _, dataloader, _ = booster.boost(model, opt, dataloader=dataloader)
logger.info(
f"Booster init max device memory: {get_accelerator().max_memory_allocated() / 1024 ** 2:.2f} MB",
ranks=[0],
)
+ # Step 9: Train
num_steps_per_epoch = len(dataloader) // args.accumulation_steps
for epoch in range(args.epochs):
@@ -148,15 +154,12 @@ def main(args):
(video_inputs.shape[0],),
device=video_inputs.device,
)
- loss_dict = diffusion.training_losses(
- model, video_inputs, t, batch, mask=mask
- )
+ loss_dict = diffusion.training_losses(model, video_inputs, t, batch, mask=mask)
loss = loss_dict["loss"].mean() / args.accumulation_steps
total_loss.add_(loss.data)
booster.backward(loss, opt)
if (step + 1) % args.accumulation_steps == 0:
-
opt.step()
opt.zero_grad()
update_ema(ema, model)
@@ -164,9 +167,7 @@ def main(args):
all_reduce_mean(total_loss)
pbar.set_postfix({"Loss": f"{total_loss.item():.4f}"})
if coordinator.is_master():
- global_step = (epoch * num_steps_per_epoch) + (
- step + 1
- ) // args.accumulation_steps
+ global_step = (epoch * num_steps_per_epoch) + (step + 1) // args.accumulation_steps
writer.add_scalar(
tag="Loss",
scalar_value=total_loss.item(),
@@ -176,20 +177,13 @@ def main(args):
total_loss.zero_()
# Save DiT checkpoint:
- if (
- args.save_interval > 0
- and (step + 1) % (args.save_interval * args.accumulation_steps) == 0
- ) or (step + 1) == len(dataloader):
- save_path = os.path.join(
- args.checkpoint_dir, f"epoch-{epoch}-step-{step}"
- )
+ if (args.save_interval > 0 and (step + 1) % (args.save_interval * args.accumulation_steps) == 0) or (
+ step + 1
+ ) == len(dataloader):
+ save_path = os.path.join(args.checkpoint_dir, f"epoch-{epoch}-step-{step}")
os.makedirs(save_path, exist_ok=True)
- booster.save_model(
- model, os.path.join(save_path, "model"), shard=True
- )
- booster.save_optimizer(
- opt, os.path.join(save_path, "optimizer"), shard=True
- )
+ booster.save_model(model, os.path.join(save_path, "model"), shard=True)
+ booster.save_optimizer(opt, os.path.join(save_path, "optimizer"), shard=True)
if coordinator.is_master():
ema_state_dict = ema.state_dict()
for k, v in ema_state_dict.items():
@@ -208,9 +202,7 @@ def main(args):
if __name__ == "__main__":
# Default args here will train DiT-XL/2 with the hyperparameters we used in our paper (except training iters).
parser = argparse.ArgumentParser()
- parser.add_argument(
- "-m", "--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8"
- )
+ parser.add_argument("-m", "--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8")
parser.add_argument("-d", "--dataset", nargs="+", default=[])
parser.add_argument("-v", "--video_dir", type=str, required=True)
parser.add_argument("-e", "--epochs", type=int, default=10)
diff --git a/version.txt b/version.txt
new file mode 100644
index 0000000..8acdd82
--- /dev/null
+++ b/version.txt
@@ -0,0 +1 @@
+0.0.1