Open-Sora: Democratizing Efficient Video Production for All
Python
refactored code into a package (#13)
Frank Lee0c05cd2e9d6c
diff --git a/open_sora/__init__.py b/open_sora/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/diffusion/__init__.py b/open_sora/diffusion/__init__.pysimilarity index 100%rename from diffusion/__init__.pyrename to open_sora/diffusion/__init__.pydiff --git a/diffusion/diffusion_utils.py b/open_sora/diffusion/diffusion_utils.pysimilarity index 100%rename from diffusion/diffusion_utils.pyrename to open_sora/diffusion/diffusion_utils.pydiff --git a/diffusion/gaussian_diffusion.py b/open_sora/diffusion/gaussian_diffusion.pysimilarity index 89%rename from diffusion/gaussian_diffusion.pyrename to open_sora/diffusion/gaussian_diffusion.pyindex 6c0b0de..fe94622 100644--- a/diffusion/gaussian_diffusion.py+++ b/open_sora/diffusion/gaussian_diffusion.py@@ -47,9 +47,7 @@ class ModelVarType(enum.Enum):class LossType(enum.Enum):MSE = enum.auto() # use raw MSE loss (and KL when learning variances)- RESCALED_MSE = (- enum.auto()- ) # use raw MSE loss (with RESCALED_KL when learning variances)+ RESCALED_MSE = enum.auto() # use raw MSE loss (with RESCALED_KL when learning variances)KL = enum.auto() # use the variational lower-boundRESCALED_KL = enum.auto() # like KL, but rescale to estimate the full VLB@@ -60,9 +58,7 @@ class LossType(enum.Enum):def _warmup_beta(beta_start, beta_end, num_diffusion_timesteps, warmup_frac):betas = beta_end * np.ones(num_diffusion_timesteps, dtype=np.float64)warmup_time = int(num_diffusion_timesteps * warmup_frac)- betas[:warmup_time] = np.linspace(- beta_start, beta_end, warmup_time, dtype=np.float64- )+ betas[:warmup_time] = np.linspace(beta_start, beta_end, warmup_time, dtype=np.float64)return betas@@ -82,9 +78,7 @@ def get_beta_schedule(beta_schedule, *, beta_start, beta_end, num_diffusion_time** 2)elif beta_schedule == "linear":- betas = np.linspace(- beta_start, beta_end, num_diffusion_timesteps, dtype=np.float64- )+ betas = np.linspace(beta_start, beta_end, num_diffusion_timesteps, dtype=np.float64)elif beta_schedule == "warmup10":betas = _warmup_beta(beta_start, beta_end, num_diffusion_timesteps, 0.1)elif beta_schedule == "warmup50":@@ -92,9 +86,7 @@ def get_beta_schedule(beta_schedule, *, beta_start, beta_end, num_diffusion_timeelif beta_schedule == "const":betas = beta_end * np.ones(num_diffusion_timesteps, dtype=np.float64)elif beta_schedule == "jsd": # 1/T, 1/(T-1), 1/(T-2), ..., 1- betas = 1.0 / np.linspace(- num_diffusion_timesteps, 1, num_diffusion_timesteps, dtype=np.float64- )+ betas = 1.0 / np.linspace(num_diffusion_timesteps, 1, num_diffusion_timesteps, dtype=np.float64)else:raise NotImplementedError(beta_schedule)assert betas.shape == (num_diffusion_timesteps,)@@ -183,9 +175,7 @@ class GaussianDiffusion:self.sqrt_recipm1_alphas_cumprod = np.sqrt(1.0 / self.alphas_cumprod - 1)# calculations for posterior q(x_{t-1} | x_t, x_0)- self.posterior_variance = (- betas * (1.0 - self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod)- )+ self.posterior_variance = betas * (1.0 - self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod)# below: log calculation clipped because the posterior variance is 0 at the beginning of the diffusion chainself.posterior_log_variance_clipped = (np.log(np.append(self.posterior_variance[1], self.posterior_variance[1:]))@@ -193,14 +183,8 @@ class GaussianDiffusion:else np.array([]))- self.posterior_mean_coef1 = (- betas * np.sqrt(self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod)- )- self.posterior_mean_coef2 = (- (1.0 - self.alphas_cumprod_prev)- * np.sqrt(alphas)- / (1.0 - self.alphas_cumprod)- )+ self.posterior_mean_coef1 = betas * np.sqrt(self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod)+ self.posterior_mean_coef2 = (1.0 - self.alphas_cumprod_prev) * np.sqrt(alphas) / (1.0 - self.alphas_cumprod)def q_mean_variance(self, x_start, t):"""@@ -209,13 +193,9 @@ class GaussianDiffusion::param t: the number of diffusion steps (minus 1). Here, 0 means one step.:return: A tuple (mean, variance, log_variance), all of x_start's shape."""- mean = (- _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start- )+ mean = _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_startvariance = _extract_into_tensor(1.0 - self.alphas_cumprod, t, x_start.shape)- log_variance = _extract_into_tensor(- self.log_one_minus_alphas_cumprod, t, x_start.shape- )+ log_variance = _extract_into_tensor(self.log_one_minus_alphas_cumprod, t, x_start.shape)return mean, variance, log_variancedef q_sample(self, x_start, t, noise=None):@@ -232,8 +212,7 @@ class GaussianDiffusion:assert noise.shape == x_start.shapereturn (_extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start- + _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape)- * noise+ + _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise)def q_posterior_mean_variance(self, x_start, x_t, t):@@ -247,9 +226,7 @@ class GaussianDiffusion:+ _extract_into_tensor(self.posterior_mean_coef2, t, x_t.shape) * x_t)posterior_variance = _extract_into_tensor(self.posterior_variance, t, x_t.shape)- posterior_log_variance_clipped = _extract_into_tensor(- self.posterior_log_variance_clipped, t, x_t.shape- )+ posterior_log_variance_clipped = _extract_into_tensor(self.posterior_log_variance_clipped, t, x_t.shape)assert (posterior_mean.shape[0]== posterior_variance.shape[0]@@ -258,9 +235,7 @@ class GaussianDiffusion:)return posterior_mean, posterior_variance, posterior_log_variance_clipped- def p_mean_variance(- self, model, x, t, clip_denoised=True, denoised_fn=None, model_kwargs=None- ):+ def p_mean_variance(self, model, x, t, clip_denoised=True, denoised_fn=None, model_kwargs=None):"""Apply the model to get p(x_{t-1} | x_t), as well as a prediction ofthe initial x, x_0.@@ -294,9 +269,7 @@ class GaussianDiffusion:if self.model_var_type in [ModelVarType.LEARNED, ModelVarType.LEARNED_RANGE]:assert model_output.shape == (B, S, C * 2, *x.shape[3:])model_output, model_var_values = th.split(model_output, C, dim=2)- min_log = _extract_into_tensor(- self.posterior_log_variance_clipped, t, x.shape- )+ min_log = _extract_into_tensor(self.posterior_log_variance_clipped, t, x.shape)max_log = _extract_into_tensor(np.log(self.betas), t, x.shape)# The model_var_values is [-1, 1] for [min_var, max_var].frac = (model_var_values + 1) / 2@@ -328,16 +301,10 @@ class GaussianDiffusion:if self.model_mean_type == ModelMeanType.START_X:pred_xstart = process_xstart(model_output)else:- pred_xstart = process_xstart(- self._predict_xstart_from_eps(x_t=x, t=t, eps=model_output)- )- model_mean, _, _ = self.q_posterior_mean_variance(- x_start=pred_xstart, x_t=x, t=t- )+ pred_xstart = process_xstart(self._predict_xstart_from_eps(x_t=x, t=t, eps=model_output))+ model_mean, _, _ = self.q_posterior_mean_variance(x_start=pred_xstart, x_t=x, t=t)- assert (- model_mean.shape == model_log_variance.shape == pred_xstart.shape == x.shape- )+ assert model_mean.shape == model_log_variance.shape == pred_xstart.shape == x.shapereturn {"mean": model_mean,"variance": model_variance,@@ -355,8 +322,7 @@ class GaussianDiffusion:def _predict_eps_from_xstart(self, x_t, t, pred_xstart):return (- _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t- - pred_xstart+ _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - pred_xstart) / _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape)def condition_mean(self, cond_fn, p_mean_var, x, t, model_kwargs=None):@@ -368,9 +334,7 @@ class GaussianDiffusion:This uses the conditioning strategy from Sohl-Dickstein et al. (2015)."""gradient = cond_fn(x, t, **model_kwargs)- new_mean = (- p_mean_var["mean"].float() + p_mean_var["variance"] * gradient.float()- )+ new_mean = p_mean_var["mean"].float() + p_mean_var["variance"] * gradient.float()return new_meandef condition_score(self, cond_fn, p_mean_var, x, t, model_kwargs=None):@@ -388,9 +352,7 @@ class GaussianDiffusion:out = p_mean_var.copy()out["pred_xstart"] = self._predict_xstart_from_eps(x, t, eps)- out["mean"], _, _ = self.q_posterior_mean_variance(- x_start=out["pred_xstart"], x_t=x, t=t- )+ out["mean"], _, _ = self.q_posterior_mean_variance(x_start=out["pred_xstart"], x_t=x, t=t)return outdef p_sample(@@ -428,13 +390,9 @@ class GaussianDiffusion:model_kwargs=model_kwargs,)noise = th.randn_like(x)- nonzero_mask = (- (t != 0).float().view(-1, *([1] * (len(x.shape) - 1)))- ) # no noise when t == 0+ nonzero_mask = (t != 0).float().view(-1, *([1] * (len(x.shape) - 1))) # no noise when t == 0if cond_fn is not None:- out["mean"] = self.condition_mean(- cond_fn, out, x, t, model_kwargs=model_kwargs- )+ out["mean"] = self.condition_mean(cond_fn, out, x, t, model_kwargs=model_kwargs)sample = out["mean"] + nonzero_mask * th.exp(0.5 * out["log_variance"]) * noisereturn {"sample": sample, "pred_xstart": out["pred_xstart"]}@@ -564,20 +522,11 @@ class GaussianDiffusion:alpha_bar = _extract_into_tensor(self.alphas_cumprod, t, x.shape)alpha_bar_prev = _extract_into_tensor(self.alphas_cumprod_prev, t, x.shape)- sigma = (- eta- * th.sqrt((1 - alpha_bar_prev) / (1 - alpha_bar))- * th.sqrt(1 - alpha_bar / alpha_bar_prev)- )+ sigma = eta * th.sqrt((1 - alpha_bar_prev) / (1 - alpha_bar)) * th.sqrt(1 - alpha_bar / alpha_bar_prev)# Equation 12.noise = th.randn_like(x)- mean_pred = (- out["pred_xstart"] * th.sqrt(alpha_bar_prev)- + th.sqrt(1 - alpha_bar_prev - sigma**2) * eps- )- nonzero_mask = (- (t != 0).float().view(-1, *([1] * (len(x.shape) - 1)))- ) # no noise when t == 0+ mean_pred = out["pred_xstart"] * th.sqrt(alpha_bar_prev) + th.sqrt(1 - alpha_bar_prev - sigma**2) * eps+ nonzero_mask = (t != 0).float().view(-1, *([1] * (len(x.shape) - 1))) # no noise when t == 0sample = mean_pred + nonzero_mask * sigma * noisereturn {"sample": sample, "pred_xstart": out["pred_xstart"]}@@ -609,16 +558,12 @@ class GaussianDiffusion:# Usually our model outputs epsilon, but we re-derive it# in case we used x_start or x_prev prediction.eps = (- _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x.shape) * x- - out["pred_xstart"]+ _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x.shape) * x - out["pred_xstart"]) / _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x.shape)alpha_bar_next = _extract_into_tensor(self.alphas_cumprod_next, t, x.shape)# Equation 12. reversed- mean_pred = (- out["pred_xstart"] * th.sqrt(alpha_bar_next)- + th.sqrt(1 - alpha_bar_next) * eps- )+ mean_pred = out["pred_xstart"] * th.sqrt(alpha_bar_next) + th.sqrt(1 - alpha_bar_next) * epsreturn {"sample": mean_pred, "pred_xstart": out["pred_xstart"]}@@ -704,9 +649,7 @@ class GaussianDiffusion:yield outimg = out["sample"]- def _vb_terms_bpd(- self, model, x_start, x_t, t, clip_denoised=True, model_kwargs=None, mask=None- ):+ def _vb_terms_bpd(self, model, x_start, x_t, t, clip_denoised=True, model_kwargs=None, mask=None):"""Get a term for the variational lower-bound.The resulting units are bits (rather than nats, as one might expect).@@ -715,15 +658,9 @@ class GaussianDiffusion:- 'output': a shape [N] tensor of NLLs or KLs.- 'pred_xstart': the x_0 predictions."""- true_mean, _, true_log_variance_clipped = self.q_posterior_mean_variance(- x_start=x_start, x_t=x_t, t=t- )- out = self.p_mean_variance(- model, x_t, t, clip_denoised=clip_denoised, model_kwargs=model_kwargs- )- kl = normal_kl(- true_mean, true_log_variance_clipped, out["mean"], out["log_variance"]- )+ true_mean, _, true_log_variance_clipped = self.q_posterior_mean_variance(x_start=x_start, x_t=x_t, t=t)+ out = self.p_mean_variance(model, x_t, t, clip_denoised=clip_denoised, model_kwargs=model_kwargs)+ kl = normal_kl(true_mean, true_log_variance_clipped, out["mean"], out["log_variance"])kl = mean_flat(kl, mask=mask) / np.log(2.0)decoder_nll = -discretized_gaussian_log_likelihood(@@ -743,9 +680,7 @@ class GaussianDiffusion:mask = mask.view(*mask.shape, *([1] * (ndim - mask.ndim)))return mask- def training_losses(- self, model, x_start, t, model_kwargs=None, noise=None, mask=None- ):+ def training_losses(self, model, x_start, t, model_kwargs=None, noise=None, mask=None):"""Compute training losses for a single timestep.:param model: the model to evaluate loss on.@@ -807,9 +742,7 @@ class GaussianDiffusion:terms["vb"] *= self.num_timesteps / 1000.0target = {- ModelMeanType.PREVIOUS_X: self.q_posterior_mean_variance(- x_start=x_start, x_t=x_t, t=t- )[0],+ ModelMeanType.PREVIOUS_X: self.q_posterior_mean_variance(x_start=x_start, x_t=x_t, t=t)[0],ModelMeanType.START_X: x_start,ModelMeanType.EPSILON: noise,}[self.model_mean_type]@@ -835,9 +768,7 @@ class GaussianDiffusion:batch_size = x_start.shape[0]t = th.tensor([self.num_timesteps - 1] * batch_size, device=x_start.device)qt_mean, _, qt_log_variance = self.q_mean_variance(x_start, t)- kl_prior = normal_kl(- mean1=qt_mean, logvar1=qt_log_variance, mean2=0.0, logvar2=0.0- )+ kl_prior = normal_kl(mean1=qt_mean, logvar1=qt_log_variance, mean2=0.0, logvar2=0.0)return mean_flat(kl_prior) / np.log(2.0)def calc_bpd_loop(self, model, x_start, clip_denoised=True, model_kwargs=None):diff --git a/diffusion/respace.py b/open_sora/diffusion/respace.pysimilarity index 100%rename from diffusion/respace.pyrename to open_sora/diffusion/respace.pydiff --git a/diffusion/timestep_sampler.py b/open_sora/diffusion/timestep_sampler.pysimilarity index 100%rename from diffusion/timestep_sampler.pyrename to open_sora/diffusion/timestep_sampler.pydiff --git a/open_sora/modeling/__init__.py b/open_sora/modeling/__init__.pynew file mode 100644index 0000000..72ac366--- /dev/null+++ b/open_sora/modeling/__init__.py@@ -0,0 +1,3 @@+from .dit import DiT, DiT_models++__all__ = ["DiT_models", "DiT"]diff --git a/models.py b/open_sora/modeling/dit.pysimilarity index 87%rename from models.pyrename to open_sora/modeling/dit.pyindex 45dea99..0ec1736 100644--- a/models.py+++ b/open_sora/modeling/dit.py@@ -46,9 +46,7 @@ class CrossAttention(nn.Module):):super().__init__()self.hidden_size = head_dim * num_heads- cross_attention_dim = (- cross_attention_dim if cross_attention_dim is not None else query_dim- )+ cross_attention_dim = cross_attention_dim if cross_attention_dim is not None else query_dimself.scale = head_dim**-0.5self.num_heads = num_heads@@ -59,9 +57,7 @@ class CrossAttention(nn.Module):self.to_k = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)self.to_v = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)- self.to_out = nn.Sequential(- nn.Linear(self.hidden_size, query_dim), nn.Dropout(dropout)- )+ self.to_out = nn.Sequential(nn.Linear(self.hidden_size, query_dim), nn.Dropout(dropout))def forward(self, hidden_states, context=None, mask=None):bsz, q_len, _ = hidden_states.shape@@ -75,24 +71,18 @@ class CrossAttention(nn.Module):# [B, S, H, D]query = query.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)key = key.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(1, 2)- value = value.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(- 1, 2- )+ value = value.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(1, 2)if mask is not None:assert mask.shape == (bsz, 1, q_len, kv_seq_len)if self.sdpa:- attn_output = F.scaled_dot_product_attention(- query, key, value, attn_mask=mask, scale=self.scale- )+ attn_output = F.scaled_dot_product_attention(query, key, value, attn_mask=mask, scale=self.scale)else:attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scaleassert attn_weights.shape == (bsz, self.num_heads, q_len, kv_seq_len)if mask is not None:attn_weights = attn_weights + mask- attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(- query.dtype- )+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)attn_output = torch.matmul(attn_weights, value)assert attn_output.shape == (bsz, self.num_heads, q_len, self.head_dim)attn_output = attn_output.transpose(1, 2).contiguous()@@ -136,23 +126,17 @@ class TimestepEmbedder(nn.Module):"""# https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.pyhalf = dim // 2- freqs = torch.exp(- -math.log(max_period)- * torch.arange(start=0, end=half, dtype=torch.float32)- / half- ).to(device=t.device)+ freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to(+ device=t.device+ )args = t[:, None].float() * freqs[None]embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)if dim % 2:- embedding = torch.cat(- [embedding, torch.zeros_like(embedding[:, :1])], dim=-1- )+ embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)return embeddingdef forward(self, t):- t_freq = self.timestep_embedding(t, self.frequency_embedding_size).to(- self.mlp[0].weight.dtype- )+ t_freq = self.timestep_embedding(t, self.frequency_embedding_size).to(self.mlp[0].weight.dtype)t_emb = self.mlp(t_freq)return t_emb@@ -165,9 +149,7 @@ class LabelEmbedder(nn.Module):def __init__(self, num_classes, hidden_size, dropout_prob):super().__init__()use_cfg_embedding = dropout_prob > 0- self.embedding_table = nn.Embedding(- num_classes + use_cfg_embedding, hidden_size- )+ self.embedding_table = nn.Embedding(num_classes + use_cfg_embedding, hidden_size)self.num_classes = num_classesself.dropout_prob = dropout_prob@@ -176,9 +158,7 @@ class LabelEmbedder(nn.Module):Drops labels to enable classifier-free guidance."""if force_drop_ids is None:- drop_ids = (- torch.rand(labels.shape[0], device=labels.device) < self.dropout_prob- )+ drop_ids = torch.rand(labels.shape[0], device=labels.device) < self.dropout_probelse:drop_ids = force_drop_ids == 1labels = torch.where(drop_ids, self.num_classes, labels)@@ -205,27 +185,21 @@ class PatchEmbedder(nn.Module):) -> None:super().__init__()self.patch_size = patch_size- self.proj = nn.Conv2d(- in_chans, embed_dim, kernel_size=patch_size, stride=patch_size, bias=bias- )+ self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size, bias=bias)self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()def forward(self, x: torch.Tensor) -> torch.Tensor:# [B, S, C, P, P] -> [B, S, C*P*P]# FIXME: hack diffusion and use viewx = x.view(*x.shape[:2], -1)- out = F.linear(- x, self.proj.weight.view(self.proj.weight.shape[0], -1), self.proj.bias- )+ out = F.linear(x, self.proj.weight.view(self.proj.weight.shape[0], -1), self.proj.bias)out = self.norm(out)# [B, S, H]return outclass TextEmbedder(nn.Module):- def __init__(- self, in_features: int, embed_dim: int = 768, bias: bool = True- ) -> None:+ def __init__(self, in_features: int, embed_dim: int = 768, bias: bool = True) -> None:super().__init__()self.proj = nn.Linear(in_features, embed_dim, bias=bias)@@ -267,9 +241,7 @@ class DiTBlock(nn.Module):A DiT block with adaptive layer norm zero (adaLN-Zero) conditioning."""- def __init__(- self, hidden_size, num_heads, cross_attention_dim, mlp_ratio=4.0, **block_kwargs- ):+ def __init__(self, hidden_size, num_heads, cross_attention_dim, mlp_ratio=4.0, **block_kwargs):super().__init__()self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)self.attn = CrossAttention(@@ -305,9 +277,7 @@ class FinalLayer(nn.Module):def __init__(self, hidden_size, patch_size, out_channels):super().__init__()- self.linear = nn.Linear(- hidden_size, patch_size * patch_size * out_channels, bias=True- )+ self.linear = nn.Linear(hidden_size, patch_size * patch_size * out_channels, bias=True)self.patch_size = patch_sizedef unpatchify(self, x):@@ -345,17 +315,12 @@ class DiT(nn.Module):self.patch_size = patch_sizeself.num_heads = num_heads- self.video_embedder = PatchEmbedder(- patch_size, in_channels, hidden_size, bias=True- )+ self.video_embedder = PatchEmbedder(patch_size, in_channels, hidden_size, bias=True)self.t_embedder = TimestepEmbedder(text_embed_dim)self.pos_embed = PositionEmbedding(hidden_size, max_num_embeddings)self.blocks = nn.ModuleList(- [- DiTBlock(hidden_size, num_heads, text_embed_dim, mlp_ratio=mlp_ratio)- for _ in range(depth)- ]+ [DiTBlock(hidden_size, num_heads, text_embed_dim, mlp_ratio=mlp_ratio) for _ in range(depth)])self.final_layer = FinalLayer(hidden_size, patch_size, self.out_channels)self.initialize_weights()@@ -384,9 +349,7 @@ class DiT(nn.Module):assert attention_mask.ndim == 4attention_mask = attention_mask.to(dtype)inverted_mask = 1.0 - attention_mask- return inverted_mask.masked_fill(- inverted_mask.to(torch.bool), torch.finfo(dtype).min- )+ return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)return attention_maskdef enable_gradient_checkpointing(self):@@ -417,24 +380,18 @@ class DiT(nn.Module):block, video_latent_states, text_latent_states, attention_mask)else:- video_latent_states = block(- video_latent_states, text_latent_states, attention_mask- )+ video_latent_states = block(video_latent_states, text_latent_states, attention_mask)video_latent_states = self.final_layer(video_latent_states)return video_latent_states- def forward_with_cfg(- self, x, t, text_latent_states, cfg_scale, attention_mask=None- ):+ def forward_with_cfg(self, x, t, text_latent_states, cfg_scale, attention_mask=None):"""Forward pass of DiT, but also batches the unconditional forward pass for classifier-free guidance."""# https://github.com/openai/glide-text2im/blob/main/notebooks/text2im.ipynbhalf = x[: len(x) // 2]combined = torch.cat([half, half], dim=0)- model_out = self.forward(- combined, t, text_latent_states, attention_mask=attention_mask- )+ model_out = self.forward(combined, t, text_latent_states, attention_mask=attention_mask)# For exact reproducibility reasons, we apply classifier-free guidance on only# three channels by default. The standard approach to cfg applies it to all channels.# This can be done by uncommenting the following line and commenting-out the line following that.@@ -468,9 +425,7 @@ def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=grid = grid.reshape([2, 1, grid_size, grid_size])pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)if cls_token and extra_tokens > 0:- pos_embed = np.concatenate(- [np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0- )+ pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0)return pos_embeddiff --git a/open_sora/utils/__init__.py b/open_sora/utils/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/data_utils.py b/open_sora/utils/data.pysimilarity index 87%rename from data_utils.pyrename to open_sora/utils/data.pyindex 35f67aa..8ef6ca4 100644--- a/data_utils.py+++ b/open_sora/utils/data.py@@ -37,9 +37,7 @@ def video2col(video_4d: torch.Tensor, patch_size: int) -> torch.Tensor:return torch.stack(out, dim=1).view(-1, c, patch_size, patch_size)-def col2video(- patches: torch.Tensor, video_shape: Tuple[int, int, int, int]-) -> torch.Tensor:+def col2video(patches: torch.Tensor, video_shape: Tuple[int, int, int, int]) -> torch.Tensor:"""Convert a 2D tensor of patches to a 4D video tensor.@@ -76,10 +74,7 @@ def pad_sequences(sequences: List[torch.Tensor]) -> Tuple[torch.Tensor, torch.Te"""max_len = max([sequence.shape[0] for sequence in sequences])padded_sequences = [- F.pad(- sequence, [0] * (sequence.ndim - 1) * 2 + [0, max_len - sequence.shape[0]]- )- for sequence in sequences+ F.pad(sequence, [0] * (sequence.ndim - 1) * 2 + [0, max_len - sequence.shape[0]]) for sequence in sequences]padded_sequences = torch.stack(padded_sequences, dim=0)padding_mask = torch.zeros(@@ -93,9 +88,7 @@ def pad_sequences(sequences: List[torch.Tensor]) -> Tuple[torch.Tensor, torch.Tereturn padded_sequences, padding_mask-def patchify_batch(- videos: List[torch.Tensor], patch_size: int-) -> Tuple[torch.Tensor, torch.Tensor]:+def patchify_batch(videos: List[torch.Tensor], patch_size: int) -> Tuple[torch.Tensor, torch.Tensor]:"""Patchify a batch of videos.Args:@@ -134,10 +127,7 @@ def make_batch(samples: List[dict], video_dir: str) -> dict:Returns:dict: A batch of samples."""- videos = [- read_video(os.path.join(video_dir, sample["video_file"]), pts_unit="sec")[0]- for sample in samples- ]+ videos = [read_video(os.path.join(video_dir, sample["video_file"]), pts_unit="sec")[0] for sample in samples]texts = [sample["text_latent_states"] for sample in samples]texts, text_padding_mask = pad_sequences(texts)return {@@ -156,9 +146,7 @@ def unnormalize_video(video: torch.Tensor) -> torch.Tensor:@torch.no_grad()-def preprocess_batch(- batch: dict, patch_size: int, vqvae: Optional[nn.Module] = None, device=None-) -> dict:+def preprocess_batch(batch: dict, patch_size: int, vqvae: Optional[nn.Module] = None, device=None) -> dict:if device is None:device = get_current_device()videos = []@@ -186,18 +174,14 @@ def preprocess_batch(return batch-def load_datasets(- dataset_paths: Union[PathType, List[PathType]], mode: str = "train"-) -> Optional[DatasetType]:+def load_datasets(dataset_paths: Union[PathType, List[PathType]], mode: str = "train") -> Optional[DatasetType]:"""Load pre-tokenized dataset.Each instance of dataset is a dictionary with`{'input_ids': List[int], 'labels': List[int], sequence: str}` format."""mode_map = {"train": "train", "dev": "validation", "test": "test"}- assert mode in tuple(- mode_map- ), f"Unsupported mode {mode}, it must be in {tuple(mode_map)}"+ assert mode in tuple(mode_map), f"Unsupported mode {mode}, it must be in {tuple(mode_map)}"if isinstance(dataset_paths, (str, os.PathLike)):dataset_paths = [dataset_paths]@@ -206,9 +190,7 @@ def load_datasets(for ds_path in dataset_paths:ds_path = os.path.abspath(ds_path)assert os.path.exists(ds_path), f"Not existed file path {ds_path}"- ds_dict = load_from_disk(- dataset_path=ds_path, keep_in_memory=False- ).with_format("torch")+ ds_dict = load_from_disk(dataset_path=ds_path, keep_in_memory=False).with_format("torch")if isinstance(ds_dict, HFDataset):datasets.append(ds_dict)else:diff --git a/requirements.txt b/requirements.txtindex 638876b..6e8108d 100644--- a/requirements.txt+++ b/requirements.txt@@ -4,3 +4,4 @@ datasetstransformersavtensorboard+colossalaidiff --git a/sample.py b/sample.pyindex 176de3a..b297f88 100644--- a/sample.py+++ b/sample.py@@ -17,9 +17,9 @@ from colossalai.utils import get_current_devicefrom torchvision.io import write_videofrom transformers import AutoModel, AutoTokenizer, CLIPTextModel-from data_utils import col2video-from diffusion import create_diffusion-from models import DiT_models+from open_sora.diffusion import create_diffusion+from open_sora.modeling import DiT_models+from open_sora.utils.data import col2videodef main(args):@@ -28,11 +28,7 @@ def main(args):torch.set_grad_enabled(False)device = get_current_device()if len(args.vqvae) > 0:- vqvae = (- AutoModel.from_pretrained(args.vqvae, trust_remote_code=True)- .to(device)- .eval()- )+ vqvae = AutoModel.from_pretrained(args.vqvae, trust_remote_code=True).to(device).eval()in_channels = vqvae.embedding_dimelse:# disable VQ-VAE if not provided, just use raw video frames@@ -54,9 +50,7 @@ def main(args):num_frames = args.fps * args.secz = torch.randn(1,- (args.height // patch_size // 4)- * (args.width // patch_size // 4)- * (num_frames // 2),+ (args.height // patch_size // 4) * (args.width // patch_size // 4) * (num_frames // 2),in_channels,patch_size,patch_size,@@ -66,9 +60,7 @@ def main(args):# Setup classifier-free guidance:model_kwargs = {}z = torch.cat([z, z], 0)- model_kwargs["text_latent_states"] = torch.cat(- [text_latent_states, text_latent_states], 0- )+ model_kwargs["text_latent_states"] = torch.cat([text_latent_states, text_latent_states], 0)model_kwargs["cfg_scale"] = args.cfg_scalemodel_kwargs["attention_mask"] = torch.ones(2, 1, z.shape[1], text_latent_states.shape[1], device=device, dtype=torch.int@@ -104,9 +96,7 @@ def main(args):if __name__ == "__main__":parser = argparse.ArgumentParser()- parser.add_argument(- "--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8"- )+ parser.add_argument("--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8")parser.add_argument("--text",type=str,@@ -122,9 +112,7 @@ if __name__ == "__main__":help="Optional path to a DiT checkpoint (default: auto-download a pre-trained DiT-XL/2 model).",)parser.add_argument("--vqvae", default="hpcai-tech/vqvae")- parser.add_argument(- "--text_model", type=str, default="openai/clip-vit-base-patch32"- )+ parser.add_argument("--text_model", type=str, default="openai/clip-vit-base-patch32")parser.add_argument("--width", type=int, default=480)parser.add_argument("--height", type=int, default=320)parser.add_argument("--fps", type=int, default=15)diff --git a/scripts/train.sh b/scripts/train.shindex bd02dbd..754fce7 100644--- a/scripts/train.sh+++ b/scripts/train.sh@@ -1,7 +1,7 @@#!/usr/bin/env bash# get args-GPUS=${1:8}+GPUS=${1:-8}# get root dirFOLDER_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"@@ -25,6 +25,9 @@ PROCESSED_DATASET=(./dataset/MSRVTT-processed/val/part-00009)+# create timestamp to differentiate between runs+timestamp=$(date +%Y-%m-%d-%H-%M)+# run single node trainingtorchrun --standalone \--nproc_per_node $GPUS \@@ -37,5 +40,5 @@ torchrun --standalone \--dataset $PROCESSED_DATASET \--video_dir $COLLATED_VIDEO_DIR \--save_interval 224 \- --checkpoint_dir ./checkpoints \- --tensorboard_dir ./runs+ --checkpoint_dir ./checkpoints/$timestamp \+ --tensorboard_dir ./runs/$timestampdiff --git a/setup.py b/setup.pynew file mode 100644index 0000000..582beef--- /dev/null+++ b/setup.py@@ -0,0 +1,82 @@+import os+from typing import List++from setuptools import find_packages, setup+++def fetch_requirements(path) -> List[str]:+ """+ This function reads the requirements file.++ Args:+ path (str): the path to the requirements file.++ Returns:+ The lines in the requirements file.+ """+ with open(path, "r") as fd:+ return [r.strip() for r in fd.readlines()]+++def fetch_readme() -> str:+ """+ This function reads the README.md file in the current directory.++ Returns:+ The lines in the README file.+ """+ with open("README.md", encoding="utf-8") as f:+ return f.read()+++def get_version() -> str:+ """+ This function reads the version.txt and generates the colossalai/version.py file.++ Returns:+ The library version stored in version.txt.+ """++ setup_file_path = os.path.abspath(__file__)+ project_path = os.path.dirname(setup_file_path)+ version_txt_path = os.path.join(project_path, "version.txt")++ with open(version_txt_path) as f:+ version = f.read().strip()+ return version+++setup(+ name="open-sora",+ version=get_version(),+ packages=find_packages(+ exclude=(+ "docker",+ "tests",+ "docs",+ "examples",+ "tests",+ "scripts",+ "requirements",+ "extensions",+ "*.egg-info",+ ),+ ),+ description="Unofficial implementation of OpenAI's Sora by the Colossal-AI Team",+ long_description=fetch_readme(),+ long_description_content_type="text/markdown",+ license="Apache Software License 2.0",+ url="https://www.colossalai.org",+ project_urls={+ "Github": "https://github.com/hpcaitech/Open-Sora",+ },+ install_requires=fetch_requirements("requirements.txt"),+ python_requires=">=3.6",+ classifiers=[+ "Programming Language :: Python :: 3",+ "License :: OSI Approved :: Apache Software License",+ "Environment :: GPU :: NVIDIA CUDA",+ "Topic :: Scientific/Engineering :: Artificial Intelligence",+ "Topic :: System :: Distributed Computing",+ ],+)diff --git a/train.py b/train.pyindex 821ae8b..3a353c2 100644--- a/train.py+++ b/train.py@@ -7,16 +7,12 @@"""A minimal training script for DiT using PyTorch DDP."""-import torch--# the first flag below was False when we tested this script but True makes A100 training a lot faster:-torch.backends.cuda.matmul.allow_tf32 = True-torch.backends.cudnn.allow_tf32 = Trueimport argparseimport osfrom copy import deepcopyfrom functools import partial+import torchimport torch.distributed as distfrom colossalai import launch_from_torchfrom colossalai.accelerator import get_accelerator@@ -29,15 +25,21 @@ from torch.utils.tensorboard import SummaryWriterfrom tqdm import tqdmfrom transformers import AutoModel-from data_utils import load_datasets, make_batch, preprocess_batch-from diffusion import create_diffusion-from models import DiT_models+from open_sora.diffusion import create_diffusion+from open_sora.modeling import DiT_models+from open_sora.utils.data import load_datasets, make_batch, preprocess_batch################################################################################## Training Helper Functions ##################################################################################+def configure_backends():+ # the first flag below was False when we tested this script but True makes A100 training a lot faster:+ torch.backends.cuda.matmul.allow_tf32 = True+ torch.backends.cudnn.allow_tf32 = True++@torch.no_grad()def update_ema(ema_model, model, decay=0.9999):"""@@ -71,9 +73,12 @@ def main(args):"""Trains a new DiT model."""+ # Step 1: init distributed environmentlaunch_from_torch({})coordinator = DistCoordinator()logger = get_dist_logger()++ # Step 2: set up acceleration pluginsplugin = LowLevelZeroPlugin(stage=2, precision="fp16")booster = Booster(plugin=plugin)@@ -82,35 +87,35 @@ def main(args):os.makedirs(args.tensorboard_dir, exist_ok=True)writer = SummaryWriter(args.tensorboard_dir)- # Setup model+ # Step 3: Create VQ-VAEif len(args.vqvae) > 0:- vqvae = (- AutoModel.from_pretrained(args.vqvae, trust_remote_code=True)- .to(get_current_device())- .eval()- )+ vqvae = AutoModel.from_pretrained(args.vqvae, trust_remote_code=True).to(get_current_device()).eval()model_kwargs = {"in_channels": vqvae.embedding_dim}else:# disable VQ-VAE if not provided, just use raw video framesvqvae = Nonemodel_kwargs = {}++ # Step 4: Create DiT and EMAmodel = DiT_models[args.model](**model_kwargs).to(get_current_device())patch_size = model.patch_sizeema = deepcopy(model)requires_grad(ema, False)+model.train() # important! This enables embedding dropout for classifier-free guidanceema.eval() # EMA model should always be in eval mode++ # configure gradient checkpointingif args.grad_checkpoint:model.enable_gradient_checkpointing()- diffusion = create_diffusion(- timestep_respacing=""- ) # default: 1000 steps, linear noise schedule+ # Step 5: create diffusion pipeline+ diffusion = create_diffusion(timestep_respacing="") # default: 1000 steps, linear noise schedule- # Setup optimizer (we used default Adam betas=(0.9, 0.999) and a constant learning rate of 1e-4 in our paper):+ # Step 6: setup optimizer (we used default Adam betas=(0.9, 0.999) and a constant learning rate of 1e-4 in our paper):opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=0)- # Setup dataloader+ # Step 7: Setup dataloaderdataset = load_datasets(args.dataset)dataloader = plugin.prepare_dataloader(dataset,@@ -121,13 +126,14 @@ def main(args):)logger.info(f"Dataset contains {len(dataset)} samples", ranks=[0])- # Setup booster+ # Step 8: setup boostermodel, opt, _, dataloader, _ = booster.boost(model, opt, dataloader=dataloader)logger.info(f"Booster init max device memory: {get_accelerator().max_memory_allocated() / 1024 ** 2:.2f} MB",ranks=[0],)+ # Step 9: Trainnum_steps_per_epoch = len(dataloader) // args.accumulation_stepsfor epoch in range(args.epochs):@@ -148,15 +154,12 @@ def main(args):(video_inputs.shape[0],),device=video_inputs.device,)- loss_dict = diffusion.training_losses(- model, video_inputs, t, batch, mask=mask- )+ loss_dict = diffusion.training_losses(model, video_inputs, t, batch, mask=mask)loss = loss_dict["loss"].mean() / args.accumulation_stepstotal_loss.add_(loss.data)booster.backward(loss, opt)if (step + 1) % args.accumulation_steps == 0:-opt.step()opt.zero_grad()update_ema(ema, model)@@ -164,9 +167,7 @@ def main(args):all_reduce_mean(total_loss)pbar.set_postfix({"Loss": f"{total_loss.item():.4f}"})if coordinator.is_master():- global_step = (epoch * num_steps_per_epoch) + (- step + 1- ) // args.accumulation_steps+ global_step = (epoch * num_steps_per_epoch) + (step + 1) // args.accumulation_stepswriter.add_scalar(tag="Loss",scalar_value=total_loss.item(),@@ -176,20 +177,13 @@ def main(args):total_loss.zero_()# Save DiT checkpoint:- if (- args.save_interval > 0- and (step + 1) % (args.save_interval * args.accumulation_steps) == 0- ) or (step + 1) == len(dataloader):- save_path = os.path.join(- args.checkpoint_dir, f"epoch-{epoch}-step-{step}"- )+ if (args.save_interval > 0 and (step + 1) % (args.save_interval * args.accumulation_steps) == 0) or (+ step + 1+ ) == len(dataloader):+ save_path = os.path.join(args.checkpoint_dir, f"epoch-{epoch}-step-{step}")os.makedirs(save_path, exist_ok=True)- booster.save_model(- model, os.path.join(save_path, "model"), shard=True- )- booster.save_optimizer(- opt, os.path.join(save_path, "optimizer"), shard=True- )+ booster.save_model(model, os.path.join(save_path, "model"), shard=True)+ booster.save_optimizer(opt, os.path.join(save_path, "optimizer"), shard=True)if coordinator.is_master():ema_state_dict = ema.state_dict()for k, v in ema_state_dict.items():@@ -208,9 +202,7 @@ def main(args):if __name__ == "__main__":# Default args here will train DiT-XL/2 with the hyperparameters we used in our paper (except training iters).parser = argparse.ArgumentParser()- parser.add_argument(- "-m", "--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8"- )+ parser.add_argument("-m", "--model", type=str, choices=list(DiT_models.keys()), default="DiT-S/8")parser.add_argument("-d", "--dataset", nargs="+", default=[])parser.add_argument("-v", "--video_dir", type=str, required=True)parser.add_argument("-e", "--epochs", type=int, default=10)diff --git a/version.txt b/version.txtnew file mode 100644index 0000000..8acdd82--- /dev/null+++ b/version.txt@@ -0,0 +1 @@+0.0.1