Open-Sora: Democratizing Efficient Video Production for All
Python
update mask loss calculation
Zangwei Zheng11665b86a327
diff --git a/opensora/models/stdit/stdit.py b/opensora/models/stdit/stdit.pyindex 22a459b..70c5343 100644--- a/opensora/models/stdit/stdit.py+++ b/opensora/models/stdit/stdit.py@@ -86,8 +86,8 @@ class STDiTBlock(nn.Module):)def t_mask_select(self, x, masked_x, x_mask):- # x: [B, T, S, C]- # mased_x: [B, T, S, C]+ # x: [B, (T, S), C]+ # mased_x: [B, (T, S), C]# x_mask: [B, T]x = rearrange(x, "B (T S) C -> B T S C", T=self.d_t, S=self.d_s)masked_x = rearrange(masked_x, "B (T S) C -> B T S C", T=self.d_t, S=self.d_s)diff --git a/opensora/schedulers/iddpm/gaussian_diffusion.py b/opensora/schedulers/iddpm/gaussian_diffusion.pyindex b099de7..9c51934 100644--- a/opensora/schedulers/iddpm/gaussian_diffusion.py+++ b/opensora/schedulers/iddpm/gaussian_diffusion.py@@ -410,8 +410,8 @@ class GaussianDiffusion:sample = out["mean"] + nonzero_mask * th.exp(0.5 * out["log_variance"]) * noiseif mask is not None:if mask.shape[0] != x.shape[0]:- mask = mask.repeat(2, 1) # HACK- sample = th.where(mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1), sample, x)+ mask = mask.repeat(2, 1) # HACK+ sample = th.where(mask[:, None, :, None, None], sample, x)return {"sample": sample, "pred_xstart": out["pred_xstart"]}@@ -684,13 +684,16 @@ class GaussianDiffusion:true_mean, _, true_log_variance_clipped = self.q_posterior_mean_variance(x_start=x_start, x_t=x_t, t=t)out = self.p_mean_variance(model, x_t, t, clip_denoised=clip_denoised, model_kwargs=model_kwargs)kl = normal_kl(true_mean, true_log_variance_clipped, out["mean"], out["log_variance"])- kl = mean_flat(kl, mask=mask) / np.log(2.0)decoder_nll = -discretized_gaussian_log_likelihood(x_start, means=out["mean"], log_scales=0.5 * out["log_variance"])assert decoder_nll.shape == x_start.shape- decoder_nll = mean_flat(decoder_nll, mask=mask) / np.log(2.0)++ if mask is not None:+ kl = th.where(mask[:, None, :, None, None], kl, decoder_nll)+ kl = mean_flat(kl) / np.log(2.0)+ decoder_nll = mean_flat(decoder_nll) / np.log(2.0)# At the first timestep return the decoder NLL,# otherwise return KL(q(x_{t-1}|x_t,x_0) || p(x_{t-1}|x_t))@@ -715,7 +718,9 @@ class GaussianDiffusion:noise = th.randn_like(x_start)x_t = self.q_sample(x_start, t, noise=noise)if mask is not None:- x_t = th.where(mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1), x_t, x_start)+ t0 = th.zeros_like(t)+ x_t0 = self.q_sample(x_start, t0, noise=noise)+ x_t = th.where(mask[:, None, :, None, None], x_t, x_t0)terms = {}@@ -764,10 +769,10 @@ class GaussianDiffusion:}[self.model_mean_type]assert model_output.shape == target.shape == x_start.shapeif weights is None:- terms["mse"] = mean_flat((target - model_output) ** 2, mask=mask)+ terms["mse"] = mean_flat((target - model_output) ** 2)else:weight = _extract_into_tensor(weights, t, target.shape)- terms["mse"] = mean_flat(weight * (target - model_output) ** 2, mask=mask)+ terms["mse"] = mean_flat(weight * (target - model_output) ** 2)if "vb" in terms:terms["loss"] = terms["mse"] + terms["vb"]else: