Open-Sora: Democratizing Efficient Video Production for All
Python
added latte sampling (#22)
Frank Lee9648d53d4dd6
diff --git a/.gitignore b/.gitignoreindex 5296d86..47e4823 100644--- a/.gitignore+++ b/.gitignore@@ -164,3 +164,4 @@ cython_debug/dataset/runs/checkpoints/+outputs/diff --git a/open_sora/modeling/__init__.py b/open_sora/modeling/__init__.pyindex 72ac366..f1355ce 100644--- a/open_sora/modeling/__init__.py+++ b/open_sora/modeling/__init__.py@@ -1,3 +1,4 @@from .dit import DiT, DiT_models+from .latte import LatteT2V-__all__ = ["DiT_models", "DiT"]+__all__ = ["DiT_models", "DiT", "LatteT2V"]diff --git a/open_sora/modeling/dit/__init__.py b/open_sora/modeling/dit/__init__.pynew file mode 100644index 0000000..72ac366--- /dev/null+++ b/open_sora/modeling/dit/__init__.py@@ -0,0 +1,3 @@+from .dit import DiT, DiT_models++__all__ = ["DiT_models", "DiT"]diff --git a/open_sora/modeling/attn.py b/open_sora/modeling/dit/attn.pysimilarity index 79%rename from open_sora/modeling/attn.pyrename to open_sora/modeling/dit/attn.pyindex f81d36b..6ff95f2 100644--- a/open_sora/modeling/attn.py+++ b/open_sora/modeling/dit/attn.py@@ -37,9 +37,7 @@ class CrossAttention(nn.Module):):super().__init__()self.hidden_size = head_dim * num_heads- cross_attention_dim = (- cross_attention_dim if cross_attention_dim is not None else query_dim- )+ cross_attention_dim = cross_attention_dim if cross_attention_dim is not None else query_dimself.scale = head_dim**-0.5self.num_heads = num_heads@@ -50,9 +48,7 @@ class CrossAttention(nn.Module):self.to_k = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)self.to_v = nn.Linear(cross_attention_dim, self.hidden_size, bias=bias)- self.to_out = nn.Sequential(- nn.Linear(self.hidden_size, query_dim), nn.Dropout(dropout)- )+ self.to_out = nn.Sequential(nn.Linear(self.hidden_size, query_dim), nn.Dropout(dropout))def forward(self, hidden_states, context=None, mask=None):bsz, q_len, _ = hidden_states.shape@@ -66,24 +62,18 @@ class CrossAttention(nn.Module):# [B, S, H, D]query = query.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)key = key.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(1, 2)- value = value.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(- 1, 2- )+ value = value.view(bsz, kv_seq_len, self.num_heads, self.head_dim).transpose(1, 2)if mask is not None:assert mask.shape == (bsz, 1, q_len, kv_seq_len)if self.sdpa:- attn_output = F.scaled_dot_product_attention(- query, key, value, attn_mask=mask, scale=self.scale- )+ attn_output = F.scaled_dot_product_attention(query, key, value, attn_mask=mask, scale=self.scale)else:attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scaleassert attn_weights.shape == (bsz, self.num_heads, q_len, kv_seq_len)if mask is not None:attn_weights = attn_weights + mask- attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(- query.dtype- )+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)attn_output = torch.matmul(attn_weights, value)assert attn_output.shape == (bsz, self.num_heads, q_len, self.head_dim)attn_output = attn_output.transpose(1, 2).contiguous()@@ -128,11 +118,7 @@ class SeqParallelCrossAttention(CrossAttention):sdpa,)self.seq_parallel_group = seq_parallel_group- self.seq_parallel_size = (- dist.get_world_size(self.seq_parallel_group)- if seq_parallel_group is not None- else 1- )+ self.seq_parallel_size = dist.get_world_size(self.seq_parallel_group) if seq_parallel_group is not None else 1assert self.num_heads % self.seq_parallel_size == 0def forward(self, hidden_states, context=None, mask=None):@@ -148,51 +134,35 @@ class SeqParallelCrossAttention(CrossAttention):num_heads_parallel = self.num_heads // self.seq_parallel_sizehidden_size_parallel = self.hidden_size // self.seq_parallel_sizeif self.seq_parallel_size > 1:- query = all_to_all(- query, self.seq_parallel_group, scatter_dim=2, gather_dim=1- )+ query = all_to_all(query, self.seq_parallel_group, scatter_dim=2, gather_dim=1)key = all_to_all(key, self.seq_parallel_group, scatter_dim=2, gather_dim=1)- value = all_to_all(- value, self.seq_parallel_group, scatter_dim=2, gather_dim=1- )+ value = all_to_all(value, self.seq_parallel_group, scatter_dim=2, gather_dim=1)q_len *= self.seq_parallel_sizekv_seq_len *= self.seq_parallel_size# [B, S, H/P] -> [B, S, N/P, D] -> [B, N/P, S, D]- query = query.view(bsz, q_len, num_heads_parallel, self.head_dim).transpose(- 1, 2- )- key = key.view(bsz, kv_seq_len, num_heads_parallel, self.head_dim).transpose(- 1, 2- )- value = value.view(- bsz, kv_seq_len, num_heads_parallel, self.head_dim- ).transpose(1, 2)+ query = query.view(bsz, q_len, num_heads_parallel, self.head_dim).transpose(1, 2)+ key = key.view(bsz, kv_seq_len, num_heads_parallel, self.head_dim).transpose(1, 2)+ value = value.view(bsz, kv_seq_len, num_heads_parallel, self.head_dim).transpose(1, 2)if mask is not None:assert mask.shape == (bsz, 1, q_len, kv_seq_len)if self.sdpa:- attn_output = F.scaled_dot_product_attention(- query, key, value, attn_mask=mask, scale=self.scale- )+ attn_output = F.scaled_dot_product_attention(query, key, value, attn_mask=mask, scale=self.scale)else:attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scaleassert attn_weights.shape == (bsz, num_heads_parallel, q_len, kv_seq_len)if mask is not None:attn_weights = attn_weights + mask- attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(- query.dtype- )+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)attn_output = torch.matmul(attn_weights, value)assert attn_output.shape == (bsz, num_heads_parallel, q_len, self.head_dim)attn_output = attn_output.transpose(1, 2).contiguous()attn_output = attn_output.reshape(bsz, q_len, hidden_size_parallel)# [B, S, H/P] -> [B, S/P, H]if self.seq_parallel_size > 1:- attn_output = all_to_all(- attn_output, self.seq_parallel_group, scatter_dim=1, gather_dim=2- )+ attn_output = all_to_all(attn_output, self.seq_parallel_group, scatter_dim=1, gather_dim=2)attn_output = self.to_out(attn_output)return attn_output@@ -220,11 +190,7 @@ class FastSeqParallelCrossAttention(SeqParallelCrossAttention):sdpa,seq_parallel_group,)- self.seq_parallel_rank = (- dist.get_rank(self.seq_parallel_group)- if seq_parallel_group is not None- else 0- )+ self.seq_parallel_rank = dist.get_rank(self.seq_parallel_group) if seq_parallel_group is not None else 0self.sequence_parallel_param_slice = slice(self.hidden_size // self.seq_parallel_size * self.seq_parallel_rank,self.hidden_size // self.seq_parallel_size * (self.seq_parallel_rank + 1),@@ -239,11 +205,7 @@ class FastSeqParallelCrossAttention(SeqParallelCrossAttention):self.overlap = overlapdef _get_sliced_params(self, proj_layer: nn.Linear):- bias = bias = (- proj_layer.bias[self.sequence_parallel_param_slice]- if proj_layer.bias is not None- else None- )+ bias = bias = proj_layer.bias[self.sequence_parallel_param_slice] if proj_layer.bias is not None else Nonereturn proj_layer.weight[self.sequence_parallel_param_slice], biasdef _proj(self, x: torch.Tensor, proj_layer: nn.Linear):@@ -272,12 +234,8 @@ class FastSeqParallelCrossAttention(SeqParallelCrossAttention):)else:# [B, S/P, H] -> [B, S, H]- hidden_states = gather_forward_split_backward(- hidden_states, 1, self.seq_parallel_group- )- context = gather_forward_split_backward(- context, 1, self.seq_parallel_group- )+ hidden_states = gather_forward_split_backward(hidden_states, 1, self.seq_parallel_group)+ context = gather_forward_split_backward(context, 1, self.seq_parallel_group)query = self._proj(hidden_states, self.to_q)key = self._proj(context, self.to_k)value = self._proj(context, self.to_v)@@ -293,38 +251,26 @@ class FastSeqParallelCrossAttention(SeqParallelCrossAttention):kv_seq_len *= self.seq_parallel_size# [B, S, H/P] -> [B, S, N/P, D] -> [B, N/P, S, D]- query = query.view(bsz, q_len, num_heads_parallel, self.head_dim).transpose(- 1, 2- )- key = key.view(bsz, kv_seq_len, num_heads_parallel, self.head_dim).transpose(- 1, 2- )- value = value.view(- bsz, kv_seq_len, num_heads_parallel, self.head_dim- ).transpose(1, 2)+ query = query.view(bsz, q_len, num_heads_parallel, self.head_dim).transpose(1, 2)+ key = key.view(bsz, kv_seq_len, num_heads_parallel, self.head_dim).transpose(1, 2)+ value = value.view(bsz, kv_seq_len, num_heads_parallel, self.head_dim).transpose(1, 2)if mask is not None:assert mask.shape == (bsz, 1, q_len, kv_seq_len)if self.sdpa:- attn_output = F.scaled_dot_product_attention(- query, key, value, attn_mask=mask, scale=self.scale- )+ attn_output = F.scaled_dot_product_attention(query, key, value, attn_mask=mask, scale=self.scale)else:attn_weights = torch.matmul(query, key.transpose(2, 3)) / self.scaleassert attn_weights.shape == (bsz, num_heads_parallel, q_len, kv_seq_len)if mask is not None:attn_weights = attn_weights + mask- attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(- query.dtype- )+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)attn_output = torch.matmul(attn_weights, value)assert attn_output.shape == (bsz, num_heads_parallel, q_len, self.head_dim)attn_output = attn_output.transpose(1, 2).contiguous()attn_output = attn_output.reshape(bsz, q_len, hidden_size_parallel)# [B, S, H/P] -> [B, S/P, H]if self.seq_parallel_size > 1:- attn_output = all_to_all(- attn_output, self.seq_parallel_group, scatter_dim=1, gather_dim=2- )+ attn_output = all_to_all(attn_output, self.seq_parallel_group, scatter_dim=1, gather_dim=2)attn_output = self.to_out(attn_output)return attn_outputdiff --git a/open_sora/modeling/dit.py b/open_sora/modeling/dit/dit.pysimilarity index 86%rename from open_sora/modeling/dit.pyrename to open_sora/modeling/dit/dit.pyindex 776f736..93e1271 100644--- a/open_sora/modeling/dit.py+++ b/open_sora/modeling/dit/dit.py@@ -21,11 +21,7 @@ from timm.models.vision_transformer import Mlpfrom open_sora.utils.comm import gather_seq, split_seq-from .attn import (- CrossAttention,- FastSeqParallelCrossAttention,- SeqParallelCrossAttention,-)+from .attn import CrossAttention, FastSeqParallelCrossAttention, SeqParallelCrossAttentionSUPPORTED_SEQ_PARALLEL_MODES = ["ulysses", "fastseq"]@@ -65,23 +61,17 @@ class TimestepEmbedder(nn.Module):"""# https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.pyhalf = dim // 2- freqs = torch.exp(- -math.log(max_period)- * torch.arange(start=0, end=half, dtype=torch.float32)- / half- ).to(device=t.device)+ freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to(+ device=t.device+ )args = t[:, None].float() * freqs[None]embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)if dim % 2:- embedding = torch.cat(- [embedding, torch.zeros_like(embedding[:, :1])], dim=-1- )+ embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)return embeddingdef forward(self, t):- t_freq = self.timestep_embedding(t, self.frequency_embedding_size).to(- self.mlp[0].weight.dtype- )+ t_freq = self.timestep_embedding(t, self.frequency_embedding_size).to(self.mlp[0].weight.dtype)t_emb = self.mlp(t_freq)return t_emb@@ -94,9 +84,7 @@ class LabelEmbedder(nn.Module):def __init__(self, num_classes, hidden_size, dropout_prob):super().__init__()use_cfg_embedding = dropout_prob > 0- self.embedding_table = nn.Embedding(- num_classes + use_cfg_embedding, hidden_size- )+ self.embedding_table = nn.Embedding(num_classes + use_cfg_embedding, hidden_size)self.num_classes = num_classesself.dropout_prob = dropout_prob@@ -105,9 +93,7 @@ class LabelEmbedder(nn.Module):Drops labels to enable classifier-free guidance."""if force_drop_ids is None:- drop_ids = (- torch.rand(labels.shape[0], device=labels.device) < self.dropout_prob- )+ drop_ids = torch.rand(labels.shape[0], device=labels.device) < self.dropout_probelse:drop_ids = force_drop_ids == 1labels = torch.where(drop_ids, self.num_classes, labels)@@ -134,17 +120,13 @@ class PatchEmbedder(nn.Module):) -> None:super().__init__()self.patch_size = patch_size- self.proj = nn.Conv2d(- in_chans, embed_dim, kernel_size=patch_size, stride=patch_size, bias=bias- )+ self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size, bias=bias)self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()def forward(self, x: torch.Tensor) -> torch.Tensor:# [B, S, C, P, P] -> [B, S, C*P*P]x = x.view(*x.shape[:2], -1)- out = F.linear(- x, self.proj.weight.view(self.proj.weight.shape[0], -1), self.proj.bias- )+ out = F.linear(x, self.proj.weight.view(self.proj.weight.shape[0], -1), self.proj.bias)out = self.norm(out)# [B, S, H]return out@@ -234,9 +216,7 @@ class DiTBlock(nn.Module):attn_cls = FastSeqParallelCrossAttentionattn_kwargs["overlap"] = seq_parallel_overlapelse:- raise ValueError(- f"seq_parallel_mode must be one of {SUPPORTED_SEQ_PARALLEL_MODES}"- )+ raise ValueError(f"seq_parallel_mode must be one of {SUPPORTED_SEQ_PARALLEL_MODES}")else:attn_cls = CrossAttentionself.attn = attn_cls(@@ -264,20 +244,14 @@ class DiTBlock(nn.Module):act_layer=approx_gelu,drop=0,)- self.adaLN_modulation = nn.Sequential(- nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True)- )+ self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 6 * hidden_size, bias=True))def forward(self, x, attention_mask, t, context=None):- shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (- self.adaLN_modulation(t).chunk(6, dim=1)- )+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaLN_modulation(t).chunk(6, dim=1)x = x + gate_msa.unsqueeze(1) * self.attn(modulate(self.norm1(x), shift_msa, scale_msa), context, attention_mask)- x = x + gate_mlp.unsqueeze(1) * self.mlp(- modulate(self.norm2(x), shift_mlp, scale_mlp)- )+ x = x + gate_mlp.unsqueeze(1) * self.mlp(modulate(self.norm2(x), shift_mlp, scale_mlp))return x@@ -289,13 +263,9 @@ class FinalLayer(nn.Module):def __init__(self, hidden_size, patch_size, out_channels):super().__init__()self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)- self.linear = nn.Linear(- hidden_size, patch_size * patch_size * out_channels, bias=True- )+ self.linear = nn.Linear(hidden_size, patch_size * patch_size * out_channels, bias=True)self.patch_size = patch_size- self.adaLN_modulation = nn.Sequential(- nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True)- )+ self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True))def unpatchify(self, x):b, s, h = x.shape@@ -339,20 +309,10 @@ class DiT(nn.Module):self.patch_size = patch_sizeself.num_heads = num_headsself.seq_parallel_group = seq_parallel_group- self.seq_parallel_size = (- dist.get_world_size(self.seq_parallel_group)- if seq_parallel_group is not None- else 1- )- self.seq_parallel_rank = (- dist.get_rank(self.seq_parallel_group)- if seq_parallel_group is not None- else 0- )+ self.seq_parallel_size = dist.get_world_size(self.seq_parallel_group) if seq_parallel_group is not None else 1+ self.seq_parallel_rank = dist.get_rank(self.seq_parallel_group) if seq_parallel_group is not None else 0- self.video_embedder = PatchEmbedder(- patch_size, in_channels, hidden_size, bias=True- )+ self.video_embedder = PatchEmbedder(patch_size, in_channels, hidden_size, bias=True)self.t_embedder = TimestepEmbedder(hidden_size)self.pos_embed = PositionEmbedding(hidden_size, max_num_embeddings)self.text_embedder = TextEmbedder(@@ -419,9 +379,7 @@ class DiT(nn.Module):assert attention_mask.ndim == 4attention_mask = attention_mask.to(dtype)inverted_mask = 1.0 - attention_mask- return inverted_mask.masked_fill(- inverted_mask.to(torch.bool), torch.finfo(dtype).min- )+ return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)return attention_maskdef enable_gradient_checkpointing(self):@@ -444,9 +402,7 @@ class DiT(nn.Module):text_len = text_latent_states.shape[1]text_latent_states = self.text_embedder(text_latent_states)if not self.use_cross_attn:- video_latent_states = torch.cat(- [text_latent_states, video_latent_states], dim=1- )+ video_latent_states = torch.cat([text_latent_states, video_latent_states], dim=1)text_latent_states = Nonepos_embed = self.pos_embed(video_latent_states)video_latent_states = video_latent_states + pos_embed@@ -455,14 +411,10 @@ class DiT(nn.Module):if self.seq_parallel_group is not None and self.seq_parallel_size > 1:assert video_latent_states.shape[1] % self.seq_parallel_size == 0- video_latent_states = split_seq(- video_latent_states, self.seq_parallel_size, self.seq_parallel_rank- )+ video_latent_states = split_seq(video_latent_states, self.seq_parallel_size, self.seq_parallel_rank)if text_latent_states is not None:assert text_latent_states.shape[1] % self.seq_parallel_size == 0- text_latent_states = split_seq(- text_latent_states, self.seq_parallel_size, self.seq_parallel_rank- )+ text_latent_states = split_seq(text_latent_states, self.seq_parallel_size, self.seq_parallel_rank)for block in self.blocks:if self.grad_checkpointing and self.training:@@ -474,9 +426,7 @@ class DiT(nn.Module):text_latent_states,)else:- video_latent_states = block(- video_latent_states, attention_mask, t, text_latent_states- )+ video_latent_states = block(video_latent_states, attention_mask, t, text_latent_states)if self.seq_parallel_group is not None and self.seq_parallel_size > 1:video_latent_states = gather_seq(@@ -491,18 +441,14 @@ class DiT(nn.Module):video_latent_states = self.final_layer(video_latent_states, t)return video_latent_states- def forward_with_cfg(- self, x, t, text_latent_states, cfg_scale, attention_mask=None- ):+ def forward_with_cfg(self, x, t, text_latent_states, cfg_scale, attention_mask=None):"""Forward pass of DiT, but also batches the unconditional forward pass for classifier-free guidance."""# https://github.com/openai/glide-text2im/blob/main/notebooks/text2im.ipynbhalf = x[: len(x) // 2]combined = torch.cat([half, half], dim=0)- model_out = self.forward(- combined, t, text_latent_states, attention_mask=attention_mask- )+ model_out = self.forward(combined, t, text_latent_states, attention_mask=attention_mask)# For exact reproducibility reasons, we apply classifier-free guidance on only# three channels by default. The standard approach to cfg applies it to all channels.# This can be done by uncommenting the following line and commenting-out the line following that.@@ -541,9 +487,7 @@ def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=grid = grid.reshape([2, 1, grid_size, grid_size])pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)if cls_token and extra_tokens > 0:- pos_embed = np.concatenate(- [np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0- )+ pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0)return pos_embeddiff --git a/open_sora/modeling/latte/__init__.py b/open_sora/modeling/latte/__init__.pynew file mode 100644index 0000000..159db0f--- /dev/null+++ b/open_sora/modeling/latte/__init__.py@@ -0,0 +1,3 @@+from .latte_t2v import LatteT2V++__all__ = ["LatteT2V"]diff --git a/open_sora/modeling/latte/latte_t2v.py b/open_sora/modeling/latte/latte_t2v.pynew file mode 100644index 0000000..5c1360b--- /dev/null+++ b/open_sora/modeling/latte/latte_t2v.py@@ -0,0 +1,1003 @@+import json+import os+from dataclasses import dataclass+from typing import Any, Dict, Optional, Tuple++import torch+import torch.nn.functional as F+from diffusers.configuration_utils import ConfigMixin, register_to_config+from diffusers.models.activations import GEGLU, GELU, ApproximateGELU+from diffusers.models.attention import BasicTransformerBlock+from diffusers.models.attention_processor import Attention+from diffusers.models.embeddings import (+ CaptionProjection,+ CombinedTimestepSizeEmbeddings,+ ImagePositionalEmbeddings,+ PatchEmbed,+ SinusoidalPositionalEmbedding,+ get_1d_sincos_pos_embed_from_grid,+)+from diffusers.models.lora import LoRACompatibleConv, LoRACompatibleLinear+from diffusers.models.modeling_utils import ModelMixin+from diffusers.models.normalization import AdaLayerNorm, AdaLayerNormZero+from diffusers.utils import USE_PEFT_BACKEND, BaseOutput, deprecate+from diffusers.utils.torch_utils import maybe_allow_in_graph+from einops import rearrange, repeat+from torch import nn+++@maybe_allow_in_graph+class GatedSelfAttentionDense(nn.Module):+ r"""+ A gated self-attention dense layer that combines visual features and object features.++ Parameters:+ query_dim (`int`): The number of channels in the query.+ context_dim (`int`): The number of channels in the context.+ n_heads (`int`): The number of heads to use for attention.+ d_head (`int`): The number of channels in each head.+ """++ def __init__(self, query_dim: int, context_dim: int, n_heads: int, d_head: int):+ super().__init__()++ # we need a linear projection since we need cat visual feature and obj feature+ self.linear = nn.Linear(context_dim, query_dim)++ self.attn = Attention(query_dim=query_dim, heads=n_heads, dim_head=d_head)+ self.ff = FeedForward(query_dim, activation_fn="geglu")++ self.norm1 = nn.LayerNorm(query_dim)+ self.norm2 = nn.LayerNorm(query_dim)++ self.register_parameter("alpha_attn", nn.Parameter(torch.tensor(0.0)))+ self.register_parameter("alpha_dense", nn.Parameter(torch.tensor(0.0)))++ self.enabled = True++ def forward(self, x: torch.Tensor, objs: torch.Tensor) -> torch.Tensor:+ if not self.enabled:+ return x++ n_visual = x.shape[1]+ objs = self.linear(objs)++ x = x + self.alpha_attn.tanh() * self.attn(self.norm1(torch.cat([x, objs], dim=1)))[:, :n_visual, :]+ x = x + self.alpha_dense.tanh() * self.ff(self.norm2(x))++ return x+++class FeedForward(nn.Module):+ r"""+ A feed-forward layer.++ Parameters:+ dim (`int`): The number of channels in the input.+ dim_out (`int`, *optional*): The number of channels in the output. If not given, defaults to `dim`.+ mult (`int`, *optional*, defaults to 4): The multiplier to use for the hidden dimension.+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.+ activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward.+ final_dropout (`bool` *optional*, defaults to False): Apply a final dropout.+ """++ def __init__(+ self,+ dim: int,+ dim_out: Optional[int] = None,+ mult: int = 4,+ dropout: float = 0.0,+ activation_fn: str = "geglu",+ final_dropout: bool = False,+ ):+ super().__init__()+ inner_dim = int(dim * mult)+ dim_out = dim_out if dim_out is not None else dim+ linear_cls = LoRACompatibleLinear if not USE_PEFT_BACKEND else nn.Linear++ if activation_fn == "gelu":+ act_fn = GELU(dim, inner_dim)+ if activation_fn == "gelu-approximate":+ act_fn = GELU(dim, inner_dim, approximate="tanh")+ elif activation_fn == "geglu":+ act_fn = GEGLU(dim, inner_dim)+ elif activation_fn == "geglu-approximate":+ act_fn = ApproximateGELU(dim, inner_dim)++ self.net = nn.ModuleList([])+ # project in+ self.net.append(act_fn)+ # project dropout+ self.net.append(nn.Dropout(dropout))+ # project out+ self.net.append(linear_cls(inner_dim, dim_out))+ # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout+ if final_dropout:+ self.net.append(nn.Dropout(dropout))++ def forward(self, hidden_states: torch.Tensor, scale: float = 1.0) -> torch.Tensor:+ compatible_cls = (GEGLU,) if USE_PEFT_BACKEND else (GEGLU, LoRACompatibleLinear)+ for module in self.net:+ if isinstance(module, compatible_cls):+ hidden_states = module(hidden_states, scale)+ else:+ hidden_states = module(hidden_states)+ return hidden_states+++@maybe_allow_in_graph+class BasicTransformerBlock_(nn.Module):+ r"""+ A basic Transformer block.++ Parameters:+ dim (`int`): The number of channels in the input and output.+ num_attention_heads (`int`): The number of heads to use for multi-head attention.+ attention_head_dim (`int`): The number of channels in each head.+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.+ cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention.+ activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward.+ num_embeds_ada_norm (:+ obj: `int`, *optional*): The number of diffusion steps used during training. See `Transformer2DModel`.+ attention_bias (:+ obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter.+ only_cross_attention (`bool`, *optional*):+ Whether to use only cross-attention layers. In this case two cross attention layers are used.+ double_self_attention (`bool`, *optional*):+ Whether to use two self-attention layers. In this case no cross attention layers are used.+ upcast_attention (`bool`, *optional*):+ Whether to upcast the attention computation to float32. This is useful for mixed precision training.+ norm_elementwise_affine (`bool`, *optional*, defaults to `True`):+ Whether to use learnable elementwise affine parameters for normalization.+ norm_type (`str`, *optional*, defaults to `"layer_norm"`):+ The normalization layer to use. Can be `"layer_norm"`, `"ada_norm"` or `"ada_norm_zero"`.+ final_dropout (`bool` *optional*, defaults to False):+ Whether to apply a final dropout after the last feed-forward layer.+ attention_type (`str`, *optional*, defaults to `"default"`):+ The type of attention to use. Can be `"default"` or `"gated"` or `"gated-text-image"`.+ positional_embeddings (`str`, *optional*, defaults to `None`):+ The type of positional embeddings to apply to.+ num_positional_embeddings (`int`, *optional*, defaults to `None`):+ The maximum number of positional embeddings to apply.+ """++ def __init__(+ self,+ dim: int,+ num_attention_heads: int,+ attention_head_dim: int,+ dropout=0.0,+ cross_attention_dim: Optional[int] = None,+ activation_fn: str = "geglu",+ num_embeds_ada_norm: Optional[int] = None,+ attention_bias: bool = False,+ only_cross_attention: bool = False,+ double_self_attention: bool = False,+ upcast_attention: bool = False,+ norm_elementwise_affine: bool = True,+ norm_type: str = "layer_norm", # 'layer_norm', 'ada_norm', 'ada_norm_zero', 'ada_norm_single'+ norm_eps: float = 1e-5,+ final_dropout: bool = False,+ attention_type: str = "default",+ positional_embeddings: Optional[str] = None,+ num_positional_embeddings: Optional[int] = None,+ ):+ super().__init__()+ self.only_cross_attention = only_cross_attention++ self.use_ada_layer_norm_zero = (num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"+ self.use_ada_layer_norm = (num_embeds_ada_norm is not None) and norm_type == "ada_norm"+ self.use_ada_layer_norm_single = norm_type == "ada_norm_single"+ self.use_layer_norm = norm_type == "layer_norm"++ if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:+ raise ValueError(+ f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"+ f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."+ )++ if positional_embeddings and (num_positional_embeddings is None):+ raise ValueError(+ "If `positional_embedding` type is defined, `num_positition_embeddings` must also be defined."+ )++ if positional_embeddings == "sinusoidal":+ self.pos_embed = SinusoidalPositionalEmbedding(dim, max_seq_length=num_positional_embeddings)+ else:+ self.pos_embed = None++ # Define 3 blocks. Each block has its own normalization layer.+ # 1. Self-Attn+ if self.use_ada_layer_norm:+ self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm)+ elif self.use_ada_layer_norm_zero:+ self.norm1 = AdaLayerNormZero(dim, num_embeds_ada_norm)+ else:+ self.norm1 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps)++ self.attn1 = Attention(+ query_dim=dim,+ heads=num_attention_heads,+ dim_head=attention_head_dim,+ dropout=dropout,+ bias=attention_bias,+ cross_attention_dim=cross_attention_dim if only_cross_attention else None,+ upcast_attention=upcast_attention,+ )++ # # 2. Cross-Attn+ # if cross_attention_dim is not None or double_self_attention:+ # # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.+ # # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during+ # # the second cross attention block.+ # self.norm2 = (+ # AdaLayerNorm(dim, num_embeds_ada_norm)+ # if self.use_ada_layer_norm+ # else nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps)+ # )+ # self.attn2 = Attention(+ # query_dim=dim,+ # cross_attention_dim=cross_attention_dim if not double_self_attention else None,+ # heads=num_attention_heads,+ # dim_head=attention_head_dim,+ # dropout=dropout,+ # bias=attention_bias,+ # upcast_attention=upcast_attention,+ # ) # is self-attn if encoder_hidden_states is none+ # else:+ # self.norm2 = None+ # self.attn2 = None++ # 3. Feed-forward+ # if not self.use_ada_layer_norm_single:+ # self.norm3 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps)+ self.norm3 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps)++ self.ff = FeedForward(dim, dropout=dropout, activation_fn=activation_fn, final_dropout=final_dropout)++ # 4. Fuser+ if attention_type == "gated" or attention_type == "gated-text-image":+ self.fuser = GatedSelfAttentionDense(dim, cross_attention_dim, num_attention_heads, attention_head_dim)++ # 5. Scale-shift for PixArt-Alpha.+ if self.use_ada_layer_norm_single:+ self.scale_shift_table = nn.Parameter(torch.randn(6, dim) / dim**0.5)++ # let chunk size default to None+ self._chunk_size = None+ self._chunk_dim = 0++ def set_chunk_feed_forward(self, chunk_size: Optional[int], dim: int):+ # Sets chunk feed-forward+ self._chunk_size = chunk_size+ self._chunk_dim = dim++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ attention_mask: Optional[torch.FloatTensor] = None,+ encoder_hidden_states: Optional[torch.FloatTensor] = None,+ encoder_attention_mask: Optional[torch.FloatTensor] = None,+ timestep: Optional[torch.LongTensor] = None,+ cross_attention_kwargs: Dict[str, Any] = None,+ class_labels: Optional[torch.LongTensor] = None,+ ) -> torch.FloatTensor:+ # Notice that normalization is always applied before the real computation in the following blocks.+ # 0. Self-Attention+ batch_size = hidden_states.shape[0]++ if self.use_ada_layer_norm:+ norm_hidden_states = self.norm1(hidden_states, timestep)+ elif self.use_ada_layer_norm_zero:+ norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(+ hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype+ )+ elif self.use_layer_norm:+ norm_hidden_states = self.norm1(hidden_states)+ elif self.use_ada_layer_norm_single:+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (+ self.scale_shift_table[None] + timestep.reshape(batch_size, 6, -1)+ ).chunk(6, dim=1)+ norm_hidden_states = self.norm1(hidden_states)+ norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa+ norm_hidden_states = norm_hidden_states.squeeze(1)+ else:+ raise ValueError("Incorrect norm used")++ if self.pos_embed is not None:+ norm_hidden_states = self.pos_embed(norm_hidden_states)++ # 1. Retrieve lora scale.+ lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0++ # 2. Prepare GLIGEN inputs+ cross_attention_kwargs = cross_attention_kwargs.copy() if cross_attention_kwargs is not None else {}+ gligen_kwargs = cross_attention_kwargs.pop("gligen", None)++ attn_output = self.attn1(+ norm_hidden_states,+ encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None,+ attention_mask=attention_mask,+ **cross_attention_kwargs,+ )+ if self.use_ada_layer_norm_zero:+ attn_output = gate_msa.unsqueeze(1) * attn_output+ elif self.use_ada_layer_norm_single:+ attn_output = gate_msa * attn_output++ hidden_states = attn_output + hidden_states+ if hidden_states.ndim == 4:+ hidden_states = hidden_states.squeeze(1)++ # 2.5 GLIGEN Control+ if gligen_kwargs is not None:+ hidden_states = self.fuser(hidden_states, gligen_kwargs["objs"])++ # # 3. Cross-Attention+ # if self.attn2 is not None:+ # if self.use_ada_layer_norm:+ # norm_hidden_states = self.norm2(hidden_states, timestep)+ # elif self.use_ada_layer_norm_zero or self.use_layer_norm:+ # norm_hidden_states = self.norm2(hidden_states)+ # elif self.use_ada_layer_norm_single:+ # # For PixArt norm2 isn't applied here:+ # # https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L70C1-L76C103+ # norm_hidden_states = hidden_states+ # else:+ # raise ValueError("Incorrect norm")++ # if self.pos_embed is not None and self.use_ada_layer_norm_single is False:+ # norm_hidden_states = self.pos_embed(norm_hidden_states)++ # attn_output = self.attn2(+ # norm_hidden_states,+ # encoder_hidden_states=encoder_hidden_states,+ # attention_mask=encoder_attention_mask,+ # **cross_attention_kwargs,+ # )+ # hidden_states = attn_output + hidden_states++ # 4. Feed-forward+ # if not self.use_ada_layer_norm_single:+ # norm_hidden_states = self.norm3(hidden_states)++ if self.use_ada_layer_norm_zero:+ norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]++ if self.use_ada_layer_norm_single:+ # norm_hidden_states = self.norm2(hidden_states)+ norm_hidden_states = self.norm3(hidden_states)+ norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp++ if self._chunk_size is not None:+ # "feed_forward_chunk_size" can be used to save memory+ if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:+ raise ValueError(+ f"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."+ )++ num_chunks = norm_hidden_states.shape[self._chunk_dim] // self._chunk_size+ ff_output = torch.cat(+ [+ self.ff(hid_slice, scale=lora_scale)+ for hid_slice in norm_hidden_states.chunk(num_chunks, dim=self._chunk_dim)+ ],+ dim=self._chunk_dim,+ )+ else:+ ff_output = self.ff(norm_hidden_states, scale=lora_scale)++ if self.use_ada_layer_norm_zero:+ ff_output = gate_mlp.unsqueeze(1) * ff_output+ elif self.use_ada_layer_norm_single:+ ff_output = gate_mlp * ff_output++ hidden_states = ff_output + hidden_states+ if hidden_states.ndim == 4:+ hidden_states = hidden_states.squeeze(1)++ return hidden_states+++class AdaLayerNormSingle(nn.Module):+ r"""+ Norm layer adaptive layer norm single (adaLN-single).++ As proposed in PixArt-Alpha (see: https://arxiv.org/abs/2310.00426; Section 2.3).++ Parameters:+ embedding_dim (`int`): The size of each embedding vector.+ use_additional_conditions (`bool`): To use additional conditions for normalization or not.+ """++ def __init__(self, embedding_dim: int, use_additional_conditions: bool = False):+ super().__init__()++ self.emb = CombinedTimestepSizeEmbeddings(+ embedding_dim, size_emb_dim=embedding_dim // 3, use_additional_conditions=use_additional_conditions+ )++ self.silu = nn.SiLU()+ self.linear = nn.Linear(embedding_dim, 6 * embedding_dim, bias=True)++ def forward(+ self,+ timestep: torch.Tensor,+ added_cond_kwargs: Dict[str, torch.Tensor] = None,+ batch_size: int = None,+ hidden_dtype: Optional[torch.dtype] = None,+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:+ # No modulation happening here.+ embedded_timestep = self.emb(+ timestep, batch_size=batch_size, hidden_dtype=hidden_dtype, resolution=None, aspect_ratio=None+ )+ return self.linear(self.silu(embedded_timestep)), embedded_timestep+++@dataclass+class Transformer3DModelOutput(BaseOutput):+ """+ The output of [`Transformer2DModel`].++ Args:+ sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `(batch size, num_vector_embeds - 1, num_latent_pixels)` if [`Transformer2DModel`] is discrete):+ The hidden states output conditioned on the `encoder_hidden_states` input. If discrete, returns probability+ distributions for the unnoised latent pixels.+ """++ sample: torch.FloatTensor+++class LatteT2V(ModelMixin, ConfigMixin):+ _supports_gradient_checkpointing = True++ """+ A 2D Transformer model for image-like data.++ Parameters:+ num_attention_heads (`int`, *optional*, defaults to 16): The number of heads to use for multi-head attention.+ attention_head_dim (`int`, *optional*, defaults to 88): The number of channels in each head.+ in_channels (`int`, *optional*):+ The number of channels in the input and output (specify if the input is **continuous**).+ num_layers (`int`, *optional*, defaults to 1): The number of layers of Transformer blocks to use.+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.+ cross_attention_dim (`int`, *optional*): The number of `encoder_hidden_states` dimensions to use.+ sample_size (`int`, *optional*): The width of the latent images (specify if the input is **discrete**).+ This is fixed during training since it is used to learn a number of position embeddings.+ num_vector_embeds (`int`, *optional*):+ The number of classes of the vector embeddings of the latent pixels (specify if the input is **discrete**).+ Includes the class for the masked latent pixel.+ activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to use in feed-forward.+ num_embeds_ada_norm ( `int`, *optional*):+ The number of diffusion steps used during training. Pass if at least one of the norm_layers is+ `AdaLayerNorm`. This is fixed during training since it is used to learn a number of embeddings that are+ added to the hidden states.++ During inference, you can denoise for up to but not more steps than `num_embeds_ada_norm`.+ attention_bias (`bool`, *optional*):+ Configure if the `TransformerBlocks` attention should contain a bias parameter.+ """++ @register_to_config+ def __init__(+ self,+ num_attention_heads: int = 16,+ attention_head_dim: int = 88,+ in_channels: Optional[int] = None,+ out_channels: Optional[int] = None,+ num_layers: int = 1,+ dropout: float = 0.0,+ norm_num_groups: int = 32,+ cross_attention_dim: Optional[int] = None,+ attention_bias: bool = False,+ sample_size: Optional[int] = None,+ num_vector_embeds: Optional[int] = None,+ patch_size: Optional[int] = None,+ activation_fn: str = "geglu",+ num_embeds_ada_norm: Optional[int] = None,+ use_linear_projection: bool = False,+ only_cross_attention: bool = False,+ double_self_attention: bool = False,+ upcast_attention: bool = False,+ norm_type: str = "layer_norm",+ norm_elementwise_affine: bool = True,+ norm_eps: float = 1e-5,+ attention_type: str = "default",+ caption_channels: int = None,+ video_length: int = 16,+ ):+ super().__init__()+ self.use_linear_projection = use_linear_projection+ self.num_attention_heads = num_attention_heads+ self.attention_head_dim = attention_head_dim+ inner_dim = num_attention_heads * attention_head_dim+ self.video_length = video_length++ conv_cls = nn.Conv2d if USE_PEFT_BACKEND else LoRACompatibleConv+ linear_cls = nn.Linear if USE_PEFT_BACKEND else LoRACompatibleLinear++ # 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`+ # Define whether input is continuous or discrete depending on configuration+ self.is_input_continuous = (in_channels is not None) and (patch_size is None)+ self.is_input_vectorized = num_vector_embeds is not None+ self.is_input_patches = in_channels is not None and patch_size is not None++ if norm_type == "layer_norm" and num_embeds_ada_norm is not None:+ deprecation_message = (+ f"The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or"+ " incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config."+ " Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect"+ " results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it"+ " would be very nice if you could open a Pull request for the `transformer/config.json` file"+ )+ deprecate("norm_type!=num_embeds_ada_norm", "1.0.0", deprecation_message, standard_warn=False)+ norm_type = "ada_norm"++ if self.is_input_continuous and self.is_input_vectorized:+ raise ValueError(+ f"Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make"+ " sure that either `in_channels` or `num_vector_embeds` is None."+ )+ elif self.is_input_vectorized and self.is_input_patches:+ raise ValueError(+ f"Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make"+ " sure that either `num_vector_embeds` or `num_patches` is None."+ )+ elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:+ raise ValueError(+ f"Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:"+ f" {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None."+ )++ # 2. Define input layers+ if self.is_input_continuous:+ self.in_channels = in_channels++ self.norm = torch.nn.GroupNorm(num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True)+ if use_linear_projection:+ self.proj_in = linear_cls(in_channels, inner_dim)+ else:+ self.proj_in = conv_cls(in_channels, inner_dim, kernel_size=1, stride=1, padding=0)+ elif self.is_input_vectorized:+ assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"+ assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"++ self.height = sample_size+ self.width = sample_size+ self.num_vector_embeds = num_vector_embeds+ self.num_latent_pixels = self.height * self.width++ self.latent_image_embedding = ImagePositionalEmbeddings(+ num_embed=num_vector_embeds, embed_dim=inner_dim, height=self.height, width=self.width+ )+ elif self.is_input_patches:+ assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"++ self.height = sample_size+ self.width = sample_size++ self.patch_size = patch_size+ interpolation_scale = self.config.sample_size // 64 # => 64 (= 512 pixart) has interpolation scale 1+ interpolation_scale = max(interpolation_scale, 1)+ self.pos_embed = PatchEmbed(+ height=sample_size,+ width=sample_size,+ patch_size=patch_size,+ in_channels=in_channels,+ embed_dim=inner_dim,+ interpolation_scale=interpolation_scale,+ )++ # 3. Define transformers blocks+ self.transformer_blocks = nn.ModuleList(+ [+ BasicTransformerBlock(+ inner_dim,+ num_attention_heads,+ attention_head_dim,+ dropout=dropout,+ cross_attention_dim=cross_attention_dim,+ activation_fn=activation_fn,+ num_embeds_ada_norm=num_embeds_ada_norm,+ attention_bias=attention_bias,+ only_cross_attention=only_cross_attention,+ double_self_attention=double_self_attention,+ upcast_attention=upcast_attention,+ norm_type=norm_type,+ norm_elementwise_affine=norm_elementwise_affine,+ norm_eps=norm_eps,+ attention_type=attention_type,+ )+ for d in range(num_layers)+ ]+ )++ # Define temporal transformers blocks+ self.temporal_transformer_blocks = nn.ModuleList(+ [+ BasicTransformerBlock_( # one attention+ inner_dim,+ num_attention_heads, # num_attention_heads+ attention_head_dim, # attention_head_dim 72+ dropout=dropout,+ cross_attention_dim=None,+ activation_fn=activation_fn,+ num_embeds_ada_norm=num_embeds_ada_norm,+ attention_bias=attention_bias,+ only_cross_attention=only_cross_attention,+ double_self_attention=False,+ upcast_attention=upcast_attention,+ norm_type=norm_type,+ norm_elementwise_affine=norm_elementwise_affine,+ norm_eps=norm_eps,+ attention_type=attention_type,+ )+ for d in range(num_layers)+ ]+ )++ # 4. Define output layers+ self.out_channels = in_channels if out_channels is None else out_channels+ if self.is_input_continuous:+ # TODO: should use out_channels for continuous projections+ if use_linear_projection:+ self.proj_out = linear_cls(inner_dim, in_channels)+ else:+ self.proj_out = conv_cls(inner_dim, in_channels, kernel_size=1, stride=1, padding=0)+ elif self.is_input_vectorized:+ self.norm_out = nn.LayerNorm(inner_dim)+ self.out = nn.Linear(inner_dim, self.num_vector_embeds - 1)+ elif self.is_input_patches and norm_type != "ada_norm_single":+ self.norm_out = nn.LayerNorm(inner_dim, elementwise_affine=False, eps=1e-6)+ self.proj_out_1 = nn.Linear(inner_dim, 2 * inner_dim)+ self.proj_out_2 = nn.Linear(inner_dim, patch_size * patch_size * self.out_channels)+ elif self.is_input_patches and norm_type == "ada_norm_single":+ self.norm_out = nn.LayerNorm(inner_dim, elementwise_affine=False, eps=1e-6)+ self.scale_shift_table = nn.Parameter(torch.randn(2, inner_dim) / inner_dim**0.5)+ self.proj_out = nn.Linear(inner_dim, patch_size * patch_size * self.out_channels)++ # 5. PixArt-Alpha blocks.+ self.adaln_single = None+ self.use_additional_conditions = False+ if norm_type == "ada_norm_single":+ self.use_additional_conditions = self.config.sample_size == 128 # False, 128 -> 1024+ # TODO(Sayak, PVP) clean this, for now we use sample size to determine whether to use+ # additional conditions until we find better name+ self.adaln_single = AdaLayerNormSingle(inner_dim, use_additional_conditions=self.use_additional_conditions)++ self.caption_projection = None+ if caption_channels is not None:+ self.caption_projection = CaptionProjection(in_features=caption_channels, hidden_size=inner_dim)++ self.gradient_checkpointing = False++ # define temporal positional embedding+ temp_pos_embed = self.get_1d_sincos_temp_embed(inner_dim, video_length) # 1152 hidden size+ self.register_buffer("temp_pos_embed", torch.from_numpy(temp_pos_embed).float().unsqueeze(0), persistent=False)++ def _set_gradient_checkpointing(self, module, value=False):+ self.gradient_checkpointing = value++ def forward(+ self,+ hidden_states: torch.Tensor,+ timestep: Optional[torch.LongTensor] = None,+ encoder_hidden_states: Optional[torch.Tensor] = None,+ added_cond_kwargs: Dict[str, torch.Tensor] = None,+ class_labels: Optional[torch.LongTensor] = None,+ cross_attention_kwargs: Dict[str, Any] = None,+ attention_mask: Optional[torch.Tensor] = None,+ encoder_attention_mask: Optional[torch.Tensor] = None,+ use_image_num: int = 0,+ enable_temporal_attentions: bool = True,+ return_dict: bool = True,+ ):+ """+ The [`Transformer2DModel`] forward method.++ Args:+ hidden_states (`torch.LongTensor` of shape `(batch size, num latent pixels)` if discrete, `torch.FloatTensor` of shape `(batch size, frame, channel, height, width)` if continuous):+ Input `hidden_states`.+ encoder_hidden_states ( `torch.FloatTensor` of shape `(batch size, sequence len, embed dims)`, *optional*):+ Conditional embeddings for cross attention layer. If not given, cross-attention defaults to+ self-attention.+ timestep ( `torch.LongTensor`, *optional*):+ Used to indicate denoising step. Optional timestep to be applied as an embedding in `AdaLayerNorm`.+ class_labels ( `torch.LongTensor` of shape `(batch size, num classes)`, *optional*):+ Used to indicate class labels conditioning. Optional class labels to be applied as an embedding in+ `AdaLayerZeroNorm`.+ cross_attention_kwargs ( `Dict[str, Any]`, *optional*):+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under+ `self.processor` in+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).+ attention_mask ( `torch.Tensor`, *optional*):+ An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask+ is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large+ negative values to the attention scores corresponding to "discard" tokens.+ encoder_attention_mask ( `torch.Tensor`, *optional*):+ Cross-attention mask applied to `encoder_hidden_states`. Two formats supported:++ * Mask `(batch, sequence_length)` True = keep, False = discard.+ * Bias `(batch, 1, sequence_length)` 0 = keep, -10000 = discard.++ If `ndim == 2`: will be interpreted as a mask, then converted into a bias consistent with the format+ above. This bias will be added to the cross-attention scores.+ return_dict (`bool`, *optional*, defaults to `True`):+ Whether or not to return a [`~models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain+ tuple.++ Returns:+ If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a+ `tuple` where the first element is the sample tensor.+ """+ input_batch_size, c, frame, h, w = hidden_states.shape+ frame = frame - use_image_num+ hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w").contiguous()++ # ensure attention_mask is a bias, and give it a singleton query_tokens dimension.+ # we may have done this conversion already, e.g. if we came here via UNet2DConditionModel#forward.+ # we can tell by counting dims; if ndim == 2: it's a mask rather than a bias.+ # expects mask of shape:+ # [batch, key_tokens]+ # adds singleton query_tokens dimension:+ # [batch, 1, key_tokens]+ # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:+ # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn)+ # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)+ if attention_mask is not None and attention_mask.ndim == 2:+ # assume that mask is expressed as:+ # (1 = keep, 0 = discard)+ # convert mask into a bias that can be added to attention scores:+ # (keep = +0, discard = -10000.0)+ attention_mask = (1 - attention_mask.to(hidden_states.dtype)) * -10000.0+ attention_mask = attention_mask.unsqueeze(1)++ # convert encoder_attention_mask to a bias the same way we do for attention_mask+ if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2: # ndim == 2 means no image joint+ encoder_attention_mask = (1 - encoder_attention_mask.to(hidden_states.dtype)) * -10000.0+ encoder_attention_mask = encoder_attention_mask.unsqueeze(1)+ encoder_attention_mask = repeat(encoder_attention_mask, "b 1 l -> (b f) 1 l", f=frame).contiguous()+ elif encoder_attention_mask is not None and encoder_attention_mask.ndim == 3: # ndim == 3 means image joint+ encoder_attention_mask = (1 - encoder_attention_mask.to(hidden_states.dtype)) * -10000.0+ encoder_attention_mask_video = encoder_attention_mask[:, :1, ...]+ encoder_attention_mask_video = repeat(+ encoder_attention_mask_video, "b 1 l -> b (1 f) l", f=frame+ ).contiguous()+ encoder_attention_mask_image = encoder_attention_mask[:, 1:, ...]+ encoder_attention_mask = torch.cat([encoder_attention_mask_video, encoder_attention_mask_image], dim=1)+ encoder_attention_mask = rearrange(encoder_attention_mask, "b n l -> (b n) l").contiguous().unsqueeze(1)++ # Retrieve lora scale.+ cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0++ # 1. Input+ if self.is_input_patches: # here+ height, width = hidden_states.shape[-2] // self.patch_size, hidden_states.shape[-1] // self.patch_size+ num_patches = height * width++ hidden_states = self.pos_embed(hidden_states) # alrady add positional embeddings++ if self.adaln_single is not None:+ if self.use_additional_conditions and added_cond_kwargs is None:+ raise ValueError(+ "`added_cond_kwargs` cannot be None when using additional conditions for `adaln_single`."+ )+ # batch_size = hidden_states.shape[0]+ batch_size = input_batch_size+ timestep, embedded_timestep = self.adaln_single(+ timestep, added_cond_kwargs, batch_size=batch_size, hidden_dtype=hidden_states.dtype+ )++ # 2. Blocks+ if self.caption_projection is not None:+ batch_size = hidden_states.shape[0]+ encoder_hidden_states = self.caption_projection(encoder_hidden_states) # 3 120 1152++ if use_image_num != 0 and self.training:+ encoder_hidden_states_video = encoder_hidden_states[:, :1, ...]+ encoder_hidden_states_video = repeat(+ encoder_hidden_states_video, "b 1 t d -> b (1 f) t d", f=frame+ ).contiguous()+ encoder_hidden_states_image = encoder_hidden_states[:, 1:, ...]+ encoder_hidden_states = torch.cat([encoder_hidden_states_video, encoder_hidden_states_image], dim=1)+ encoder_hidden_states_spatial = rearrange(encoder_hidden_states, "b f t d -> (b f) t d").contiguous()+ else:+ encoder_hidden_states_spatial = repeat(+ encoder_hidden_states, "b t d -> (b f) t d", f=frame+ ).contiguous()++ # prepare timesteps for spatial and temporal block+ timestep_spatial = repeat(timestep, "b d -> (b f) d", f=frame + use_image_num).contiguous()+ timestep_temp = repeat(timestep, "b d -> (b p) d", p=num_patches).contiguous()++ for i, (spatial_block, temp_block) in enumerate(zip(self.transformer_blocks, self.temporal_transformer_blocks)):+ if self.training and self.gradient_checkpointing:+ hidden_states = torch.utils.checkpoint.checkpoint(+ spatial_block,+ hidden_states,+ attention_mask,+ encoder_hidden_states_spatial,+ encoder_attention_mask,+ timestep_spatial,+ cross_attention_kwargs,+ class_labels,+ use_reentrant=False,+ )++ if enable_temporal_attentions:+ hidden_states = rearrange(hidden_states, "(b f) t d -> (b t) f d", b=input_batch_size).contiguous()++ if use_image_num != 0: # image-video joitn training+ hidden_states_video = hidden_states[:, :frame, ...]+ hidden_states_image = hidden_states[:, frame:, ...]++ if i == 0:+ hidden_states_video = hidden_states_video + self.temp_pos_embed++ hidden_states_video = torch.utils.checkpoint.checkpoint(+ temp_block,+ hidden_states_video,+ None, # attention_mask+ None, # encoder_hidden_states+ None, # encoder_attention_mask+ timestep_temp,+ cross_attention_kwargs,+ class_labels,+ use_reentrant=False,+ )++ hidden_states = torch.cat([hidden_states_video, hidden_states_image], dim=1)+ hidden_states = rearrange(+ hidden_states, "(b t) f d -> (b f) t d", b=input_batch_size+ ).contiguous()++ else:+ if i == 0:+ hidden_states = hidden_states + self.temp_pos_embed++ hidden_states = torch.utils.checkpoint.checkpoint(+ temp_block,+ hidden_states,+ None, # attention_mask+ None, # encoder_hidden_states+ None, # encoder_attention_mask+ timestep_temp,+ cross_attention_kwargs,+ class_labels,+ use_reentrant=False,+ )++ hidden_states = rearrange(+ hidden_states, "(b t) f d -> (b f) t d", b=input_batch_size+ ).contiguous()+ else:+ hidden_states = spatial_block(+ hidden_states,+ attention_mask,+ encoder_hidden_states_spatial,+ encoder_attention_mask,+ timestep_spatial,+ cross_attention_kwargs,+ class_labels,+ )++ if enable_temporal_attentions:+ hidden_states = rearrange(hidden_states, "(b f) t d -> (b t) f d", b=input_batch_size).contiguous()++ if use_image_num != 0 and self.training:+ hidden_states_video = hidden_states[:, :frame, ...]+ hidden_states_image = hidden_states[:, frame:, ...]++ hidden_states_video = temp_block(+ hidden_states_video,+ None, # attention_mask+ None, # encoder_hidden_states+ None, # encoder_attention_mask+ timestep_temp,+ cross_attention_kwargs,+ class_labels,+ )++ hidden_states = torch.cat([hidden_states_video, hidden_states_image], dim=1)+ hidden_states = rearrange(+ hidden_states, "(b t) f d -> (b f) t d", b=input_batch_size+ ).contiguous()++ else:+ if i == 0:+ hidden_states = hidden_states + self.temp_pos_embed++ hidden_states = temp_block(+ hidden_states,+ None, # attention_mask+ None, # encoder_hidden_states+ None, # encoder_attention_mask+ timestep_temp,+ cross_attention_kwargs,+ class_labels,+ )++ hidden_states = rearrange(+ hidden_states, "(b t) f d -> (b f) t d", b=input_batch_size+ ).contiguous()++ if self.is_input_patches:+ if self.config.norm_type != "ada_norm_single":+ conditioning = self.transformer_blocks[0].norm1.emb(+ timestep, class_labels, hidden_dtype=hidden_states.dtype+ )+ shift, scale = self.proj_out_1(F.silu(conditioning)).chunk(2, dim=1)+ hidden_states = self.norm_out(hidden_states) * (1 + scale[:, None]) + shift[:, None]+ hidden_states = self.proj_out_2(hidden_states)+ elif self.config.norm_type == "ada_norm_single":+ embedded_timestep = repeat(embedded_timestep, "b d -> (b f) d", f=frame + use_image_num).contiguous()+ shift, scale = (self.scale_shift_table[None] + embedded_timestep[:, None]).chunk(2, dim=1)+ hidden_states = self.norm_out(hidden_states)+ # Modulation+ hidden_states = hidden_states * (1 + scale) + shift+ hidden_states = self.proj_out(hidden_states)++ # unpatchify+ if self.adaln_single is None:+ height = width = int(hidden_states.shape[1] ** 0.5)+ hidden_states = hidden_states.reshape(+ shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)+ )+ hidden_states = torch.einsum("nhwpqc->nchpwq", hidden_states)+ output = hidden_states.reshape(+ shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)+ )+ output = rearrange(output, "(b f) c h w -> b c f h w", b=input_batch_size).contiguous()++ if not return_dict:+ return (output,)++ return Transformer3DModelOutput(sample=output)++ def get_1d_sincos_temp_embed(self, embed_dim, length):+ pos = torch.arange(0, length).unsqueeze(1)+ return get_1d_sincos_pos_embed_from_grid(embed_dim, pos)++ @classmethod+ def from_pretrained_2d(cls, pretrained_model_path, subfolder=None, **kwargs):+ if subfolder is not None:+ pretrained_model_path = os.path.join(pretrained_model_path, subfolder)++ config_file = os.path.join(pretrained_model_path, "config.json")+ if not os.path.isfile(config_file):+ raise RuntimeError(f"{config_file} does not exist")+ with open(config_file, "r") as f:+ config = json.load(f)++ model = cls.from_config(config, **kwargs)++ # model_files = [+ # os.path.join(pretrained_model_path, 'diffusion_pytorch_model.bin'),+ # os.path.join(pretrained_model_path, 'diffusion_pytorch_model.safetensors')+ # ]++ # model_file = None++ # for fp in model_files:+ # if os.path.exists(fp):+ # model_file = fp++ # if not model_file:+ # raise RuntimeError(f"{model_file} does not exist")++ # if model_file.split(".")[-1] == "safetensors":+ # from safetensors import safe_open+ # state_dict = {}+ # with safe_open(model_file, framework="pt", device="cpu") as f:+ # for key in f.keys():+ # state_dict[key] = f.get_tensor(key)+ # else:+ # state_dict = torch.load(model_file, map_location="cpu")++ # for k, v in model.state_dict().items():+ # if 'temporal_transformer_blocks' in k:+ # state_dict.update({k: v})++ # model.load_state_dict(state_dict)++ return modeldiff --git a/scripts/sample/latte/pipeline_videogen.py b/scripts/sample/latte/pipeline_videogen.pynew file mode 100644index 0000000..8acd627--- /dev/null+++ b/scripts/sample/latte/pipeline_videogen.py@@ -0,0 +1,792 @@+# All rights reserved.+# Copyright 2024 Vchitect/Latte+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+# copied from https://github.com/Vchitect/Latte/blob/main/sample/pipeline_videogen.py++import html+import inspect+import re+import urllib.parse as ul+from dataclasses import dataclass+from typing import Callable, List, Optional, Tuple, Union++import einops+import torch+from diffusers.image_processor import VaeImageProcessor+from diffusers.models import AutoencoderKL, Transformer2DModel+from diffusers.pipelines.pipeline_utils import DiffusionPipeline+from diffusers.schedulers import DPMSolverMultistepScheduler+from diffusers.utils import (+ BACKENDS_MAPPING,+ BaseOutput,+ is_bs4_available,+ is_ftfy_available,+ logging,+ replace_example_docstring,+)+from diffusers.utils.torch_utils import randn_tensor+from transformers import T5EncoderModel, T5Tokenizer++logger = logging.get_logger(__name__) # pylint: disable=invalid-name++if is_bs4_available():+ from bs4 import BeautifulSoup++if is_ftfy_available():+ import ftfy+++EXAMPLE_DOC_STRING = """+ Examples:+ ```py+ >>> import torch+ >>> from diffusers import PixArtAlphaPipeline++ >>> # You can replace the checkpoint id with "PixArt-alpha/PixArt-XL-2-512x512" too.+ >>> pipe = PixArtAlphaPipeline.from_pretrained("PixArt-alpha/PixArt-XL-2-1024-MS", torch_dtype=torch.float16)+ >>> # Enable memory optimizations.+ >>> pipe.enable_model_cpu_offload()++ >>> prompt = "A small cactus with a happy face in the Sahara desert."+ >>> image = pipe(prompt).images[0]+ ```+"""+++@dataclass+class VideoPipelineOutput(BaseOutput):+ video: torch.Tensor+++class VideoGenPipeline(DiffusionPipeline):+ r"""+ Pipeline for text-to-image generation using PixArt-Alpha.++ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the+ library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)++ Args:+ vae ([`AutoencoderKL`]):+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.+ text_encoder ([`T5EncoderModel`]):+ Frozen text-encoder. PixArt-Alpha uses+ [T5](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5EncoderModel), specifically the+ [t5-v1_1-xxl](https://huggingface.co/PixArt-alpha/PixArt-alpha/tree/main/t5-v1_1-xxl) variant.+ tokenizer (`T5Tokenizer`):+ Tokenizer of class+ [T5Tokenizer](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5Tokenizer).+ transformer ([`Transformer2DModel`]):+ A text conditioned `Transformer2DModel` to denoise the encoded image latents.+ scheduler ([`SchedulerMixin`]):+ A scheduler to be used in combination with `transformer` to denoise the encoded image latents.+ """+ bad_punct_regex = re.compile(+ r"[" + "#®•©™&@·º½¾¿¡§~" + "\)" + "\(" + "\]" + "\[" + "\}" + "\{" + "\|" + "\\" + "\/" + "\*" + r"]{1,}"+ ) # noqa++ _optional_components = ["tokenizer", "text_encoder"]+ model_cpu_offload_seq = "text_encoder->transformer->vae"++ def __init__(+ self,+ tokenizer: T5Tokenizer,+ text_encoder: T5EncoderModel,+ vae: AutoencoderKL,+ transformer: Transformer2DModel,+ scheduler: DPMSolverMultistepScheduler,+ ):+ super().__init__()++ self.register_modules(+ tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, scheduler=scheduler+ )++ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)++ # Adapted from https://github.com/PixArt-alpha/PixArt-alpha/blob/master/diffusion/model/utils.py+ def mask_text_embeddings(self, emb, mask):+ if emb.shape[0] == 1:+ keep_index = mask.sum().item()+ return emb[:, :, :keep_index, :], keep_index # 1, 120, 4096 -> 1 7 4096+ else:+ masked_feature = emb * mask[:, None, :, None] # 1 120 4096+ return masked_feature, emb.shape[2]++ # Adapted from diffusers.pipelines.deepfloyd_if.pipeline_if.encode_prompt+ def encode_prompt(+ self,+ prompt: Union[str, List[str]],+ do_classifier_free_guidance: bool = True,+ negative_prompt: str = "",+ num_images_per_prompt: int = 1,+ device: Optional[torch.device] = None,+ prompt_embeds: Optional[torch.FloatTensor] = None,+ negative_prompt_embeds: Optional[torch.FloatTensor] = None,+ clean_caption: bool = False,+ mask_feature: bool = True,+ ):+ r"""+ Encodes the prompt into text encoder hidden states.++ Args:+ prompt (`str` or `List[str]`, *optional*):+ prompt to be encoded+ negative_prompt (`str` or `List[str]`, *optional*):+ The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds`+ instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). For+ PixArt-Alpha, this should be "".+ do_classifier_free_guidance (`bool`, *optional*, defaults to `True`):+ whether to use classifier free guidance or not+ num_images_per_prompt (`int`, *optional*, defaults to 1):+ number of images that should be generated per prompt+ device: (`torch.device`, *optional*):+ torch device to place the resulting embeddings on+ prompt_embeds (`torch.FloatTensor`, *optional*):+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not+ provided, text embeddings will be generated from `prompt` input argument.+ negative_prompt_embeds (`torch.FloatTensor`, *optional*):+ Pre-generated negative text embeddings. For PixArt-Alpha, it's should be the embeddings of the ""+ string.+ clean_caption (bool, defaults to `False`):+ If `True`, the function will preprocess and clean the provided caption before encoding.+ mask_feature: (bool, defaults to `True`):+ If `True`, the function will mask the text embeddings.+ """+ embeds_initially_provided = prompt_embeds is not None and negative_prompt_embeds is not None++ if device is None:+ device = self._execution_device++ if prompt is not None and isinstance(prompt, str):+ batch_size = 1+ elif prompt is not None and isinstance(prompt, list):+ batch_size = len(prompt)+ else:+ batch_size = prompt_embeds.shape[0]++ # See Section 3.1. of the paper.+ max_length = 120++ if prompt_embeds is None:+ prompt = self._text_preprocessing(prompt, clean_caption=clean_caption)+ text_inputs = self.tokenizer(+ prompt,+ padding="max_length",+ max_length=max_length,+ truncation=True,+ return_attention_mask=True,+ add_special_tokens=True,+ return_tensors="pt",+ )+ text_input_ids = text_inputs.input_ids+ untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids++ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(+ text_input_ids, untruncated_ids+ ):+ removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_length - 1 : -1])+ logger.warning(+ "The following part of your input was truncated because CLIP can only handle sequences up to"+ f" {max_length} tokens: {removed_text}"+ )++ attention_mask = text_inputs.attention_mask.to(device)+ prompt_embeds_attention_mask = attention_mask++ prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask)+ prompt_embeds = prompt_embeds[0]+ else:+ prompt_embeds_attention_mask = torch.ones_like(prompt_embeds)++ if self.text_encoder is not None:+ dtype = self.text_encoder.dtype+ elif self.transformer is not None:+ dtype = self.transformer.dtype+ else:+ dtype = None++ prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)++ bs_embed, seq_len, _ = prompt_embeds.shape+ # duplicate text embeddings and attention mask for each generation per prompt, using mps friendly method+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)+ prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)+ prompt_embeds_attention_mask = prompt_embeds_attention_mask.view(bs_embed, -1)+ prompt_embeds_attention_mask = prompt_embeds_attention_mask.repeat(num_images_per_prompt, 1)++ # get unconditional embeddings for classifier free guidance+ if do_classifier_free_guidance and negative_prompt_embeds is None:+ uncond_tokens = [negative_prompt] * batch_size+ uncond_tokens = self._text_preprocessing(uncond_tokens, clean_caption=clean_caption)+ max_length = prompt_embeds.shape[1]+ uncond_input = self.tokenizer(+ uncond_tokens,+ padding="max_length",+ max_length=max_length,+ truncation=True,+ return_attention_mask=True,+ add_special_tokens=True,+ return_tensors="pt",+ )+ attention_mask = uncond_input.attention_mask.to(device)++ negative_prompt_embeds = self.text_encoder(+ uncond_input.input_ids.to(device),+ attention_mask=attention_mask,+ )+ negative_prompt_embeds = negative_prompt_embeds[0]++ if do_classifier_free_guidance:+ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method+ seq_len = negative_prompt_embeds.shape[1]++ negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device)++ negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)+ negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)++ # For classifier free guidance, we need to do two forward passes.+ # Here we concatenate the unconditional and text embeddings into a single batch+ # to avoid doing two forward passes+ else:+ negative_prompt_embeds = None++ # print(prompt_embeds.shape) # 1 120 4096+ # print(negative_prompt_embeds.shape) # 1 120 4096++ # Perform additional masking.+ if mask_feature and not embeds_initially_provided:+ prompt_embeds = prompt_embeds.unsqueeze(1)+ masked_prompt_embeds, keep_indices = self.mask_text_embeddings(prompt_embeds, prompt_embeds_attention_mask)+ masked_prompt_embeds = masked_prompt_embeds.squeeze(1)+ masked_negative_prompt_embeds = (+ negative_prompt_embeds[:, :keep_indices, :] if negative_prompt_embeds is not None else None+ )++ # import torch.nn.functional as F++ # padding = (0, 0, 0, 113) # (左, 右, 下, 上)+ # masked_prompt_embeds_ = F.pad(masked_prompt_embeds, padding, "constant", 0)+ # masked_negative_prompt_embeds_ = F.pad(masked_negative_prompt_embeds, padding, "constant", 0)++ # print(masked_prompt_embeds == masked_prompt_embeds_[:, :masked_negative_prompt_embeds.shape[1], ...])++ return masked_prompt_embeds, masked_negative_prompt_embeds+ # return masked_prompt_embeds_, masked_negative_prompt_embeds_++ return prompt_embeds, negative_prompt_embeds++ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs+ def prepare_extra_step_kwargs(self, generator, eta):+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.+ # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502+ # and should be between [0, 1]++ accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())+ extra_step_kwargs = {}+ if accepts_eta:+ extra_step_kwargs["eta"] = eta++ # check if the scheduler accepts generator+ accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())+ if accepts_generator:+ extra_step_kwargs["generator"] = generator+ return extra_step_kwargs++ def check_inputs(+ self,+ prompt,+ height,+ width,+ negative_prompt,+ callback_steps,+ prompt_embeds=None,+ negative_prompt_embeds=None,+ ):+ if height % 8 != 0 or width % 8 != 0:+ raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")++ if (callback_steps is None) or (+ callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)+ ):+ raise ValueError(+ f"`callback_steps` has to be a positive integer but is {callback_steps} of type"+ f" {type(callback_steps)}."+ )++ if prompt is not None and prompt_embeds is not None:+ raise ValueError(+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"+ " only forward one of the two."+ )+ elif prompt is None and prompt_embeds is None:+ raise ValueError(+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."+ )+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")++ if prompt is not None and negative_prompt_embeds is not None:+ raise ValueError(+ f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:"+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."+ )++ if negative_prompt is not None and negative_prompt_embeds is not None:+ raise ValueError(+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."+ )++ if prompt_embeds is not None and negative_prompt_embeds is not None:+ if prompt_embeds.shape != negative_prompt_embeds.shape:+ raise ValueError(+ "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"+ f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"+ f" {negative_prompt_embeds.shape}."+ )++ # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._text_preprocessing+ def _text_preprocessing(self, text, clean_caption=False):+ if clean_caption and not is_bs4_available():+ logger.warn(BACKENDS_MAPPING["bs4"][-1].format("Setting `clean_caption=True`"))+ logger.warn("Setting `clean_caption` to False...")+ clean_caption = False++ if clean_caption and not is_ftfy_available():+ logger.warn(BACKENDS_MAPPING["ftfy"][-1].format("Setting `clean_caption=True`"))+ logger.warn("Setting `clean_caption` to False...")+ clean_caption = False++ if not isinstance(text, (tuple, list)):+ text = [text]++ def process(text: str):+ if clean_caption:+ text = self._clean_caption(text)+ text = self._clean_caption(text)+ else:+ text = text.lower().strip()+ return text++ return [process(t) for t in text]++ # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._clean_caption+ def _clean_caption(self, caption):+ caption = str(caption)+ caption = ul.unquote_plus(caption)+ caption = caption.strip().lower()+ caption = re.sub("<person>", "person", caption)+ # urls:+ caption = re.sub(+ r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa+ "",+ caption,+ ) # regex for urls+ caption = re.sub(+ r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa+ "",+ caption,+ ) # regex for urls+ # html:+ caption = BeautifulSoup(caption, features="html.parser").text++ # @<nickname>+ caption = re.sub(r"@[\w\d]+\b", "", caption)++ # 31C0—31EF CJK Strokes+ # 31F0—31FF Katakana Phonetic Extensions+ # 3200—32FF Enclosed CJK Letters and Months+ # 3300—33FF CJK Compatibility+ # 3400—4DBF CJK Unified Ideographs Extension A+ # 4DC0—4DFF Yijing Hexagram Symbols+ # 4E00—9FFF CJK Unified Ideographs+ caption = re.sub(r"[\u31c0-\u31ef]+", "", caption)+ caption = re.sub(r"[\u31f0-\u31ff]+", "", caption)+ caption = re.sub(r"[\u3200-\u32ff]+", "", caption)+ caption = re.sub(r"[\u3300-\u33ff]+", "", caption)+ caption = re.sub(r"[\u3400-\u4dbf]+", "", caption)+ caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption)+ caption = re.sub(r"[\u4e00-\u9fff]+", "", caption)+ #######################################################++ # все виды тире / all types of dash --> "-"+ caption = re.sub(+ r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa+ "-",+ caption,+ )++ # кавычки к одному стандарту+ caption = re.sub(r"[`´«»“”¨]", '"', caption)+ caption = re.sub(r"[‘’]", "'", caption)++ # "+ caption = re.sub(r""?", "", caption)+ # &+ caption = re.sub(r"&", "", caption)++ # ip adresses:+ caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption)++ # article ids:+ caption = re.sub(r"\d:\d\d\s+$", "", caption)++ # \n+ caption = re.sub(r"\\n", " ", caption)++ # "#123"+ caption = re.sub(r"#\d{1,3}\b", "", caption)+ # "#12345.."+ caption = re.sub(r"#\d{5,}\b", "", caption)+ # "123456.."+ caption = re.sub(r"\b\d{6,}\b", "", caption)+ # filenames:+ caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption)++ #+ caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT"""+ caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT"""++ caption = re.sub(self.bad_punct_regex, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT+ caption = re.sub(r"\s+\.\s+", r" ", caption) # " . "++ # this-is-my-cute-cat / this_is_my_cute_cat+ regex2 = re.compile(r"(?:\-|\_)")+ if len(re.findall(regex2, caption)) > 3:+ caption = re.sub(regex2, " ", caption)++ caption = ftfy.fix_text(caption)+ caption = html.unescape(html.unescape(caption))++ caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640+ caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc+ caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231++ caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)+ caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)+ caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption)+ caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption)+ caption = re.sub(r"\bpage\s+\d+\b", "", caption)++ caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a...++ caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption)++ caption = re.sub(r"\b\s+\:\s+", r": ", caption)+ caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption)+ caption = re.sub(r"\s+", " ", caption)++ caption.strip()++ caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption)+ caption = re.sub(r"^[\'\_,\-\:;]", r"", caption)+ caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption)+ caption = re.sub(r"^\.\S+$", "", caption)++ return caption.strip()++ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents+ def prepare_latents(+ self, batch_size, num_channels_latents, video_length, height, width, dtype, device, generator, latents=None+ ):+ shape = (+ batch_size,+ num_channels_latents,+ video_length,+ height // self.vae_scale_factor,+ width // self.vae_scale_factor,+ )+ if isinstance(generator, list) and len(generator) != batch_size:+ raise ValueError(+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."+ )++ if latents is None:+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)+ else:+ latents = latents.to(device)++ # scale the initial noise by the standard deviation required by the scheduler+ latents = latents * self.scheduler.init_noise_sigma+ return latents++ @torch.no_grad()+ @replace_example_docstring(EXAMPLE_DOC_STRING)+ def __call__(+ self,+ prompt: Union[str, List[str]] = None,+ negative_prompt: str = "",+ num_inference_steps: int = 20,+ timesteps: List[int] = None,+ guidance_scale: float = 4.5,+ num_images_per_prompt: Optional[int] = 1,+ video_length: Optional[int] = None,+ height: Optional[int] = None,+ width: Optional[int] = None,+ eta: float = 0.0,+ generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,+ latents: Optional[torch.FloatTensor] = None,+ prompt_embeds: Optional[torch.FloatTensor] = None,+ negative_prompt_embeds: Optional[torch.FloatTensor] = None,+ output_type: Optional[str] = "pil",+ return_dict: bool = True,+ callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,+ callback_steps: int = 1,+ clean_caption: bool = True,+ mask_feature: bool = True,+ enable_temporal_attentions: bool = True,+ enable_vae_temporal_decoder: bool = False,+ ) -> Union[VideoPipelineOutput, Tuple]:+ """+ Function invoked when calling the pipeline for generation.++ Args:+ prompt (`str` or `List[str]`, *optional*):+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.+ instead.+ negative_prompt (`str` or `List[str]`, *optional*):+ The prompt or prompts not to guide the image generation. If not defined, one has to pass+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is+ less than `1`).+ num_inference_steps (`int`, *optional*, defaults to 100):+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the+ expense of slower inference.+ timesteps (`List[int]`, *optional*):+ Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps`+ timesteps are used. Must be in descending order.+ guidance_scale (`float`, *optional*, defaults to 7.0):+ Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).+ `guidance_scale` is defined as `w` of equation 2. of [Imagen+ Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >+ 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,+ usually at the expense of lower image quality.+ num_images_per_prompt (`int`, *optional*, defaults to 1):+ The number of images to generate per prompt.+ height (`int`, *optional*, defaults to self.unet.config.sample_size):+ The height in pixels of the generated image.+ width (`int`, *optional*, defaults to self.unet.config.sample_size):+ The width in pixels of the generated image.+ eta (`float`, *optional*, defaults to 0.0):+ Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to+ [`schedulers.DDIMScheduler`], will be ignored for others.+ generator (`torch.Generator` or `List[torch.Generator]`, *optional*):+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)+ to make generation deterministic.+ latents (`torch.FloatTensor`, *optional*):+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents+ tensor will ge generated by sampling using the supplied random `generator`.+ prompt_embeds (`torch.FloatTensor`, *optional*):+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not+ provided, text embeddings will be generated from `prompt` input argument.+ negative_prompt_embeds (`torch.FloatTensor`, *optional*):+ Pre-generated negative text embeddings. For PixArt-Alpha this negative prompt should be "". If not+ provided, negative_prompt_embeds will be generated from `negative_prompt` input argument.+ output_type (`str`, *optional*, defaults to `"pil"`):+ The output format of the generate image. Choose between+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.+ return_dict (`bool`, *optional*, defaults to `True`):+ Whether or not to return a [`~pipelines.stable_diffusion.IFPipelineOutput`] instead of a plain tuple.+ callback (`Callable`, *optional*):+ A function that will be called every `callback_steps` steps during inference. The function will be+ called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.+ callback_steps (`int`, *optional*, defaults to 1):+ The frequency at which the `callback` function will be called. If not specified, the callback will be+ called at every step.+ clean_caption (`bool`, *optional*, defaults to `True`):+ Whether or not to clean the caption before creating embeddings. Requires `beautifulsoup4` and `ftfy` to+ be installed. If the dependencies are not installed, the embeddings will be created from the raw+ prompt.+ mask_feature (`bool` defaults to `True`): If set to `True`, the text embeddings will be masked.++ Examples:++ Returns:+ [`~pipelines.ImagePipelineOutput`] or `tuple`:+ If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is+ returned where the first element is a list with the generated images+ """+ # 1. Check inputs. Raise error if not correct+ height = height or self.transformer.config.sample_size * self.vae_scale_factor+ width = width or self.transformer.config.sample_size * self.vae_scale_factor+ self.check_inputs(prompt, height, width, negative_prompt, callback_steps, prompt_embeds, negative_prompt_embeds)++ # 2. Default height and width to transformer+ if prompt is not None and isinstance(prompt, str):+ batch_size = 1+ elif prompt is not None and isinstance(prompt, list):+ batch_size = len(prompt)+ else:+ batch_size = prompt_embeds.shape[0]++ device = self._execution_device++ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)+ # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`+ # corresponds to doing no classifier free guidance.+ do_classifier_free_guidance = guidance_scale > 1.0++ # 3. Encode input prompt+ prompt_embeds, negative_prompt_embeds = self.encode_prompt(+ prompt,+ do_classifier_free_guidance,+ negative_prompt=negative_prompt,+ num_images_per_prompt=num_images_per_prompt,+ device=device,+ prompt_embeds=prompt_embeds,+ negative_prompt_embeds=negative_prompt_embeds,+ clean_caption=clean_caption,+ mask_feature=mask_feature,+ )+ if do_classifier_free_guidance:+ prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)++ # 4. Prepare timesteps+ self.scheduler.set_timesteps(num_inference_steps, device=device)+ timesteps = self.scheduler.timesteps++ # 5. Prepare latents.+ latent_channels = self.transformer.config.in_channels+ latents = self.prepare_latents(+ batch_size * num_images_per_prompt,+ latent_channels,+ video_length,+ height,+ width,+ prompt_embeds.dtype,+ device,+ generator,+ latents,+ )++ # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)++ # 6.1 Prepare micro-conditions.+ added_cond_kwargs = {"resolution": None, "aspect_ratio": None}+ if self.transformer.config.sample_size == 128:+ resolution = torch.tensor([height, width]).repeat(batch_size * num_images_per_prompt, 1)+ aspect_ratio = torch.tensor([float(height / width)]).repeat(batch_size * num_images_per_prompt, 1)+ resolution = resolution.to(dtype=prompt_embeds.dtype, device=device)+ aspect_ratio = aspect_ratio.to(dtype=prompt_embeds.dtype, device=device)+ added_cond_kwargs = {"resolution": resolution, "aspect_ratio": aspect_ratio}++ # 7. Denoising loop+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)++ with self.progress_bar(total=num_inference_steps) as progress_bar:+ for i, t in enumerate(timesteps):+ latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents+ latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)++ current_timestep = t+ if not torch.is_tensor(current_timestep):+ # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can+ # This would be a good case for the `match` statement (Python 3.10+)+ is_mps = latent_model_input.device.type == "mps"+ if isinstance(current_timestep, float):+ dtype = torch.float32 if is_mps else torch.float64+ else:+ dtype = torch.int32 if is_mps else torch.int64+ current_timestep = torch.tensor([current_timestep], dtype=dtype, device=latent_model_input.device)+ elif len(current_timestep.shape) == 0:+ current_timestep = current_timestep[None].to(latent_model_input.device)+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML+ current_timestep = current_timestep.expand(latent_model_input.shape[0])++ # predict noise model_output+ noise_pred = self.transformer(+ latent_model_input,+ encoder_hidden_states=prompt_embeds,+ timestep=current_timestep,+ added_cond_kwargs=added_cond_kwargs,+ enable_temporal_attentions=enable_temporal_attentions,+ return_dict=False,+ )[0]++ # perform guidance+ if do_classifier_free_guidance:+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)+ noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)++ # learned sigma+ if self.transformer.config.out_channels // 2 == latent_channels:+ noise_pred = noise_pred.chunk(2, dim=1)[0]+ else:+ noise_pred = noise_pred++ # compute previous image: x_t -> x_t-1+ latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]++ # call the callback, if provided+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):+ progress_bar.update()+ if callback is not None and i % callback_steps == 0:+ step_idx = i // getattr(self.scheduler, "order", 1)+ callback(step_idx, t, latents)++ if not output_type == "latents":+ if enable_vae_temporal_decoder:+ video = self.decode_latents_with_temporal_decoder(latents)+ else:+ video = self.decode_latents(latents)+ else:+ video = latents+ return VideoPipelineOutput(video=video)++ # Offload all models+ self.maybe_free_model_hooks()++ if not return_dict:+ return (video,)++ return VideoPipelineOutput(video=video)++ def decode_latents(self, latents):+ video_length = latents.shape[2]+ latents = 1 / self.vae.config.scaling_factor * latents+ latents = einops.rearrange(latents, "b c f h w -> (b f) c h w")+ video = []+ for frame_idx in range(latents.shape[0]):+ video.append(self.vae.decode(latents[frame_idx : frame_idx + 1]).sample)+ video = torch.cat(video)+ video = einops.rearrange(video, "(b f) c h w -> b f h w c", f=video_length)+ video = ((video / 2.0 + 0.5).clamp(0, 1) * 255).to(dtype=torch.uint8).cpu().contiguous()+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16+ return video++ def decode_latents_with_temporal_decoder(self, latents):+ video_length = latents.shape[2]+ latents = 1 / self.vae.config.scaling_factor * latents+ latents = einops.rearrange(latents, "b c f h w -> (b f) c h w")+ video = []++ decode_chunk_size = 14+ for frame_idx in range(0, latents.shape[0], decode_chunk_size):+ num_frames_in = latents[frame_idx : frame_idx + decode_chunk_size].shape[0]++ decode_kwargs = {}+ decode_kwargs["num_frames"] = num_frames_in++ video.append(self.vae.decode(latents[frame_idx : frame_idx + decode_chunk_size], **decode_kwargs).sample)++ video = torch.cat(video)+ video = einops.rearrange(video, "(b f) c h w -> b f h w c", f=video_length)+ video = ((video / 2.0 + 0.5).clamp(0, 1) * 255).to(dtype=torch.uint8).cpu().contiguous()+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16+ return videodiff --git a/scripts/sample/latte/sample.sh b/scripts/sample/latte/sample.shnew file mode 100644index 0000000..c85272f--- /dev/null+++ b/scripts/sample/latte/sample.sh@@ -0,0 +1,19 @@+#!/usr/bin/env bash++# get args+GPUS=${1:-8}++# get root dir+FOLDER_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"+ROOT_DIR=$FOLDER_DIR/../../..++# go to root dir+cd $ROOT_DIR+++export PYTHONPATH=$FOLDER_DIR:$PYTHONPATH+python $FOLDER_DIR/sample_t2v.py \+ --checkpoint /home/lishenggui/projects/sora/hf-weights/models--maxin-cn--Latte/snapshots/8f0591220fa329f9d917086810b3c0f6544a87c7/t2v.pt \+ --model_path /home/lishenggui/projects/sora/hf-weights/models--maxin-cn--Latte/snapshots/8f0591220fa329f9d917086810b3c0f6544a87c7/t2v_required_models/ \+ --text_prompt "A dog in astronaut suit and sunglasses floating in space" \+ --output_path $ROOT_DIR/outputs/lattediff --git a/scripts/sample/latte/sample_t2v.py b/scripts/sample/latte/sample_t2v.pynew file mode 100644index 0000000..c65d007--- /dev/null+++ b/scripts/sample/latte/sample_t2v.py@@ -0,0 +1,242 @@+# All rights reserved.+# Copyright 2024 Vchitect/Latte+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+# modified from https://github.com/Vchitect/Latte/blob/main/sample/sample_t2v.py++import argparse+import os+import sys++import torch+from diffusers.models import AutoencoderKL, AutoencoderKLTemporalDecoder+from diffusers.schedulers import (+ DDIMScheduler,+ DDPMScheduler,+ DEISMultistepScheduler,+ DPMSolverMultistepScheduler,+ EulerAncestralDiscreteScheduler,+ EulerDiscreteScheduler,+ HeunDiscreteScheduler,+ KDPM2AncestralDiscreteScheduler,+ PNDMScheduler,+)+from diffusers.schedulers.scheduling_dpmsolver_singlestep import DPMSolverSinglestepScheduler+from transformers import T5EncoderModel, T5Tokenizer++sys.path.append(os.path.split(sys.path[0])[0])+import imageio+from pipeline_videogen import VideoGenPipeline+from utils import save_video_grid++from download import find_model+from open_sora.modeling import LatteT2V+++def parse_args():+ parser = argparse.ArgumentParser()+ parser.add_argument("--model_path", type=str, required=True, help="The path to the pretrained model files")+ parser.add_argument("--checkpoint", type=str, required=True, help="The path to the t2v.pt file.")+ parser.add_argument("--output_path", type=str, required=True, help="The path to save the output")++ # generation configs+ parser.add_argument(+ "--text_prompt", type=str, nargs="+", required=True, help="The text prompt to generate the video."+ )+ parser.add_argument("--video_length", type=int, default=16, help="The number of frames in the video.")+ parser.add_argument("--image_height", type=int, default=256, help="The size of the generated images.")+ parser.add_argument("--image_width", type=int, default=256, help="The size of the generated images.")+ parser.add_argument("--guidance_scale", type=float, default=7.5, help="The scale of the guidance loss.")+ parser.add_argument("--sample_method", type=str, default="PNDM", help="The sampling method to use.")+ parser.add_argument("--num_sampling_steps", type=int, default=50, help="The number of sampling steps.")+ parser.add_argument(+ "--enable_temporal_attentions", action="store_true", default=True, help="Whether to enable temporal attentions."+ )+ parser.add_argument(+ "--enable_vae_temporal_decoder",+ action="store_true",+ default=True,+ help="Whether to enable the VAE temporal decoder.",+ )++ # Scheduler configs+ parser.add_argument("--beta_start", type=float, default=0.0001)+ parser.add_argument("--beta_end", type=float, default=0.02)+ parser.add_argument("--beta_schedule", type=str, default="linear")+ parser.add_argument("--variance_type", type=str, default="learned_range")++ args = parser.parse_args()+ return args+++def main(args):+ torch.set_grad_enabled(False)+ device = "cuda" if torch.cuda.is_available() else "cpu"++ transformer_model = LatteT2V.from_pretrained_2d(+ args.model_path, subfolder="transformer", video_length=args.video_length+ ).to(device, dtype=torch.float16)+ state_dict = find_model(args.checkpoint)+ transformer_model.load_state_dict(state_dict["model"])++ if args.enable_vae_temporal_decoder:+ vae = AutoencoderKLTemporalDecoder.from_pretrained(+ args.model_path, subfolder="vae_temporal_decoder", torch_dtype=torch.float16+ ).to(device)+ else:+ vae = AutoencoderKL.from_pretrained(args.model_path, subfolder="vae", torch_dtype=torch.float16).to(device)+ tokenizer = T5Tokenizer.from_pretrained(args.model_path, subfolder="tokenizer")+ text_encoder = T5EncoderModel.from_pretrained(+ args.model_path, subfolder="text_encoder", torch_dtype=torch.float16+ ).to(device)++ # set eval mode+ transformer_model.eval()+ vae.eval()+ text_encoder.eval()++ if args.sample_method == "DDIM":+ scheduler = DDIMScheduler.from_pretrained(+ args.model_path,+ subfolder="scheduler",+ beta_start=args.beta_start,+ beta_end=args.beta_end,+ beta_schedule=args.beta_schedule,+ variance_type=args.variance_type,+ )+ elif args.sample_method == "EulerDiscrete":+ scheduler = EulerDiscreteScheduler.from_pretrained(+ args.model_path,+ subfolder="scheduler",+ beta_start=args.beta_start,+ beta_end=args.beta_end,+ beta_schedule=args.beta_schedule,+ variance_type=args.variance_type,+ )+ elif args.sample_method == "DDPM":+ scheduler = DDPMScheduler.from_pretrained(+ args.model_path,+ subfolder="scheduler",+ beta_start=args.beta_start,+ beta_end=args.beta_end,+ beta_schedule=args.beta_schedule,+ variance_type=args.variance_type,+ )+ elif args.sample_method == "DPMSolverMultistep":+ scheduler = DPMSolverMultistepScheduler.from_pretrained(+ args.model_path,+ subfolder="scheduler",+ beta_start=args.beta_start,+ beta_end=args.beta_end,+ beta_schedule=args.beta_schedule,+ variance_type=args.variance_type,+ )+ elif args.sample_method == "DPMSolverSinglestep":+ scheduler = DPMSolverSinglestepScheduler.from_pretrained(+ args.model_path,+ subfolder="scheduler",+ beta_start=args.beta_start,+ beta_end=args.beta_end,+ beta_schedule=args.beta_schedule,+ variance_type=args.variance_type,+ )+ elif args.sample_method == "PNDM":+ scheduler = PNDMScheduler.from_pretrained(+ args.model_path,+ subfolder="scheduler",+ beta_start=args.beta_start,+ beta_end=args.beta_end,+ beta_schedule=args.beta_schedule,+ variance_type=args.variance_type,+ )+ elif args.sample_method == "HeunDiscrete":+ scheduler = HeunDiscreteScheduler.from_pretrained(+ args.model_path,+ subfolder="scheduler",+ beta_start=args.beta_start,+ beta_end=args.beta_end,+ beta_schedule=args.beta_schedule,+ variance_type=args.variance_type,+ )+ elif args.sample_method == "EulerAncestralDiscrete":+ scheduler = EulerAncestralDiscreteScheduler.from_pretrained(+ args.model_path,+ subfolder="scheduler",+ beta_start=args.beta_start,+ beta_end=args.beta_end,+ beta_schedule=args.beta_schedule,+ variance_type=args.variance_type,+ )+ elif args.sample_method == "DEISMultistep":+ scheduler = DEISMultistepScheduler.from_pretrained(+ args.model_path,+ subfolder="scheduler",+ beta_start=args.beta_start,+ beta_end=args.beta_end,+ beta_schedule=args.beta_schedule,+ variance_type=args.variance_type,+ )+ elif args.sample_method == "KDPM2AncestralDiscrete":+ scheduler = KDPM2AncestralDiscreteScheduler.from_pretrained(+ args.model_path,+ subfolder="scheduler",+ beta_start=args.beta_start,+ beta_end=args.beta_end,+ beta_schedule=args.beta_schedule,+ variance_type=args.variance_type,+ )++ videogen_pipeline = VideoGenPipeline(+ vae=vae, text_encoder=text_encoder, tokenizer=tokenizer, scheduler=scheduler, transformer=transformer_model+ ).to(device)+ # videogen_pipeline.enable_xformers_memory_efficient_attention()++ if not os.path.exists(args.output_path):+ os.makedirs(args.output_path, exist_ok=True)++ video_grids = []+ for prompt in args.text_prompt:+ print("Processing the ({}) prompt".format(prompt))+ videos = videogen_pipeline(+ prompt,+ video_length=args.video_length,+ height=args.image_height,+ width=args.image_width,+ num_inference_steps=args.num_sampling_steps,+ guidance_scale=args.guidance_scale,+ enable_temporal_attentions=args.enable_temporal_attentions,+ num_images_per_prompt=1,+ mask_feature=True,+ enable_vae_temporal_decoder=args.enable_vae_temporal_decoder,+ ).video+ try:+ save_path = os.path.join(args.output_path, prompt.replace(" ", "_") + "_webv-imageio.mp4")+ imageio.mimwrite(save_path, videos[0], fps=8, quality=9) # highest quality is 10, lowest is 0+ except:+ print("Error when saving {}".format(prompt))+ video_grids.append(videos)+ video_grids = torch.cat(video_grids, dim=0)++ video_grids = save_video_grid(video_grids)++ # torchvision.io.write_video(args.output_path + '_%04d' % args.run_time + '-.mp4', video_grids, fps=6)+ save_path = os.path.join(args.output_path, "grid.mp4")+ imageio.mimwrite(save_path, video_grids, fps=8, quality=5)+ print("save path {}".format(abspath(args.output_path)))++ # save_videos_grid(video, f"./{prompt}.gif")+++if __name__ == "__main__":+ args = parse_args()+ main(args)diff --git a/scripts/sample/latte/utils.py b/scripts/sample/latte/utils.pynew file mode 100644index 0000000..3a9e4f1--- /dev/null+++ b/scripts/sample/latte/utils.py@@ -0,0 +1,471 @@+# All rights reserved.+# Copyright 2024 Vchitect/Latte+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+# copied from https://github.com/Vchitect/Latte/blob/main/utils.py++import html+import logging+import math+import os+import re+import subprocess+import urllib.parse as ul+from collections import OrderedDict+from typing import Iterable, Union++import torch+import torch.distributed as dist+from diffusers.utils import is_bs4_available, is_ftfy_available++# from torch._six import inf+from torch import inf+from torch.utils.tensorboard import SummaryWriter++if is_bs4_available():+ from bs4 import BeautifulSoup++if is_ftfy_available():+ import ftfy++_tensor_or_tensors = Union[torch.Tensor, Iterable[torch.Tensor]]+++#################################################################################+# Training Clip Gradients #+#################################################################################+++def get_grad_norm(parameters: _tensor_or_tensors, norm_type: float = 2.0) -> torch.Tensor:+ r"""+ Copy from torch.nn.utils.clip_grad_norm_++ Clips gradient norm of an iterable of parameters.++ The norm is computed over all gradients together, as if they were+ concatenated into a single vector. Gradients are modified in-place.++ Args:+ parameters (Iterable[Tensor] or Tensor): an iterable of Tensors or a+ single Tensor that will have gradients normalized+ max_norm (float or int): max norm of the gradients+ norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for+ infinity norm.+ error_if_nonfinite (bool): if True, an error is thrown if the total+ norm of the gradients from :attr:`parameters` is ``nan``,+ ``inf``, or ``-inf``. Default: False (will switch to True in the future)++ Returns:+ Total norm of the parameter gradients (viewed as a single vector).+ """+ if isinstance(parameters, torch.Tensor):+ parameters = [parameters]+ grads = [p.grad for p in parameters if p.grad is not None]+ norm_type = float(norm_type)+ if len(grads) == 0:+ return torch.tensor(0.0)+ device = grads[0].device+ if norm_type == inf:+ norms = [g.detach().abs().max().to(device) for g in grads]+ total_norm = norms[0] if len(norms) == 1 else torch.max(torch.stack(norms))+ else:+ total_norm = torch.norm(torch.stack([torch.norm(g.detach(), norm_type).to(device) for g in grads]), norm_type)+ return total_norm+++def clip_grad_norm_(+ parameters: _tensor_or_tensors,+ max_norm: float,+ norm_type: float = 2.0,+ error_if_nonfinite: bool = False,+ clip_grad=True,+) -> torch.Tensor:+ r"""+ Copy from torch.nn.utils.clip_grad_norm_++ Clips gradient norm of an iterable of parameters.++ The norm is computed over all gradients together, as if they were+ concatenated into a single vector. Gradients are modified in-place.++ Args:+ parameters (Iterable[Tensor] or Tensor): an iterable of Tensors or a+ single Tensor that will have gradients normalized+ max_norm (float or int): max norm of the gradients+ norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for+ infinity norm.+ error_if_nonfinite (bool): if True, an error is thrown if the total+ norm of the gradients from :attr:`parameters` is ``nan``,+ ``inf``, or ``-inf``. Default: False (will switch to True in the future)++ Returns:+ Total norm of the parameter gradients (viewed as a single vector).+ """+ if isinstance(parameters, torch.Tensor):+ parameters = [parameters]+ grads = [p.grad for p in parameters if p.grad is not None]+ max_norm = float(max_norm)+ norm_type = float(norm_type)+ if len(grads) == 0:+ return torch.tensor(0.0)+ device = grads[0].device+ if norm_type == inf:+ norms = [g.detach().abs().max().to(device) for g in grads]+ total_norm = norms[0] if len(norms) == 1 else torch.max(torch.stack(norms))+ else:+ total_norm = torch.norm(torch.stack([torch.norm(g.detach(), norm_type).to(device) for g in grads]), norm_type)+ # print(total_norm)++ if clip_grad:+ if error_if_nonfinite and torch.logical_or(total_norm.isnan(), total_norm.isinf()):+ raise RuntimeError(+ f"The total norm of order {norm_type} for gradients from "+ "`parameters` is non-finite, so it cannot be clipped. To disable "+ "this error and scale the gradients by the non-finite norm anyway, "+ "set `error_if_nonfinite=False`"+ )+ clip_coef = max_norm / (total_norm + 1e-6)+ # Note: multiplying by the clamped coef is redundant when the coef is clamped to 1, but doing so+ # avoids a `if clip_coef < 1:` conditional which can require a CPU <=> device synchronization+ # when the gradients do not reside in CPU memory.+ clip_coef_clamped = torch.clamp(clip_coef, max=1.0)+ for g in grads:+ g.detach().mul_(clip_coef_clamped.to(g.device))+ # gradient_cliped = torch.norm(torch.stack([torch.norm(g.detach(), norm_type).to(device) for g in grads]), norm_type)+ # print(gradient_cliped)+ return total_norm+++def get_experiment_dir(root_dir, args):+ # if args.pretrained is not None and 'Latte-XL-2-256x256.pt' not in args.pretrained:+ # root_dir += '-WOPRE'+ if args.use_compile:+ root_dir += "-Compile" # speedup by torch compile+ if args.fixed_spatial:+ root_dir += "-FixedSpa"+ if args.enable_xformers_memory_efficient_attention:+ root_dir += "-Xfor"+ if args.gradient_checkpointing:+ root_dir += "-Gc"+ if args.mixed_precision:+ root_dir += "-Amp"+ if args.image_size == 512:+ root_dir += "-512"+ return root_dir+++#################################################################################+# Training Logger #+#################################################################################+++def create_logger(logging_dir):+ """+ Create a logger that writes to a log file and stdout.+ """+ if dist.get_rank() == 0: # real logger+ logging.basicConfig(+ level=logging.INFO,+ # format='[\033[34m%(asctime)s\033[0m] %(message)s',+ format="[%(asctime)s] %(message)s",+ datefmt="%Y-%m-%d %H:%M:%S",+ handlers=[logging.StreamHandler(), logging.FileHandler(f"{logging_dir}/log.txt")],+ )+ logger = logging.getLogger(__name__)++ else: # dummy logger (does nothing)+ logger = logging.getLogger(__name__)+ logger.addHandler(logging.NullHandler())+ return logger+++def create_tensorboard(tensorboard_dir):+ """+ Create a tensorboard that saves losses.+ """+ if dist.get_rank() == 0: # real tensorboard+ # tensorboard+ writer = SummaryWriter(tensorboard_dir)++ return writer+++def write_tensorboard(writer, *args):+ """+ write the loss information to a tensorboard file.+ Only for pytorch DDP mode.+ """+ if dist.get_rank() == 0: # real tensorboard+ writer.add_scalar(args[0], args[1], args[2])+++#################################################################################+# EMA Update/ DDP Training Utils #+#################################################################################++[email protected]_grad()+def update_ema(ema_model, model, decay=0.9999):+ """+ Step the EMA model towards the current model.+ """+ ema_params = OrderedDict(ema_model.named_parameters())+ model_params = OrderedDict(model.named_parameters())++ for name, param in model_params.items():+ # TODO: Consider applying only to params that require_grad to avoid small numerical changes of pos_embed+ ema_params[name].mul_(decay).add_(param.data, alpha=1 - decay)+++def requires_grad(model, flag=True):+ """+ Set requires_grad flag for all parameters in a model.+ """+ for p in model.parameters():+ p.requires_grad = flag+++def cleanup():+ """+ End DDP training.+ """+ dist.destroy_process_group()+++def setup_distributed(backend="nccl", port=None):+ """Initialize distributed training environment.+ support both slurm and torch.distributed.launch+ see torch.distributed.init_process_group() for more details+ """+ num_gpus = torch.cuda.device_count()++ if "SLURM_JOB_ID" in os.environ:+ rank = int(os.environ["SLURM_PROCID"])+ world_size = int(os.environ["SLURM_NTASKS"])+ node_list = os.environ["SLURM_NODELIST"]+ addr = subprocess.getoutput(f"scontrol show hostname {node_list} | head -n1")+ # specify master port+ if port is not None:+ os.environ["MASTER_PORT"] = str(port)+ elif "MASTER_PORT" not in os.environ:+ # os.environ["MASTER_PORT"] = "29566"+ os.environ["MASTER_PORT"] = str(29567 + num_gpus)+ if "MASTER_ADDR" not in os.environ:+ os.environ["MASTER_ADDR"] = addr+ os.environ["WORLD_SIZE"] = str(world_size)+ os.environ["LOCAL_RANK"] = str(rank % num_gpus)+ os.environ["RANK"] = str(rank)+ else:+ rank = int(os.environ["RANK"])+ world_size = int(os.environ["WORLD_SIZE"])++ # torch.cuda.set_device(rank % num_gpus)++ dist.init_process_group(+ backend=backend,+ world_size=world_size,+ rank=rank,+ )+++#################################################################################+# Testing Utils #+#################################################################################+++def save_video_grid(video, nrow=None):+ b, t, h, w, c = video.shape++ if nrow is None:+ nrow = math.ceil(math.sqrt(b))+ ncol = math.ceil(b / nrow)+ padding = 1+ video_grid = torch.zeros((t, (padding + h) * nrow + padding, (padding + w) * ncol + padding, c), dtype=torch.uint8)++ print(video_grid.shape)+ for i in range(b):+ r = i // ncol+ c = i % ncol+ start_r = (padding + h) * r+ start_c = (padding + w) * c+ video_grid[:, start_r : start_r + h, start_c : start_c + w] = video[i]++ return video_grid+++#################################################################################+# MMCV Utils #+#################################################################################+++def collect_env():+ # Copyright (c) OpenMMLab. All rights reserved.+ from mmcv.utils import collect_env as collect_base_env+ from mmcv.utils import get_git_hash++ """Collect the information of the running environments."""++ env_info = collect_base_env()+ env_info["MMClassification"] = get_git_hash()[:7]++ for name, val in env_info.items():+ print(f"{name}: {val}")++ print(torch.cuda.get_arch_list())+ print(torch.version.cuda)+++#################################################################################+# Pixart-alpha Utils #+#################################################################################++bad_punct_regex = re.compile(+ r"[" + "#®•©™&@·º½¾¿¡§~" + "\)" + "\(" + "\]" + "\[" + "\}" + "\{" + "\|" + "\\" + "\/" + "\*" + r"]{1,}"+)+++def text_preprocessing(text, clean_caption=False):+ if clean_caption and not is_bs4_available():+ clean_caption = False++ if clean_caption and not is_ftfy_available():+ clean_caption = False++ if not isinstance(text, (tuple, list)):+ text = [text]++ def process(text: str):+ if clean_caption:+ text = clean_caption(text)+ text = clean_caption(text)+ else:+ text = text.lower().strip()+ return text++ return [process(t) for t in text]+++# Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._clean_caption+def clean_caption(caption):+ caption = str(caption)+ caption = ul.unquote_plus(caption)+ caption = caption.strip().lower()+ caption = re.sub("<person>", "person", caption)+ # urls:+ caption = re.sub(+ r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa+ "",+ caption,+ ) # regex for urls+ caption = re.sub(+ r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa+ "",+ caption,+ ) # regex for urls+ # html:+ caption = BeautifulSoup(caption, features="html.parser").text++ # @<nickname>+ caption = re.sub(r"@[\w\d]+\b", "", caption)++ # 31C0—31EF CJK Strokes+ # 31F0—31FF Katakana Phonetic Extensions+ # 3200—32FF Enclosed CJK Letters and Months+ # 3300—33FF CJK Compatibility+ # 3400—4DBF CJK Unified Ideographs Extension A+ # 4DC0—4DFF Yijing Hexagram Symbols+ # 4E00—9FFF CJK Unified Ideographs+ caption = re.sub(r"[\u31c0-\u31ef]+", "", caption)+ caption = re.sub(r"[\u31f0-\u31ff]+", "", caption)+ caption = re.sub(r"[\u3200-\u32ff]+", "", caption)+ caption = re.sub(r"[\u3300-\u33ff]+", "", caption)+ caption = re.sub(r"[\u3400-\u4dbf]+", "", caption)+ caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption)+ caption = re.sub(r"[\u4e00-\u9fff]+", "", caption)+ #######################################################++ # все виды тире / all types of dash --> "-"+ caption = re.sub(+ r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa+ "-",+ caption,+ )++ # кавычки к одному стандарту+ caption = re.sub(r"[`´«»“”¨]", '"', caption)+ caption = re.sub(r"[‘’]", "'", caption)++ # "+ caption = re.sub(r""?", "", caption)+ # &+ caption = re.sub(r"&", "", caption)++ # ip adresses:+ caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption)++ # article ids:+ caption = re.sub(r"\d:\d\d\s+$", "", caption)++ # \n+ caption = re.sub(r"\\n", " ", caption)++ # "#123"+ caption = re.sub(r"#\d{1,3}\b", "", caption)+ # "#12345.."+ caption = re.sub(r"#\d{5,}\b", "", caption)+ # "123456.."+ caption = re.sub(r"\b\d{6,}\b", "", caption)+ # filenames:+ caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption)++ #+ caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT"""+ caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT"""++ caption = re.sub(bad_punct_regex, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT+ caption = re.sub(r"\s+\.\s+", r" ", caption) # " . "++ # this-is-my-cute-cat / this_is_my_cute_cat+ regex2 = re.compile(r"(?:\-|\_)")+ if len(re.findall(regex2, caption)) > 3:+ caption = re.sub(regex2, " ", caption)++ caption = ftfy.fix_text(caption)+ caption = html.unescape(html.unescape(caption))++ caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640+ caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc+ caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231++ caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)+ caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)+ caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption)+ caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption)+ caption = re.sub(r"\bpage\s+\d+\b", "", caption)++ caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a...++ caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption)++ caption = re.sub(r"\b\s+\:\s+", r": ", caption)+ caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption)+ caption = re.sub(r"\s+", " ", caption)++ caption.strip()++ caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption)+ caption = re.sub(r"^[\'\_,\-\:;]", r"", caption)+ caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption)+ caption = re.sub(r"^\.\S+$", "", caption)++ return caption.strip()