Python
Hunyuan 1.5 support
DeepBeepMeepa2a1161e9bf6
diff --git a/README.md b/README.mdindex 2576be8..230aa29 100644--- a/README.md+++ b/README.md@@ -36,6 +36,24 @@ WanGP supports the Wan (and derived models), Hunyuan Video and LTV Video models-----## 🔥 Latest Updates :+### November 24 2025: WanGP v9.6, The Return of the King++So here is *Tencet* who is back in the race: let's welcome **Hunyuan Video 1.5**++Despite only 8B parameters it offers quite a high level of quality. It is not just one model but a family of models:+- Text 2 Video+- Image 2 Video+- Upsamplers (720p & 1080p)++Each model comes on day one with several finetunes specialized for a specific resolution.+The downside right now is that to get the best quality you need to use guidance > 1 and a high number of Steps (20+).++But dont go away yet ! *LightX2V* (https://huggingface.co/lightx2v/Hy1.5-Distill-Models/) is on deck and has already delivered an *Accelerated 4 steps Finetune* for the *t2v 480p* model. It is part of today's delivery.++WanGP implementation of Hunyuan 1.5 is quite complete as you will get straight away *Video Gen Preview* (WanGP exclusivity!) and *Sliding Window* support. It is also ready for *Tea Cache* or *Mag Cache* (just waiting for the official parameters)++Please note Hunyuan v1 Loras are not compatible since the latent space is different. You can add loras for Hunyuan Video 1.5 in the *loras_hunyuan/1.5* folder.+### November 21 2025: WanGP v9.52, And there was motionIn this release WanGP turns you into a Motion Master:diff --git a/defaults/t2v_nexus_1.3B.json b/defaults/t2v_nexus_1.3B.jsonindex 282aae5..12fb6ae 100644--- a/defaults/t2v_nexus_1.3B.json+++ b/defaults/t2v_nexus_1.3B.json@@ -9,7 +9,7 @@"auto_quantize": false,"profile_dirs": [""]},- "prompt": "A medium close-up shot of a samurai and a knight engaged in a fierce duel in front of a traditional Japanese dojo. The samurai, on the left, wears a traditional black kimono with a white sash, his katana raised high, ready to strike. The knight, on the right, dons medieval armor with a silver helmet and a red plume, holding a broadsword in a defensive stance. The background features a wooden dojo floor and shoji screens. The camera angle is slightly low, capturing the intensity of the duel. The lighting is natural, highlighting the sheen on the samurai's katana and the knight's armor. The scene is dynamic, with swift, precise movements as the samurai and knight clash. The dojo's wooden texture and the soft light from the shoji screens add a traditional, serene atmosphere to the intense combat.",+ "prompt": "scene: \"traditional Japanese dojo, wooden floor, shoji screens, serene yet intense atmosphere\", subjects: \"samurai on left and medieval knight on right engaged in fierce duel, medium close-up, slightly low angle\", samurai: \"traditional black kimono, white sash, raised katana held with perfect hands, poised to strike, swift precise movement\", knight: \"medieval armour, silver helmet with red plume, holding broadsword in defensive stance, perfect hands gripping hilt\", combat: \"dynamic clash of blades, swift controlled movements, clear separation of weapons, realistic metallic sheen, intense duel energy\", camera: \"medium close-up, slightly low angle, cinematic framing\", lighting: \"natural light highlighting katana blade and knight armour reflections\", style: \"photorealistic, traditional, cinematic, nothing modern\", environment: \"polished wooden dojo floor, shoji screens diffusing soft light\", quality: \"high detail, crisp edges on weapons, clean silhouettes\", neg: \"blurry weapons, fused blades, melted shapes, extra limbs, modern items, text, watermark\".","resolution": "832x480","video_length": 81,"seed": -1,diff --git a/docs/LORAS.md b/docs/LORAS.mdindex f17b51f..e031e26 100644--- a/docs/LORAS.md+++ b/docs/LORAS.md@@ -18,6 +18,7 @@ Optional sub folders:### Other Models- `loras_hunyuan/` - Hunyuan Video t2v loras+- `loras_hunyuan/1.5/` - Loras specifically for Hunyuan 1.5 models- `loras_hunyuan_i2v/` - Hunyuan Video i2v loras- `loras_ltxv/` - LTX Video loras- `loras_flux/` - Flux lorasdiff --git a/models/hyvideo/constants.py b/models/hyvideo/constants.pyindex 942777c..2fd4154 100644--- a/models/hyvideo/constants.py+++ b/models/hyvideo/constants.py@@ -72,6 +72,25 @@ PROMPT_TEMPLATE_ENCODE_VIDEO_I2V = (NEGATIVE_PROMPT = "Aerial view, aerial view, overexposed, low quality, deformation, a poor composition, bad hands, bad teeth, bad eyes, bad limbs, distortion"NEGATIVE_PROMPT_I2V = "deformation, a poor composition and deformed video, bad teeth, bad eyes, bad limbs"+PROMPT_TEMPLATE_ENCODE_IMAGE_JSON = [+ {"role": "system", "content": "You are a helpful assistant. Describe the image by detailing the following aspects: \+ 1. The main content and theme of the image. \+ 2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects. \+ 3. The background environment, light, style and atmosphere."},+ {"role": "user", "content": "{}"}+]++PROMPT_TEMPLATE_ENCODE_VIDEO_JSON = [+ {"role": "system", "content": "You are a helpful assistant. Describe the video by detailing the following aspects: \+ 1. The main content and theme of the video. \+ 2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects. \+ 3. Actions, events, behaviors temporal relationships, physical movement changes of the objects. \+ 4. background environment, light, style and atmosphere. \+ 5. camera angles, movements, and transitions used in the video."},+ {"role": "user", "content": "{}"}+]++PROMPT_TEMPLATE = {"dit-llm-encode": {"template": PROMPT_TEMPLATE_ENCODE,@@ -97,6 +116,8 @@ PROMPT_TEMPLATE = {"image_emb_len": 576,"double_return_token_id": 271},+ "li-dit-encode-image-json": {"template": PROMPT_TEMPLATE_ENCODE_IMAGE_JSON, "crop_start": -1}, # auto-calculate crop_start+ "li-dit-encode-video-json": {"template": PROMPT_TEMPLATE_ENCODE_VIDEO_JSON, "crop_start": -1}, # auto-calculate crop_start}# ======================= Model ======================diff --git a/models/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py b/models/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.pyindex ed91f9c..a30bfd1 100644--- a/models/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py+++ b/models/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py@@ -23,7 +23,7 @@ import torch.distributed as distimport numpy as npfrom dataclasses import dataclassfrom packaging import version-+import refrom diffusers.callbacks import MultiPipelineCallbacks, PipelineCallbackfrom diffusers.configuration_utils import FrozenDictfrom diffusers.image_processor import VaeImageProcessor@@ -175,6 +175,13 @@ class HunyuanVideoPipeline(DiffusionPipeline):transformer: HYVideoDiffusionTransformer,scheduler: KarrasDiffusionSchedulers,text_encoder_2: Optional[TextEncoder] = None,+ byt5_model= None,+ byt5_tokenizer= None,+ byt5_max_length= 256,+ vision_encoder=None,+ upsampler=None,+ prompt_format=None,+ hunyuan_1_5 = None,progress_bar_config: Dict[str, Any] = None,args=None,):@@ -233,10 +240,194 @@ class HunyuanVideoPipeline(DiffusionPipeline):transformer=transformer,scheduler=scheduler,text_encoder_2=text_encoder_2,+ byt5_model= byt5_model,+ byt5_tokenizer= byt5_tokenizer,+ vision_encoder=vision_encoder,+ upsampler=upsampler,)+ self.byt5_max_length = byt5_max_length+ self.prompt_format= prompt_formatself.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)self.noise_pertub = 0+ self.hunyuan_1_5 = hunyuan_1_5+ self.target_dtype = torch.bfloat16++ def encode_prompt_1_5(+ self,+ prompt,+ name,+ device,+ num_videos_per_prompt,+ do_classifier_free_guidance,+ negative_prompt=None,+ prompt_embeds: Optional[torch.Tensor] = None,+ attention_mask: Optional[torch.Tensor] = None,+ negative_prompt_embeds: Optional[torch.Tensor] = None,+ negative_attention_mask: Optional[torch.Tensor] = None,+ clip_skip: Optional[int] = None,+ text_encoder: Optional[TextEncoder] = None,+ data_type: Optional[str] = "image",+ pixel_value_llava: Optional[torch.Tensor] = None,+ uncond_pixel_value_llava: Optional[torch.Tensor] = None,+ lora_scale: Optional[float] = None,+ semantic_images=None+ ):+ r"""+ Encodes the prompt into text encoder hidden states.++ Args:+ prompt (`str` or `List[str]`, *optional*):+ prompt to be encoded+ device: (`torch.device`):+ torch device+ num_videos_per_prompt (`int`):+ number of videos that should be generated per prompt+ do_classifier_free_guidance (`bool`):+ whether to use classifier free guidance or not+ negative_prompt (`str` or `List[str]`, *optional*):+ The prompt or prompts not to guide the video generation. If not defined, one has to pass+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is+ less than `1`).+ prompt_embeds (`torch.Tensor`, *optional*):+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not+ provided, text embeddings will be generated from `prompt` input argument.+ attention_mask (`torch.Tensor`, *optional*):+ negative_prompt_embeds (`torch.Tensor`, *optional*):+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input+ argument.+ negative_attention_mask (`torch.Tensor`, *optional*):+ Attention mask for negative prompt embeddings.+ clip_skip (`int`, *optional*):+ Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that+ the output of the pre-final layer will be used for computing the prompt embeddings.+ text_encoder (TextEncoder, *optional*):+ Text encoder to use. If None, uses the pipeline's default text encoder.+ data_type (`str`, *optional*):+ Type of data being encoded. Defaults to "image".+ """+ if text_encoder is None:+ text_encoder = self.text_encoder++ if prompt is not None and isinstance(prompt, str):+ batch_size = 1+ elif prompt is not None and isinstance(prompt, list):+ batch_size = len(prompt)+ else:+ batch_size = prompt_embeds.shape[0]++ if prompt_embeds is None:++ text_inputs = text_encoder.text2tokens(prompt, data_type=data_type, max_length=text_encoder.max_length)+ if clip_skip is None:+ prompt_outputs = text_encoder.encode(+ text_inputs, data_type=data_type, device=device+ )+ prompt_embeds = prompt_outputs.hidden_state+ else:+ prompt_outputs = text_encoder.encode(+ text_inputs,+ output_hidden_states=True,+ data_type=data_type,+ device=device,+ )+ # Access the `hidden_states` first, that contains a tuple of+ # all the hidden states from the encoder layers. Then index into+ # the tuple to access the hidden states from the desired layer.+ prompt_embeds = prompt_outputs.hidden_states_list[-(clip_skip + 1)]+ # We also need to apply the final LayerNorm here to not mess with the+ # representations. The `last_hidden_states` that we typically use for+ # obtaining the final prompt representations passes through the LayerNorm+ # layer.+ prompt_embeds = text_encoder.model.text_model.final_layer_norm(+ prompt_embeds+ )++ attention_mask = prompt_outputs.attention_mask+ if attention_mask is not None:+ attention_mask = attention_mask.to(device)+ bs_embed, seq_len = attention_mask.shape+ attention_mask = attention_mask.repeat(1, num_videos_per_prompt)+ attention_mask = attention_mask.view(+ bs_embed * num_videos_per_prompt, seq_len+ )++ if text_encoder is not None:+ prompt_embeds_dtype = text_encoder.dtype+ elif self.transformer is not None:+ prompt_embeds_dtype = self.transformer.dtype+ else:+ prompt_embeds_dtype = prompt_embeds.dtype++ prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)++ if prompt_embeds.ndim == 2:+ bs_embed, _ = prompt_embeds.shape+ # duplicate text embeddings for each generation per prompt, using mps friendly method+ prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt)+ prompt_embeds = prompt_embeds.view(bs_embed * num_videos_per_prompt, -1)+ else:+ bs_embed, seq_len, _ = prompt_embeds.shape+ # duplicate text embeddings for each generation per prompt, using mps friendly method+ prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)+ prompt_embeds = prompt_embeds.view(+ bs_embed * num_videos_per_prompt, seq_len, -1+ )++ # get unconditional embeddings for classifier free guidance+ if do_classifier_free_guidance and negative_prompt_embeds is None:+ uncond_tokens: List[str]+ if negative_prompt is None:+ uncond_tokens = [""] * batch_size+ elif prompt is not None and type(prompt) is not type(negative_prompt):+ raise TypeError(+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="+ f" {type(prompt)}."+ )+ elif isinstance(negative_prompt, str):+ uncond_tokens = [negative_prompt]+ elif batch_size != len(negative_prompt):+ raise ValueError(+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"+ " the batch size of `prompt`."+ )+ else:+ uncond_tokens = negative_prompt++ uncond_input = text_encoder.text2tokens(uncond_tokens, data_type=data_type, max_length=text_encoder.max_length)++ negative_prompt_outputs = text_encoder.encode(uncond_input, data_type=data_type, is_uncond=True)+ negative_prompt_embeds = negative_prompt_outputs.hidden_state++ negative_attention_mask = negative_prompt_outputs.attention_mask+ if negative_attention_mask is not None:+ negative_attention_mask = negative_attention_mask.to(device)+ _, seq_len = negative_attention_mask.shape+ negative_attention_mask = negative_attention_mask.repeat(1, num_videos_per_prompt)+ negative_attention_mask = negative_attention_mask.view(batch_size * num_videos_per_prompt, seq_len)++ if do_classifier_free_guidance:+ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method+ seq_len = negative_prompt_embeds.shape[1]++ negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)++ if negative_prompt_embeds.ndim == 2:+ negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_videos_per_prompt)+ negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_videos_per_prompt, -1)+ else:+ negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_videos_per_prompt, 1)+ negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)++ return (+ prompt_embeds,+ negative_prompt_embeds,+ attention_mask,+ negative_attention_mask,+ )+def encode_prompt(self,@@ -622,8 +813,6 @@ class HunyuanVideoPipeline(DiffusionPipeline):i2v_condition_type=None,i2v_stability=True,):- if i2v_mode and i2v_condition_type == "latent_concat":- num_channels_latents = (num_channels_latents - 1) // 2shape = (batch_size,num_channels_latents,@@ -754,6 +943,137 @@ class HunyuanVideoPipeline(DiffusionPipeline):def interrupt(self):return self._interrupt+ @staticmethod+ def get_byt5_text_tokens(byt5_tokenizer, byt5_max_length, text_prompt):+ """+ Tokenize text prompt for byT5 model.++ Args:+ byt5_tokenizer: The byT5 tokenizer.+ byt5_max_length: Maximum sequence length for tokenization.+ text_prompt: Text prompt string to tokenize.++ Returns:+ tuple[torch.Tensor, torch.Tensor]:+ - input_ids: Tokenized input IDs.+ - attention_mask: Attention mask tensor.+ """+ byt5_text_inputs = byt5_tokenizer(+ text_prompt,+ padding="max_length",+ max_length=byt5_max_length,+ truncation=True,+ add_special_tokens=True,+ return_tensors="pt",+ )++ return byt5_text_inputs.input_ids, byt5_text_inputs.attention_mask++ def _extract_glyph_texts(self, prompt):+ """+ Extract glyph texts from prompt using regex pattern.++ Args:+ prompt: Input prompt string containing quoted text.++ Returns:+ List[str]: List of extracted glyph texts (deduplicated if multiple).+ """+ pattern = r'\"(.*?)\"|“(.*?)”'+ matches = re.findall(pattern, prompt)+ result = [match[0] or match[1] for match in matches]+ result = list(dict.fromkeys(result)) if len(result) > 1 else result+ return result++ def _process_single_byt5_prompt(self, prompt_text, device):+ """+ Process a single prompt for byT5 encoding.++ Args:+ prompt_text: The prompt text to process.+ device: Target device for tensors.++ Returns:+ tuple[torch.Tensor, torch.Tensor]:+ - byt5_embeddings: Encoded embeddings tensor.+ - byt5_mask: Attention mask tensor.+ """+ byt5_embeddings = torch.zeros((1, self.byt5_max_length, 1472), device=device)+ byt5_mask = torch.zeros((1, self.byt5_max_length), device=device, dtype=torch.int64)++ glyph_texts = self._extract_glyph_texts(prompt_text)++ if len(glyph_texts) > 0:+ text_styles = [{'color': None, 'font-family': None} for _ in range(len(glyph_texts))]+ formatted_text = self.prompt_format.format_prompt(glyph_texts, text_styles)++ text_ids, text_mask = self.get_byt5_text_tokens(+ self.byt5_tokenizer, self.byt5_max_length, formatted_text+ )+ text_ids = text_ids.to(device=device)+ text_mask = text_mask.to(device=device)++ byt5_outputs = self.byt5_model(text_ids, attention_mask=text_mask.float())+ byt5_embeddings = byt5_outputs[0]+ byt5_mask = text_mask++ return byt5_embeddings, byt5_mask++ def _prepare_byt5_embeddings(self, prompts, device):+ """+ Prepare byT5 embeddings for both positive and negative prompts.++ Args:+ prompts: List of prompt strings or single prompt string.+ device: Target device for tensors.++ Returns:+ dict: Dictionary containing:+ - "byt5_text_states": Combined embeddings tensor.+ - "byt5_text_mask": Combined attention mask tensor.+ Returns empty dict if glyph_byT5_v2 is disabled.+ """+ # if not self.config.glyph_byT5_v2:+ # return {}++ if isinstance(prompts, str):+ prompt_list = [prompts]+ elif isinstance(prompts, list):+ prompt_list = prompts+ else:+ raise ValueError("prompts must be str or list of str")++ positive_embeddings = []+ positive_masks = []+ negative_embeddings = []+ negative_masks = []++ for prompt in prompt_list:+ pos_emb, pos_mask = self._process_single_byt5_prompt(prompt, device)+ positive_embeddings.append(pos_emb)+ positive_masks.append(pos_mask)++ if self.do_classifier_free_guidance:+ neg_emb, neg_mask = self._process_single_byt5_prompt("", device)+ negative_embeddings.append(neg_emb)+ negative_masks.append(neg_mask)++ byt5_positive = torch.cat(positive_embeddings, dim=0)+ byt5_positive_mask = torch.cat(positive_masks, dim=0)++ if self.do_classifier_free_guidance:+ byt5_negative = torch.cat(negative_embeddings, dim=0)+ byt5_negative_mask = torch.cat(negative_masks, dim=0)++ byt5_embeddings = torch.cat([byt5_negative, byt5_positive], dim=0)+ byt5_masks = torch.cat([byt5_negative_mask, byt5_positive_mask], dim=0)+ else:+ byt5_embeddings = byt5_positive+ byt5_masks = byt5_positive_mask++ return byt5_embeddings, byt5_masks++@torch.no_grad()@replace_example_docstring(EXAMPLE_DOC_STRING)def __call__(@@ -944,6 +1264,7 @@ class HunyuanVideoPipeline(DiffusionPipeline):uncond_ref_latents = None+# 0. Default height and width to unet# height = height or self.transformer.config.sample_size * self.vae_scale_factor# width = width or self.transformer.config.sample_size * self.vae_scale_factor@@ -982,6 +1303,16 @@ class HunyuanVideoPipeline(DiffusionPipeline):self._clip_skip = clip_skipself._cross_attention_kwargs = cross_attention_kwargs++ vae_precision = "fp16" # torch.float16+ precision = "bf16" # torch.bfloat16++ disable_autocast = True+ target_dtype = PRECISION_TO_TYPE[precision]+ autocast_enabled = target_dtype != torch.float32 and not disable_autocast+ vae_dtype = self.vae._model_dtype # PRECISION_TO_TYPE[vae_precision]+ vae_autocast_enabled = vae_dtype != torch.float32 and False+# 2. Define call parametersif prompt is not None and isinstance(prompt, str):batch_size = 1@@ -998,13 +1329,14 @@ class HunyuanVideoPipeline(DiffusionPipeline):if self.cross_attention_kwargs is not Noneelse None)-+ hunyuan_1_5= self.hunyuan_1_5+ encode_prompt = self.encode_prompt_1_5 if hunyuan_1_5 else self.encode_prompt(prompt_embeds,negative_prompt_embeds,prompt_mask,negative_prompt_mask,- ) = self.encode_prompt(+ ) = encode_prompt(prompt,name,device,@@ -1028,7 +1360,7 @@ class HunyuanVideoPipeline(DiffusionPipeline):negative_prompt_embeds_2,prompt_mask_2,negative_prompt_mask_2,- ) = self.encode_prompt(+ ) = encode_prompt(prompt,name,device,@@ -1050,6 +1382,12 @@ class HunyuanVideoPipeline(DiffusionPipeline):prompt_mask_2 = Nonenegative_prompt_mask_2 = None+ if self.byt5_model is not None:+ byt5_embeddings, byt5_masks = self._prepare_byt5_embeddings(prompt, device)+ else:+ byt5_embeddings = byt5_masks = None++# For classifier free guidance, we need to do two forward passes.# Here we concatenate the unconditional and text embeddings into a single batch# to avoid doing two forward passes@@ -1069,7 +1407,7 @@ class HunyuanVideoPipeline(DiffusionPipeline):prompt_mask[0] = torch.cat([torch.ones((1, prompt_mask[0].sum() - 575)).to(prompt_mask),torch.zeros((1, prompt_mask.shape[1] - prompt_mask[0].sum() + 575)).to(prompt_mask)], dim=1)- if bg_latents is not None:+ if bg_latents is not None and self.transformer.custom:bg_latents = torch.cat([bg_latents, bg_latents], dim=0)if audio_prompts is not None:@@ -1110,10 +1448,27 @@ class HunyuanVideoPipeline(DiffusionPipeline):prompt_embeds = prompt_embeds.to(torch.bfloat16)if prompt_embeds_2 != None:prompt_embeds_2 = prompt_embeds_2.to(torch.bfloat16)- # if prompt_mask != None:- # prompt_mask = prompt_mask.to(torch.bfloat16)++ if self.upsampler is not None:+ # lq_latents = rearrange(bg_latents, "b c f h w -> (b f) c h w")+ # bg_latents = None+ # lq_latents = torch.nn.functional.interpolate(lq_latents, size=[height // 16, width // 16], mode="bilinear", align_corners=False)+ # lq_latents = rearrange(lq_latents, "(b f) c h w -> b c f h w", b=1)+ # lq_latents = self.upsampler(lq_latents.to(target_dtype))+ lq_mask_latents = torch.ones_like(bg_latents[:, :1])+ extra_latent_frames = video_length - bg_latents.shape[2]+ if extra_latent_frames:+ bg_latents = torch.cat([bg_latents, torch.zeros_like(bg_latents[:, :, :extra_latent_frames])], dim=2)+ lq_mask_latents = torch.cat([lq_mask_latents, torch.zeros_like(bg_latents[:, :1, :extra_latent_frames])], dim=2)+ lq_latents = self.upsampler(bg_latents.to(target_dtype))+ noise_scale = 0.7+ lq_latents = (1-noise_scale) * lq_latents + noise_scale * torch.randn_like(lq_latents)+ lq_latents = torch.cat( [lq_latents, lq_mask_latents ], dim=1 )+ zero_lq_latents = torch.zeros_like(lq_latents)+# 5. Prepare latent variables- num_channels_latents = self.transformer.config.in_channels++ num_channels_latents = self.transformer.config.out_channelslatents, original_latents, noise, timesteps = self.prepare_latents(batch_size * num_videos_per_prompt,num_channels_latents,@@ -1133,19 +1488,25 @@ class HunyuanVideoPipeline(DiffusionPipeline):i2v_stability=i2v_stability)- if i2v_mode and i2v_condition_type == "latent_concat":- if img_latents.shape[2] == 1:- img_latents_concat = img_latents.repeat(1, 1, video_length, 1, 1)+ if i2v_condition_type == "latent_concat" :+ if i2v_mode:+ if img_latents.shape[2] == 1:+ img_latents_concat = img_latents.repeat(1, 1, video_length, 1, 1)+ else:+ img_latents_concat = img_latents+ img_latents_concat[:, :, 1:] = 0+ mask_concat = torch.ones(img_latents_concat.shape[0], 1, *img_latents_concat.shape[2:]).to(device=img_latents.device)+ mask_concat[:, :, 1:] = 0else:- img_latents_concat = img_latents- img_latents_concat[:, :, 1:, ...] = 0+ img_latents_concat = torch.zeros(latents.shape[0], latents.shape[1], *latents.shape[2:]).to(latents.device)+ mask_concat = torch.zeros(img_latents_concat.shape[0], 1, *latents.shape[2:]).to(device=latents.device)+- i2v_mask = torch.zeros(video_length)- i2v_mask[0] = 1+ vision_states = None+ if semantic_images is not None and self.vision_encoder is not None:+ vision_states = self.vision_encoder.encode_images(np.array(semantic_images))+ vision_states = vision_states.last_hidden_state.to(device=device, dtype=target_dtype)- mask_concat = torch.ones(img_latents_concat.shape[0], 1, img_latents_concat.shape[2], img_latents_concat.shape[3],- img_latents_concat.shape[4]).to(device=img_latents.device)- mask_concat[:, :, 1:, ...] = 0# 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipelineextra_step_kwargs = self.prepare_extra_func_kwargs(@@ -1153,15 +1514,7 @@ class HunyuanVideoPipeline(DiffusionPipeline):{"generator": generator, "eta": eta},)- vae_precision = "fp16" # torch.float16- precision = "bf16" # torch.bfloat16- disable_autocast = True-- target_dtype = PRECISION_TO_TYPE[precision]- autocast_enabled = target_dtype != torch.float32 and not disable_autocast- vae_dtype = self.vae._model_dtype # PRECISION_TO_TYPE[vae_precision]- vae_autocast_enabled = vae_dtype != torch.float32 and not disable_autocast# 7. Denoising loopnum_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order@@ -1224,12 +1577,22 @@ class HunyuanVideoPipeline(DiffusionPipeline):offload.set_step_no_for_lora(self.transformer, i)if self.interrupt:continue- if i2v_mode and i2v_condition_type == "token_replace":- latents = torch.concat([img_latents, latents[:, :, 1:, :, :]], dim=2)- # expand the latents if we are doing classifier free guidance- if i2v_mode and i2v_condition_type == "latent_concat":- latent_model_input = torch.concat([latents, img_latents_concat, mask_concat], dim=1)+ timesteps_r = None+ if self.upsampler is not None:+ if i == len(timesteps) - 1:+ timesteps_r = torch.tensor([0.0], device=self.device)+ else:+ timesteps_r = timesteps[i + 1]+ timesteps_r = timesteps_r.repeat(latents.shape[0])++ if i2v_condition_type == "latent_concat":+ if self.upsampler is not None:+ latent_model_input = torch.concat([latents, img_latents_concat, mask_concat, zero_lq_latents if t < 1000 * noise_scale else lq_latents], dim=1)+ else:+ latent_model_input = torch.concat([latents, img_latents_concat, mask_concat], dim=1)+ elif i2v_condition_type == "token_replace":+ latent_model_input = torch.concat([img_latents, latents[:, :, 1:, :, :]], dim=2)else:latent_model_input = latents@@ -1284,6 +1647,10 @@ class HunyuanVideoPipeline(DiffusionPipeline):bg_latents=bg_latents[j].unsqueeze(0) if bg_latents!=None else None,audio_prompts=audio_prompts[j].unsqueeze(0) if audio_prompts!=None else None,audio_strength=audio_strength,+ vision_states= vision_states,+ byt5_text_states=byt5_embeddings[j].unsqueeze(0),+ byt5_text_mask=byt5_masks[j].unsqueeze(0),+ timesteps_r=timesteps_r,callback = callback,)if self._interrupt:@@ -1296,11 +1663,6 @@ class HunyuanVideoPipeline(DiffusionPipeline):noise_pred_ip = ret[0]ret = Noneelse:- # if self.do_classifier_free_guidance:- # noise_pred_uncond = self.transformer(latent_model_input[:1], t_expand[:1], ref_latents=ref_latents[:1], text_states=prompt_embeds[:1], text_mask=prompt_mask[:1], text_states_2=prompt_embeds_2[:1], freqs_cos=freqs_cis[0],freqs_sin=freqs_cis[1], guidance=guidance_expand,return_dict=True)['x']- # noise_pred_text = self.transformer(latent_model_input[1:], t_expand[1:], ref_latents=ref_latents[1:], text_states=prompt_embeds[1:], text_mask=prompt_mask[1:], text_states_2=prompt_embeds_2[1:], freqs_cos=freqs_cis[0],freqs_sin=freqs_cis[1], guidance=guidance_expand,return_dict=True)['x']- # noise_pred = torch.cat([noise_pred_uncond, noise_pred_text], dim=0)- # else:ret = self.transformer( # For an input image (129, 192, 336) (1, 256, 256)latent_model_input, # [2, 16, 33, 24, 42]t_expand, # [2]@@ -1316,6 +1678,10 @@ class HunyuanVideoPipeline(DiffusionPipeline):bg_latents=bg_latents,audio_prompts=audio_prompts,audio_strength=audio_strength,+ vision_states= vision_states,+ byt5_text_states=byt5_embeddings,+ byt5_text_mask=byt5_masks,+ timesteps_r=timesteps_r,callback = callback,)if self._interrupt:@@ -1378,10 +1744,10 @@ class HunyuanVideoPipeline(DiffusionPipeline):if self.interrupt:return [None]- # if load_latent:- # latents = torch.load("latent.pt")- # else:- # torch.save(latents, "latent.pt")+ if load_latent:+ latents = torch.load("latent.pt")+ else:+ torch.save(latents, "latent.pt")if mask_latents is not None:@@ -1433,11 +1799,9 @@ class HunyuanVideoPipeline(DiffusionPipeline):# we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16image = image.cpu().float()- if i2v_mode and i2v_condition_type == "latent_concat":- image = image[:, :, 4:, :, :]+ # if i2v_mode and i2v_condition_type == "latent_concat":+ # image = image[:, :, 4:, :, :]- # Offload all models- self.maybe_free_model_hooks()if not return_dict:return imagediff --git a/models/hyvideo/hunyuan.py b/models/hyvideo/hunyuan.pyindex 91aaf7f..98ef2e2 100644--- a/models/hyvideo/hunyuan.py+++ b/models/hyvideo/hunyuan.py@@ -4,6 +4,7 @@ import randomimport functoolsfrom typing import List, Optional, Tuple, Unionfrom shared.utils import files_locator as fl+import jsonfrom pathlib import Pathfrom einops import rearrangeimport torch@@ -12,6 +13,8 @@ from .constants import PROMPT_TEMPLATE, NEGATIVE_PROMPT, PRECISION_TO_TYPE, NEGAfrom .vae import load_vaefrom .modules import load_modelfrom .text_encoder import TextEncoder+from .text_encoder.text_encoder_1_5 import TextEncoder as TextEncoder_1_5+from .utils.data_utils import align_to, get_closest_ratio, generate_crop_size_listfrom .modules.posemb_layers import get_nd_rotary_pos_embed, get_nd_rotary_pos_embed_newfrom .diffusion.schedulers import FlowMatchDiscreteScheduler@@ -27,6 +30,10 @@ from transformers import WhisperModelfrom transformers import AutoFeatureExtractorfrom .data_kits.face_align import AlignImageimport librosa+from .text_encoder.byT5 import load_glyph_byT5_v2+from .text_encoder.byT5.format_prompt import MultilingualPromptFormat+from .vision_encoder import VisionEncoder+from .modules.upsample import SRTo720pUpsampler, SRTo1080pUpsamplerdef get_audio_feature(feature_extractor, audio_path, duration):audio_input, sampling_rate = librosa.load(audio_path, duration=duration, sr=16000)@@ -64,177 +71,6 @@ def pad_image(crop_img, size, color=(255, 255, 255), resize_ratio=1):return crop_img---# def _merge_input_ids_with_image_features(self, image_features, inputs_embeds, input_ids, attention_mask, labels):-# num_images, num_image_patches, embed_dim = image_features.shape-# batch_size, sequence_length = input_ids.shape-# left_padding = not torch.sum(input_ids[:, -1] == torch.tensor(self.pad_token_id))-# # 1. Create a mask to know where special image tokens are-# special_image_token_mask = input_ids == self.config.image_token_index-# num_special_image_tokens = torch.sum(special_image_token_mask, dim=-1)-# # Compute the maximum embed dimension-# max_embed_dim = (num_special_image_tokens.max() * (num_image_patches - 1)) + sequence_length-# batch_indices, non_image_indices = torch.where(input_ids != self.config.image_token_index)--# # 2. Compute the positions where text should be written-# # Calculate new positions for text tokens in merged image-text sequence.-# # `special_image_token_mask` identifies image tokens. Each image token will be replaced by `nb_text_tokens_per_images - 1` text tokens.-# # `torch.cumsum` computes how each image token shifts subsequent text token positions.-# # - 1 to adjust for zero-based indexing, as `cumsum` inherently increases indices by one.-# new_token_positions = torch.cumsum((special_image_token_mask * (num_image_patches - 1) + 1), -1) - 1-# nb_image_pad = max_embed_dim - 1 - new_token_positions[:, -1]-# if left_padding:-# new_token_positions += nb_image_pad[:, None] # offset for left padding-# text_to_overwrite = new_token_positions[batch_indices, non_image_indices]--# # 3. Create the full embedding, already padded to the maximum position-# final_embedding = torch.zeros(-# batch_size, max_embed_dim, embed_dim, dtype=inputs_embeds.dtype, device=inputs_embeds.device-# )-# final_attention_mask = torch.zeros(-# batch_size, max_embed_dim, dtype=attention_mask.dtype, device=inputs_embeds.device-# )-# if labels is not None:-# final_labels = torch.full(-# (batch_size, max_embed_dim), self.config.ignore_index, dtype=input_ids.dtype, device=input_ids.device-# )-# # In case the Vision model or the Language model has been offloaded to CPU, we need to manually-# # set the corresponding tensors into their correct target device.-# target_device = inputs_embeds.device-# batch_indices, non_image_indices, text_to_overwrite = (-# batch_indices.to(target_device),-# non_image_indices.to(target_device),-# text_to_overwrite.to(target_device),-# )-# attention_mask = attention_mask.to(target_device)--# # 4. Fill the embeddings based on the mask. If we have ["hey" "<image>", "how", "are"]-# # we need to index copy on [0, 577, 578, 579] for the text and [1:576] for the image features-# final_embedding[batch_indices, text_to_overwrite] = inputs_embeds[batch_indices, non_image_indices]-# final_attention_mask[batch_indices, text_to_overwrite] = attention_mask[batch_indices, non_image_indices]-# if labels is not None:-# final_labels[batch_indices, text_to_overwrite] = labels[batch_indices, non_image_indices]--# # 5. Fill the embeddings corresponding to the images. Anything that is not `text_positions` needs filling (#29835)-# image_to_overwrite = torch.full(-# (batch_size, max_embed_dim), True, dtype=torch.bool, device=inputs_embeds.device-# )-# image_to_overwrite[batch_indices, text_to_overwrite] = False-# image_to_overwrite &= image_to_overwrite.cumsum(-1) - 1 >= nb_image_pad[:, None].to(target_device)--# if image_to_overwrite.sum() != image_features.shape[:-1].numel():-# raise ValueError(-# f"The input provided to the model are wrong. The number of image tokens is {torch.sum(special_image_token_mask)} while"-# f" the number of image given to the model is {num_images}. This prevents correct indexing and breaks batch generation."-# )--# final_embedding[image_to_overwrite] = image_features.contiguous().reshape(-1, embed_dim).to(target_device)-# final_attention_mask |= image_to_overwrite-# position_ids = (final_attention_mask.cumsum(-1) - 1).masked_fill_((final_attention_mask == 0), 1)--# # 6. Mask out the embedding at padding positions, as we later use the past_key_value value to determine the non-attended tokens.-# batch_indices, pad_indices = torch.where(input_ids == self.pad_token_id)-# indices_to_mask = new_token_positions[batch_indices, pad_indices]--# final_embedding[batch_indices, indices_to_mask] = 0--# if labels is None:-# final_labels = None--# return final_embedding, final_attention_mask, final_labels, position_ids--# def patched_llava_forward(-# self,-# input_ids: torch.LongTensor = None,-# pixel_values: torch.FloatTensor = None,-# attention_mask: Optional[torch.Tensor] = None,-# position_ids: Optional[torch.LongTensor] = None,-# past_key_values: Optional[List[torch.FloatTensor]] = None,-# inputs_embeds: Optional[torch.FloatTensor] = None,-# vision_feature_layer: Optional[int] = None,-# vision_feature_select_strategy: Optional[str] = None,-# labels: Optional[torch.LongTensor] = None,-# use_cache: Optional[bool] = None,-# output_attentions: Optional[bool] = None,-# output_hidden_states: Optional[bool] = None,-# return_dict: Optional[bool] = None,-# cache_position: Optional[torch.LongTensor] = None,-# num_logits_to_keep: int = 0,-# ):-# from transformers.models.llava.modeling_llava import LlavaCausalLMOutputWithPast---# output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions-# output_hidden_states = (-# output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states-# )-# return_dict = return_dict if return_dict is not None else self.config.use_return_dict-# vision_feature_layer = (-# vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer-# )-# vision_feature_select_strategy = (-# vision_feature_select_strategy-# if vision_feature_select_strategy is not None-# else self.config.vision_feature_select_strategy-# )--# if (input_ids is None) ^ (inputs_embeds is not None):-# raise ValueError("You must specify exactly one of input_ids or inputs_embeds")--# if pixel_values is not None and inputs_embeds is not None:-# raise ValueError(-# "You cannot specify both pixel_values and inputs_embeds at the same time, and must specify either one"-# )--# if inputs_embeds is None:-# inputs_embeds = self.get_input_embeddings()(input_ids)--# image_features = None-# if pixel_values is not None:-# image_features = self.get_image_features(-# pixel_values=pixel_values,-# vision_feature_layer=vision_feature_layer,-# vision_feature_select_strategy=vision_feature_select_strategy,-# )---# inputs_embeds, attention_mask, labels, position_ids = self._merge_input_ids_with_image_features(-# image_features, inputs_embeds, input_ids, attention_mask, labels-# )-# cache_position = torch.arange(attention_mask.shape[1], device=attention_mask.device)---# outputs = self.language_model(-# attention_mask=attention_mask,-# position_ids=position_ids,-# past_key_values=past_key_values,-# inputs_embeds=inputs_embeds,-# use_cache=use_cache,-# output_attentions=output_attentions,-# output_hidden_states=output_hidden_states,-# return_dict=return_dict,-# cache_position=cache_position,-# num_logits_to_keep=num_logits_to_keep,-# )--# logits = outputs[0]--# loss = None--# if not return_dict:-# output = (logits,) + outputs[1:]-# return (loss,) + output if loss is not None else output--# return LlavaCausalLMOutputWithPast(-# loss=loss,-# logits=logits,-# past_key_values=outputs.past_key_values,-# hidden_states=outputs.hidden_states,-# attentions=outputs.attentions,-# image_hidden_states=image_features if pixel_values is not None else None,-# )-def adapt_model(model, audio_block_name):modules_dict= { k: m for k, m in model.named_modules()}for model_layer, avatar_layer in model.double_stream_map.items():@@ -282,6 +118,7 @@ class Inference(object):i2v,custom,avatar,+ hunyuan_1_5,enable_cfg,vae,vae_kwargs,@@ -292,6 +129,12 @@ class Inference(object):feature_extractor=None,wav2vec=None,align_instance=None,+ byt5_model= None,+ byt5_tokenizer= None,+ byt5_max_length= 256,+ vision_encoder=None,+ upsampler=None,+ prompt_format=None,device=None,):self.i2v = i2v@@ -311,11 +154,20 @@ class Inference(object):self.wav2vec=wav2vecself.align_instance=align_instance- self.device = "cuda"+ self.byt5_model= byt5_model+ self.byt5_tokenizer= byt5_tokenizer+ self.byt5_max_length= byt5_max_length+ self.vision_encoder=vision_encoder+ self.upsampler=upsampler++ self.prompt_format=prompt_format+ self.hunyuan_1_5 = hunyuan_1_5+ self.device = "cuda"+@classmethod- def from_pretrained(cls, model_filepath, model_type, base_model_type, text_encoder_filepath, dtype = torch.bfloat16, VAE_dtype = torch.float16, mixed_precision_transformer =torch.bfloat16 , quantizeTransformer = False, save_quantized = False, **kwargs):+ def from_pretrained(cls, model_filepath, model_type, base_model_type, model_def, text_encoder_filepath, dtype = torch.bfloat16, VAE_dtype = torch.float16, mixed_precision_transformer =torch.bfloat16 , quantizeTransformer = False, save_quantized = False, **kwargs):device = "cuda"@@ -327,6 +179,7 @@ class Inference(object):text_len = 512latent_channels = 16precision = "bf16"+ dtype =torch.bfloat16vae_precision = "fp32" if VAE_dtype == torch.float32 else "bf16"embedded_cfg_scale = 6filepath = model_filepath[0]@@ -334,7 +187,9 @@ class Inference(object):i2v_mode = Falsecustom = Falsecustom_audio = False- avatar = False+ avatar = False+ hunyuan_1_5 = False+ any_upsampler= Falseif base_model_type == "hunyuan_i2v":model_id = "HYVideo-T/2"i2v_condition_type = "token_replace"@@ -353,11 +208,30 @@ class Inference(object):model_id = "HYVideo-T/2-avatar"text_len = 256avatar = True+ elif base_model_type == "hunyuan_1_5_t2v":+ model_id = "HYVideo-1_5"+ hunyuan_1_5 = True+ elif base_model_type == "hunyuan_1_5_i2v":+ model_id = "HYVideo-1_5"+ hunyuan_1_5 = True+ i2v_condition_type = "latent_concat"+ i2v_mode = True+ elif base_model_type == "hunyuan_1_5_upsampler":+ model_id = "HYVideo-1_5-upsampler"+ hunyuan_1_5 = True+ i2v_condition_type = "latent_concat"+ # i2v_mode = True+ any_upsampler= True+else:model_id = "HYVideo-T/2-cfgdistill"-- if i2v_mode and i2v_condition_type == "latent_concat":+ out_channels = latent_channels+ if hunyuan_1_5:+ in_channels = 98 if any_upsampler else 65+ out_channels = 32+ image_embed_interleave = 1+ elif i2v_mode and i2v_condition_type == "latent_concat":in_channels = latent_channels * 2 + 1image_embed_interleave = 2elif i2v_mode and i2v_condition_type == "token_replace":@@ -366,12 +240,11 @@ class Inference(object):else:in_channels = latent_channelsimage_embed_interleave = 1- out_channels = latent_channelspinToMemory = kwargs.pop("pinToMemory", False)partialPinning = kwargs.pop("partialPinning", False)factor_kwargs = kwargs | {"device": "meta", "dtype": PRECISION_TO_TYPE[precision]}- if embedded_cfg_scale and i2v_mode:+ if embedded_cfg_scale and i2v_mode and not hunyuan_1_5:factor_kwargs["guidance_embed"] = Truemodel = load_model(@@ -384,47 +257,122 @@ class Inference(object):from mmgp import offload- # model = Inference.load_state_dict(args, model, model_filepath)- # model_filepath ="c:/temp/hc/mp_rank_00_model_states_video.pt"- offload.load_model_data(model, model_filepath, do_quantize= quantizeTransformer and not save_quantized, pinToMemory = pinToMemory, partialPinning = partialPinning)- pass- # offload.save_model(model, "hunyuan_video_avatar_edit_720_bf16.safetensors")- # offload.save_model(model, "hunyuan_video_avatar_edit_720_quanto_bf16_int8.safetensors", do_quantize= True)- if save_quantized:- from wgp import save_quantized_model- save_quantized_model(model, model_type, filepath, dtype, None)+ source = model_def.get("source", None)+ kwargs_light = dict(do_quantize= quantizeTransformer and not save_quantized, pinToMemory = pinToMemory, partialPinning = partialPinning)+ if source is not None:+ from wgp import save_model+ offload.load_model_data(model, fl.locate_file(source), **kwargs_light)+ else:+ offload.load_model_data(model, model_filepath, **kwargs_light)+model.mixed_precision = mixed_precision_transformer- if model.mixed_precision :- model._lock_dtype = torch.float32- model.lock_layers_dtypes(torch.float32)+ model.lock_layers_dtypes(torch.float32 if model.mixed_precision else dtype)+ offload.change_dtype(model, dtype, True)+ model.eval().requires_grad_(False)+model.eval()+ if not source is None:+ save_model(model, model_type, dtype, None, submodel_no= 1)++ if save_quantized:+ from wgp import save_quantized_model+ save_quantized_model(model, model_type, filepath, dtype, None)++ byt5_tokenizer = byt5_model = prompt_format = vision_encoder = upsampler = None+# ============================= Build extra models ========================- # VAE- if custom or avatar:- vae_configpath = fl.locate_file("hunyuan_video_custom_VAE_config.json")- vae_filepath = fl.locate_file("hunyuan_video_custom_VAE_fp32.safetensors")- # elif avatar:- # vae_configpath = "ckpts/config_vae_avatar.json"- # vae_filepath = "ckpts/vae_avatar.pt"+ # VAE+ byt5_max_length = 0+ if hunyuan_1_5:+ byt5_max_length = 256++ multilingual_prompt_format_font_path = fl.locate_file("Glyph-SDXL-v2/color_idx.json")+ multilingual_prompt_format_color_path = fl.locate_file("Glyph-SDXL-v2/multilingual_10-lang_idx.json")+ byt5_args = dict(+ byT5_google_path=fl.locate_folder("Glyph-SDXL-v2/byt5-small"),+ byT5_ckpt_path=fl.locate_file("Glyph-SDXL-v2/byt5-small/byt5_model.safetensors"),+ # byT5_ckpt_path=fl.locate_folder("Glyph-SDXL-v2/byt5-small"),++ multilingual_prompt_format_color_path= multilingual_prompt_format_font_path,+ multilingual_prompt_format_font_path= multilingual_prompt_format_color_path,+ byt5_max_length=byt5_max_length+ )++ byt5_tokenizer, byt5_model = load_glyph_byT5_v2(byt5_args, device=device)+ prompt_format = MultilingualPromptFormat(+ font_path=multilingual_prompt_format_font_path,+ color_path=multilingual_prompt_format_color_path+ )++ if i2v_mode:+ vision_encoder = VisionEncoder( vision_encoder_type="siglip", vision_encoder_precision="fp16", vision_encoder_path=fl.locate_folder("siglip_vision_model"), processor_type=None, processor_path=None, output_key=None, logger=None, device=device, )+ vision_encoder.vision_num_semantic_tokens=729+ vision_encoder.vision_states_dim=1152++ if any_upsampler:+ if model_def.get("upsampler", "720") == "720":+ upsampler_cls = SRTo720pUpsampler+ upsampler_checkpoint= fl.locate_file("hunyuan_video_1.5_upsampler_720p_sr_distilled.safetensors")+ upsampler_config= fl.locate_file("hunyuan_video_1.5_upsampler_720p_sr_distilled_config.json")+ upsampler_target_size = [720, 1280]+ else:+ upsampler_cls = SRTo1080pUpsampler+ upsampler_checkpoint= fl.locate_file("hunyuan_video_1.5_upsampler_1080p_sr_distilled.safetensors")+ upsampler_config= fl.locate_file("hunyuan_video_1.5_upsampler_1080p_sr_distilled_config.json")+ upsampler_target_size = [1088, 1920]+ with open(upsampler_config, 'r', encoding='utf-8') as f:+ config = json.load(f)+ upsampler = upsampler_cls(**config)+ upsampler.target_size = upsampler_target_size+ offload.load_model_data(upsampler, upsampler_checkpoint, writable_tensors= False)++ vae_configpath = fl.locate_file("hunyuan_video_1_5_VAE.json")+ vae_filepath = fl.locate_file("hunyuan_video_1_5_VAE_fp32.safetensors")+ from .vae.hunyuanvideo_15_vae import AutoencoderKLConv3D+ with open(vae_configpath, 'r', encoding='utf-8') as f:+ config = json.load(f)+ from accelerate import init_empty_weights+ with init_empty_weights():+ vae = AutoencoderKLConv3D(**config)+ offload.load_model_data(vae, vae_filepath )+ vae = vae.to("cpu")+ s_ratio = t_ratio = 1+ vae._model_dtype = torch.float32 if VAE_dtype == torch.float32 else torch.float16+ vae.to(vae._model_dtype)else:- vae_configpath = fl.locate_file("hunyuan_video_VAE_config.json")- vae_filepath = fl.locate_file("hunyuan_video_VAE_fp32.safetensors")+ if custom or avatar:+ vae_configpath = fl.locate_file("hunyuan_video_custom_VAE_config.json")+ vae_filepath = fl.locate_file("hunyuan_video_custom_VAE_fp32.safetensors")+ # elif avatar:+ # vae_configpath = "ckpts/config_vae_avatar.json"+ # vae_filepath = "ckpts/vae_avatar.pt"+ else:+ vae_configpath = fl.locate_file("hunyuan_video_VAE_config.json")+ vae_filepath = fl.locate_file("hunyuan_video_VAE_fp32.safetensors")- # config = AutoencoderKLCausal3D.load_config("ckpts/hunyuan_video_VAE_config.json")- # config = AutoencoderKLCausal3D.load_config("c:/temp/hvae/config_vae.json")+ # config = AutoencoderKLCausal3D.load_config("ckpts/hunyuan_video_VAE_config.json")+ # config = AutoencoderKLCausal3D.load_config("c:/temp/hvae/config_vae.json")- vae, _, s_ratio, t_ratio = load_vae( "884-16c-hy", vae_path= vae_filepath, vae_config_path= vae_configpath, vae_precision= vae_precision, device= "cpu", )+ vae, _, s_ratio, t_ratio = load_vae( "884-16c-hy", vae_path= vae_filepath, vae_config_path= vae_configpath, vae_precision= vae_precision, device= "cpu", )- vae._model_dtype = torch.float32 if VAE_dtype == torch.float32 else (torch.float16 if avatar else torch.bfloat16)- vae._model_dtype = torch.float32 if VAE_dtype == torch.float32 else torch.bfloat16+ vae._model_dtype = torch.float32 if VAE_dtype == torch.float32 else (torch.float16 if avatar else torch.bfloat16)+ vae._model_dtype = torch.float32 if VAE_dtype == torch.float32 else torch.bfloat16vae_kwargs = {"s_ratio": s_ratio, "t_ratio": t_ratio}enable_cfg = False+ text_encoder_cls = TextEncoder# Text encoder- if i2v_mode:+ if hunyuan_1_5:+ text_encoder_cls = TextEncoder_1_5+ text_encoder = "llm"+ tokenizer = "llm"+ prompt_template = "li-dit-encode-image-json"+ prompt_template_video = "li-dit-encode-video-json"+ enable_cfg = True+ elif i2v_mode:text_encoder = "llm-i2v"tokenizer = "llm-i2v"prompt_template = "dit-llm-encode-i2v"@@ -456,7 +404,7 @@ class Inference(object):prompt_template_video = PROMPT_TEMPLATE[prompt_template_video] if prompt_template_video is not None else None- text_encoder = TextEncoder(+ text_encoder = text_encoder_cls(text_encoder_type=text_encoder,max_length=max_length,text_encoder_precision="fp16",@@ -472,14 +420,17 @@ class Inference(object):text_encoder_path = text_encoder_filepath)- text_encoder_2 = TextEncoder(- text_encoder_type="clipL",- max_length=77,- text_encoder_precision="fp16",- tokenizer_type="clipL",- reproduce=True,- device="cpu",- )+ if hunyuan_1_5:+ text_encoder_2 = None+ else:+ text_encoder_2 = TextEncoder(+ text_encoder_type="clipL",+ max_length=77,+ text_encoder_precision="fp16",+ tokenizer_type="clipL",+ reproduce=True,+ device="cpu",+ )feature_extractor = Nonewav2vec = None@@ -501,6 +452,7 @@ class Inference(object):i2v=i2v_mode,custom=custom,avatar=avatar,+ hunyuan_1_5 = hunyuan_1_5,enable_cfg = enable_cfg,vae=vae,vae_kwargs=vae_kwargs,@@ -510,6 +462,12 @@ class Inference(object):feature_extractor=feature_extractor,wav2vec=wav2vec,align_instance=align_instance,+ byt5_model= byt5_model,+ byt5_tokenizer= byt5_tokenizer,+ byt5_max_length= byt5_max_length,+ vision_encoder=vision_encoder,+ upsampler=upsampler,+ prompt_format=prompt_format,device=device,)@@ -521,6 +479,7 @@ class HunyuanVideoSampler(Inference):i2v,custom,avatar,+ hunyuan_1_5,enable_cfg,vae,vae_kwargs,@@ -531,12 +490,19 @@ class HunyuanVideoSampler(Inference):feature_extractor=None,wav2vec=None,align_instance=None,+ byt5_model= None,+ byt5_tokenizer= None,+ byt5_max_length= 256,+ vision_encoder=None,+ upsampler=None,+ prompt_format=None,device=0,):super().__init__(i2v,custom,avatar,+ hunyuan_1_5,enable_cfg,vae,vae_kwargs,@@ -547,6 +513,12 @@ class HunyuanVideoSampler(Inference):feature_extractor=feature_extractor,wav2vec=wav2vec,align_instance=align_instance,+ byt5_model= byt5_model,+ byt5_tokenizer= byt5_tokenizer,+ byt5_max_length= byt5_max_length,+ vision_encoder=vision_encoder,+ upsampler=upsampler,+ prompt_format=prompt_format,device=device,)@@ -554,10 +526,17 @@ class HunyuanVideoSampler(Inference):self.enable_cfg = enable_cfgself.pipeline = self.load_diffusion_pipeline(avatar = self.avatar,+ hunyuan_1_5 = self.hunyuan_1_5,vae=self.vae,text_encoder=self.text_encoder,text_encoder_2=self.text_encoder_2,model=self.model,+ byt5_model= self.byt5_model,+ byt5_tokenizer= self.byt5_tokenizer,+ byt5_max_length= self.byt5_max_length,+ vision_encoder=self.vision_encoder,+ upsampler=self.upsampler,+ prompt_format=self.prompt_format,device=self.device,)@@ -566,6 +545,33 @@ class HunyuanVideoSampler(Inference):else:self.default_negative_prompt = NEGATIVE_PROMPT+++ # create_sr_pipeline = True+ # if create_sr_pipeline:+ # from .diffusion.pipelines.pipeline_hunyuan_video_15_sr import HunyuanVideo_1_5_SR_Pipeline+ # transformer, upsampler = self.load_sr_transformer_upsampler(cached_folder, sr_version, transformer_dtype=transformer_dtype, device=device)++ # pipeline.sr_pipeline = HunyuanVideo_1_5_SR_Pipeline(+ # vae=self.vae,+ # transformer=transformer,+ # text_encoder=self.text_encoder,+ # scheduler=self.scheduler,+ # upsampler=upsampler,+ # text_encoder_2=self.text_encoder_2,+ # progress_bar_config=None,+ # glyph_byT5_v2=self.config.glyph_byT5_v2,+ # byt5_model=self.byt5_model,+ # byt5_tokenizer=self.byt5_tokenizer,+ # byt5_max_length=self.byt5_max_length,+ # prompt_format=self.prompt_format,+ # execution_device='cuda',+ # vision_encoder=self.vision_encoder,+ # )++++@propertydef _interrupt(self):return self.pipeline._interrupt@@ -577,11 +583,18 @@ class HunyuanVideoSampler(Inference):def load_diffusion_pipeline(self,avatar,+ hunyuan_1_5,vae,text_encoder,text_encoder_2,model,scheduler=None,+ byt5_model= None,+ byt5_tokenizer= None,+ byt5_max_length= 256,+ vision_encoder=None,+ upsampler=None,+ prompt_format=None,device=None,progress_bar_config=None,#data_type="video",@@ -611,6 +624,13 @@ class HunyuanVideoSampler(Inference):transformer=model,scheduler=scheduler,progress_bar_config=progress_bar_config,+ byt5_model= byt5_model,+ byt5_tokenizer= byt5_tokenizer,+ byt5_max_length= byt5_max_length,+ vision_encoder=vision_encoder,+ upsampler=upsampler,+ prompt_format=prompt_format,+ hunyuan_1_5 = hunyuan_1_5,)return pipeline@@ -649,17 +669,17 @@ class HunyuanVideoSampler(Inference):)return freqs_cos, freqs_sin- def get_rotary_pos_embed(self, video_length, height, width, enable_riflex = False):+ def get_rotary_pos_embed(self, video_length, height, width, enable_riflex = False, spatial_compression = 8):target_ndim = 3ndim = 5 - 2# 884vae = "884-16c-hy"if "884" in vae:- latents_size = [(video_length - 1) // 4 + 1, height // 8, width // 8]+ latents_size = [(video_length - 1) // 4 + 1, height // spatial_compression, width // spatial_compression]elif "888" in vae:- latents_size = [(video_length - 1) // 8 + 1, height // 8, width // 8]+ latents_size = [(video_length - 1) // 8 + 1, height // spatial_compression, width // spatial_compression]else:- latents_size = [video_length, height // 8, width // 8]+ latents_size = [video_length, height // spatial_compression, width // spatial_compression]if isinstance(self.model.patch_size, int):assert all(s % self.model.patch_size == 0 for s in latents_size), (@@ -722,7 +742,6 @@ class HunyuanVideoSampler(Inference):num_videos_per_prompt=1,image_start=None,enable_RIFLEx = False,- i2v_condition_type: str = "token_replace",i2v_stability=True,VAE_tile_size = None,joint_pass = False,@@ -733,11 +752,14 @@ class HunyuanVideoSampler(Inference):):if VAE_tile_size != None:- self.vae.tile_sample_min_tsize = VAE_tile_size["tile_sample_min_tsize"]- self.vae.tile_latent_min_tsize = VAE_tile_size["tile_latent_min_tsize"]- self.vae.tile_sample_min_size = VAE_tile_size["tile_sample_min_size"]- self.vae.tile_latent_min_size = VAE_tile_size["tile_latent_min_size"]- self.vae.tile_overlap_factor = VAE_tile_size["tile_overlap_factor"]+ if self.hunyuan_1_5:+ self.vae.set_tile_sample_min_size(VAE_tile_size["tile_sample_min_size"], VAE_tile_size["tile_overlap_factor"])+ else:+ self.vae.tile_sample_min_tsize = VAE_tile_size["tile_sample_min_tsize"]+ self.vae.tile_latent_min_tsize = VAE_tile_size["tile_latent_min_tsize"]+ self.vae.tile_sample_min_size = VAE_tile_size["tile_sample_min_size"]+ self.vae.tile_latent_min_size = VAE_tile_size["tile_latent_min_size"]+ self.vae.tile_overlap_factor = VAE_tile_size["tile_overlap_factor"]self.vae.enable_tiling()i2v_mode= self.i2v_mode@@ -844,21 +866,31 @@ class HunyuanVideoSampler(Inference):semantic_images = Nonedenoise_strength = 0ip_cfg_scale = 0+ i2v_condition_type= "token_replace" if i2v_mode else None+ first_frame = None if input_video is None else input_video[:, 0]+ if self.upsampler is not None:+ if first_frame is None:+ first_frame = input_frames[:, 0]+ i2v_mode = True+ target_frame_num=min(input_frames.shape[1], target_frame_num)+if i2v_mode:- semantic_images = convert_tensor_to_image(image_start)- semantic_image_pixel_values = image_start.unsqueeze(0).unsqueeze(2).to(self.device)+ semantic_images = convert_tensor_to_image(first_frame)+ semantic_image_pixel_values = first_frame.unsqueeze(0).unsqueeze(2).to(self.device)with torch.autocast(device_type="cuda", dtype=torch.float16, enabled=True):img_latents = self.pipeline.vae.encode(semantic_image_pixel_values).latent_dist.mode() # B, C, F, H, Wimg_latents.mul_(self.pipeline.vae.config.scaling_factor)- target_height, target_width = image_start.shape[1:]+ target_width, target_height = semantic_images.size+ if self.hunyuan_1_5: i2v_condition_type= "latent_concat"# ========================================================================# Build Rope freqs# ========================================================================+ spatial_compression = 16 if self.hunyuan_1_5 else 8if input_ref_images == None:- freqs_cos, freqs_sin = self.get_rotary_pos_embed(target_frame_num, target_height, target_width, enable_RIFLEx)+ freqs_cos, freqs_sin = self.get_rotary_pos_embed(target_frame_num, target_height, target_width, enable_RIFLEx, spatial_compression =spatial_compression)else:if self.avatar:concat_dict = {'mode': 'timecat', 'bias': -1}@@ -886,39 +918,43 @@ class HunyuanVideoSampler(Inference):else:pixel_value_llava, uncond_pixel_value_llava, pixel_value_ref = DataPreprocess().get_batch(input_ref_images, (target_width, target_height), pad = self.custom)- ref_latents, uncond_audio_prompts, audio_prompts, face_masks, motion_exp, motion_pose = None, None, None, None, None, None--- bg_latents = None- if input_video != None:- pixel_value_bg = input_video.unsqueeze(0)- pixel_value_mask = torch.zeros_like(input_video).unsqueeze(0)- if input_frames != None:- pixel_value_video_bg = input_frames.unsqueeze(0) #.permute(-1,0,1,2).unsqueeze(0).float()- # pixel_value_video_bg = pixel_value_video_bg.div_(127.5).add_(-1.)- # pixel_value_video_mask = input_masks.unsqueeze(-1).repeat(1,1,1,3).permute(-1,0,1,2).unsqueeze(0).float()- pixel_value_video_mask = input_masks.repeat(3,1,1,1).unsqueeze(0)- if input_video != None:- pixel_value_bg = torch.cat([pixel_value_bg, pixel_value_video_bg], dim=2)- pixel_value_mask = torch.cat([ pixel_value_mask, pixel_value_video_mask], dim=2)- else:- pixel_value_bg = pixel_value_video_bg- pixel_value_mask = pixel_value_video_mask- pixel_value_video_mask, pixel_value_video_bg = None, None- if input_video != None or input_frames != None:- if pixel_value_bg.shape[2] < frame_num:- padding_shape = list(pixel_value_bg.shape[0:2]) + [frame_num-pixel_value_bg.shape[2]] + list(pixel_value_bg.shape[3:])- pixel_value_bg = torch.cat([pixel_value_bg, torch.full(padding_shape, -1, dtype=pixel_value_bg.dtype, device= pixel_value_bg.device ) ], dim=2)- # pixel_value_mask = torch.cat([ pixel_value_mask, torch.full(padding_shape, 255, dtype=pixel_value_mask.dtype, device= pixel_value_mask.device ) ], dim=2)- pixel_value_mask = torch.cat([ pixel_value_mask, torch.full(padding_shape, 1, dtype=pixel_value_mask.dtype, device= pixel_value_mask.device ) ], dim=2)+ ref_latents= uncond_audio_prompts= audio_prompts= face_masks= motion_exp= motion_pose = bg_latents = None+ if self.upsampler is not None:+ pixel_value_bg = input_frames.unsqueeze(0)bg_latents = self.vae.encode(pixel_value_bg).latent_dist.sample()- pixel_value_mask = pixel_value_mask.mul_(2).add_(-1.) # unmasked pixels is -1 (no 0 as usual) and masked is 1- mask_latents = self.vae.encode(pixel_value_mask).latent_dist.sample()- bg_latents = torch.cat([bg_latents, mask_latents], dim=1)bg_latents.mul_(self.vae.config.scaling_factor)- if self.avatar:+ elif self.custom:+ if input_video != None:+ pixel_value_bg = input_video.unsqueeze(0)+ pixel_value_mask = torch.zeros_like(input_video).unsqueeze(0)+ if input_frames != None:+ pixel_value_video_bg = input_frames.unsqueeze(0) #.permute(-1,0,1,2).unsqueeze(0).float()+ # pixel_value_video_bg = pixel_value_video_bg.div_(127.5).add_(-1.)+ # pixel_value_video_mask = input_masks.unsqueeze(-1).repeat(1,1,1,3).permute(-1,0,1,2).unsqueeze(0).float()+ pixel_value_video_mask = input_masks.repeat(3,1,1,1).unsqueeze(0)+ if input_video != None:+ pixel_value_bg = torch.cat([pixel_value_bg, pixel_value_video_bg], dim=2)+ pixel_value_mask = torch.cat([ pixel_value_mask, pixel_value_video_mask], dim=2)+ else:+ pixel_value_bg = pixel_value_video_bg+ pixel_value_mask = pixel_value_video_mask+ pixel_value_video_mask, pixel_value_video_bg = None, None+ if input_video != None or input_frames != None:+ if pixel_value_bg.shape[2] < frame_num:+ padding_shape = list(pixel_value_bg.shape[0:2]) + [frame_num-pixel_value_bg.shape[2]] + list(pixel_value_bg.shape[3:])+ pixel_value_bg = torch.cat([pixel_value_bg, torch.full(padding_shape, -1, dtype=pixel_value_bg.dtype, device= pixel_value_bg.device ) ], dim=2)+ # pixel_value_mask = torch.cat([ pixel_value_mask, torch.full(padding_shape, 255, dtype=pixel_value_mask.dtype, device= pixel_value_mask.device ) ], dim=2)+ pixel_value_mask = torch.cat([ pixel_value_mask, torch.full(padding_shape, 1, dtype=pixel_value_mask.dtype, device= pixel_value_mask.device ) ], dim=2)++ bg_latents = self.vae.encode(pixel_value_bg).latent_dist.sample()+ pixel_value_mask = pixel_value_mask.mul_(2).add_(-1.) # unmasked pixels is -1 (no 0 as usual) and masked is 1+ mask_latents = self.vae.encode(pixel_value_mask).latent_dist.sample()+ bg_latents = torch.cat([bg_latents, mask_latents], dim=1)+ bg_latents.mul_(self.vae.config.scaling_factor)++ elif self.avatar:if n_prompt == None or len(n_prompt) == 0:n_prompt = "Aerial view, aerial view, overexposed, low quality, deformation, a poor composition, bad hands, bad teeth, bad eyes, bad limbs, distortion, blurring, Lens changes"@@ -1021,7 +1057,7 @@ class HunyuanVideoSampler(Inference):enable_tiling=True,i2v_mode=i2v_mode,i2v_condition_type=i2v_condition_type,- i2v_stability=i2v_stability,+ i2v_stability=i2v_stability and not self.hunyuan_1_5,img_latents=img_latents,semantic_images=semantic_images,joint_pass = joint_pass,diff --git a/models/hyvideo/hunyuan_handler.py b/models/hyvideo/hunyuan_handler.pyindex 28c8531..aa3ae9f 100644--- a/models/hyvideo/hunyuan_handler.py+++ b/models/hyvideo/hunyuan_handler.py@@ -1,11 +1,19 @@import torchfrom shared.utils import files_locator as fl-def get_hunyuan_text_encoder_filename(text_encoder_quantization):- if text_encoder_quantization =="int8":- text_encoder_filename = "llava-llama-3-8b/llava-llama-3-8b-v1_1_vlm_quanto_int8.safetensors"+def test_hunyuan_1_5(base_model_type):+ return base_model_type in ["hunyuan_1_5_t2v", "hunyuan_1_5_i2v", "hunyuan_1_5_upsampler"]++def get_hunyuan_text_encoder_filename(base_model_type, text_encoder_quantization):+ if test_hunyuan_1_5(base_model_type):+ text_encoder_filename = "Qwen2.5-VL-7B-Instruct/Qwen2.5-VL-7B-Instruct_bf16.safetensors"+ if text_encoder_quantization =="int8":+ text_encoder_filename = text_encoder_filename.replace("bf16", "quanto_bf16_int8")else:- text_encoder_filename = "llava-llama-3-8b/llava-llama-3-8b-v1_1_vlm_fp16.safetensors"+ if text_encoder_quantization =="int8":+ text_encoder_filename = "llava-llama-3-8b/llava-llama-3-8b-v1_1_vlm_quanto_int8.safetensors"+ else:+ text_encoder_filename = "llava-llama-3-8b/llava-llama-3-8b-v1_1_vlm_fp16.safetensors"return fl.locate_file(text_encoder_filename, True)@@ -35,10 +43,15 @@ class family_handler():if base_model_type in ["hunyuan_avatar", "hunyuan_custom_audio"]:fps = 25- elif base_model_type in ["hunyuan", "hunyuan_i2v", "hunyuan_custom_edit", "hunyuan_custom"]:+ elif base_model_type in ["hunyuan", "hunyuan_i2v", "hunyuan_custom_edit", "hunyuan_custom"] or test_hunyuan_1_5(base_model_type):fps = 24else:fps = 16++ if test_hunyuan_1_5(base_model_type):+ extra_model_def["group"] = "hunyuan_1_5"++extra_model_def["fps"] = fpsextra_model_def["frames_minimum"] = 5extra_model_def["frames_steps"] = 4@@ -50,9 +63,9 @@ class family_handler():else:extra_model_def["guidance_max_phases"] = 1- extra_model_def["cfg_star"] = base_model_type in [ "hunyuan_avatar", "hunyuan_custom_audio", "hunyuan_custom_edit", "hunyuan_custom"]- extra_model_def["tea_cache"] = True- extra_model_def["mag_cache"] = True+ extra_model_def["cfg_star"] = base_model_type in [ "hunyuan_avatar", "hunyuan_custom_audio", "hunyuan_custom_edit", "hunyuan_custom"] or test_hunyuan_1_5(base_model_type)+ extra_model_def["tea_cache"] = not test_hunyuan_1_5(base_model_type)+ extra_model_def["mag_cache"] = not test_hunyuan_1_5(base_model_type)if base_model_type in ["hunyuan_custom_edit"]:extra_model_def["guide_preprocessing"] = {@@ -83,22 +96,44 @@ class family_handler():extra_model_def["one_image_ref_needed"] = True- if base_model_type in ["hunyuan_i2v"]:- extra_model_def["image_prompt_types_allowed"] = "S"+ if base_model_type in ["hunyuan_i2v", "hunyuan_1_5_i2v"]:+ extra_model_def["image_prompt_types_allowed"] = "SVL"++ if base_model_type in ["hunyuan_1_5_upsampler"]:+ extra_model_def["image_prompt_types_allowed"] = "TVL"+ extra_model_def["guide_custom_choices"] = {+ "choices":[+ ("Upsample", "V"),+ ],+ "default": "V",+ "letters_filter": "V",+ "label": "Type of Process",+ "scale": 3,+ "show_label" : False,+ "visible": False,+ }+ if test_hunyuan_1_5(base_model_type) and base_model_type not in ["hunyuan_1_5_t2v"]:+ extra_model_def["sliding_window"] = True+ extra_model_def["sliding_window_defaults"] = { "overlap_min" : 1, "overlap_max" : 1, "overlap_step": 0, "overlap_default": 1}+return extra_model_def@staticmethoddef query_supported_types():- return ["hunyuan", "hunyuan_i2v", "hunyuan_custom", "hunyuan_custom_audio", "hunyuan_custom_edit", "hunyuan_avatar"]+ return ["hunyuan", "hunyuan_i2v", "hunyuan_custom", "hunyuan_custom_audio", "hunyuan_custom_edit", "hunyuan_avatar", "hunyuan_1_5_t2v", "hunyuan_1_5_i2v", "hunyuan_1_5_upsampler"]@staticmethoddef query_family_maps():models_eqv_map = {+ "hunyuan_1_5_t2v": "hunyuan",+ "hunyuan_1_5_i2v": "hunyuan_i2v",}models_comp_map = {"hunyuan_custom": ["hunyuan_custom_edit", "hunyuan_custom_audio"],+ "hunyuan": ["hunyuan_1_5_t2v"],+ "hunyuan_i2v": ["hunyuan_1_5_i2v"],}return models_eqv_map, models_comp_map@@ -109,18 +144,36 @@ class family_handler():@staticmethoddef query_family_infos():- return {"hunyuan":(20, "Hunyuan Video")}+ return {"hunyuan":(20, "Hunyuan Video"), "hunyuan_1_5":(21, "Hunyuan Video 1.5")}@staticmethoddef get_rgb_factors(base_model_type ):from shared.RGB_factors import get_rgb_factors- latent_rgb_factors, latent_rgb_factors_bias = get_rgb_factors("hunyuan")+ latent_rgb_factors, latent_rgb_factors_bias = get_rgb_factors("hunyuan", sub_family = "hunyuan1.5" if test_hunyuan_1_5(base_model_type) else "")return latent_rgb_factors, latent_rgb_factors_bias@staticmethoddef query_model_files(computeList, base_model_type, model_filename, text_encoder_quantization):- text_encoder_filename = get_hunyuan_text_encoder_filename(text_encoder_quantization)- return {+ text_encoder_filename = get_hunyuan_text_encoder_filename(base_model_type,text_encoder_quantization)++ if test_hunyuan_1_5(base_model_type):+ download_def = [{+ "repoId" : "DeepBeepMeep/Qwen_image",+ "sourceFolderList" : ["", "Qwen2.5-VL-7B-Instruct"],+ "fileList" : [ ["qwen_vae.safetensors", "qwen_vae_config.json"], ["merges.txt", "tokenizer_config.json", "config.json", "vocab.json", "video_preprocessor_config.json", "preprocessor_config.json", "chat_template.json"] + computeList(text_encoder_filename) ]+ },+ {+ "repoId" : "DeepBeepMeep/HunyuanVideo1.5",+ "sourceFolderList" : [ "Glyph-SDXL-v2", "Glyph-SDXL-v2/byt5-small", "siglip_vision_model", "" ],+ "fileList" :[ ["color_idx.json", "multilingual_10-lang_idx.json"] ,+ [ "config.json", "model.safetensors", "byt5_model.safetensors"], # "byt5_model.pt", "pytorch_model.bin"],+ [ "model.safetensors", "config.json", "preprocessor_config.json"],+ [ "hunyuan_video_1_5_VAE_fp32.safetensors", "hunyuan_video_1_5_VAE.json"] + computeList(model_filename) ,+ ]+ } ]++ else:+ download_def= {"repoId" : "DeepBeepMeep/HunyuanVideo","sourceFolderList" : [ "llava-llama-3-8b", "clip_vit_large_patch14", "whisper-tiny" , "det_align", "" ],"fileList" :[ ["config.json", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json", "preprocessor_config.json"] + computeList(text_encoder_filename) ,@@ -129,7 +182,8 @@ class family_handler():["detface.pt"],[ "hunyuan_video_720_quanto_int8_map.json", "hunyuan_video_custom_VAE_fp32.safetensors", "hunyuan_video_custom_VAE_config.json", "hunyuan_video_VAE_fp32.safetensors", "hunyuan_video_VAE_config.json" , "hunyuan_video_720_quanto_int8_map.json" ] + computeList(model_filename)]- }+ }+ return download_def@staticmethoddef load_model(model_filename, model_type = None, base_model_type = None, model_def = None, quantizeTransformer = False, text_encoder_quantization = None, dtype = torch.bfloat16, VAE_dtype = torch.float32, mixed_precision_transformer = False, save_quantized = False, submodel_no_list = None, override_text_encoder = None):@@ -140,7 +194,8 @@ class family_handler():model_filepath = model_filename,model_type = model_type,base_model_type = base_model_type,- text_encoder_filepath = get_hunyuan_text_encoder_filename(text_encoder_quantization) if override_text_encoder is None else override_text_encoder,+ model_def = model_def,+ text_encoder_filepath = get_hunyuan_text_encoder_filename(base_model_type, text_encoder_quantization) if override_text_encoder is None else override_text_encoder,dtype = dtype,quantizeTransformer = quantizeTransformer,VAE_dtype = VAE_dtype,@@ -149,8 +204,16 @@ class family_handler():)pipe = { "transformer" : hunyuan_model.model, "text_encoder" : hunyuan_model.text_encoder, "text_encoder_2" : hunyuan_model.text_encoder_2, "vae" : hunyuan_model.vae }+ if hunyuan_model.byt5_model is not None:+ pipe["byt5_model"] = hunyuan_model.byt5_model++ if hunyuan_model.vision_encoder is not None:+ pipe["vision_encoder"] = hunyuan_model.vision_encoder++ if hunyuan_model.upsampler is not None:+ pipe["upsampler"] = hunyuan_model.upsampler- if hunyuan_model.wav2vec != None:+ if hunyuan_model.wav2vec is not None:pipe["wav2vec"] = hunyuan_model.wav2vec@@ -183,6 +246,11 @@ class family_handler():audio_prompt_type += "A"ui_defaults["audio_prompt_type"] = audio_prompt_type++ if settings_version < 2.41:+ if base_model_type in ["hunyuan_i2v"]:+ ui_defaults["sliding_window_overlap"] = 1+@staticmethod@@ -225,3 +293,15 @@ class family_handler():"video_prompt_type": "KI","audio_prompt_type": "A",})++ if base_model_type in ["hunyuan_1_5_i2v","hunyuan_i2v"]:+ ui_defaults.update({+ "image_prompt_type": "S",+ "sliding_window_overlap" : 1,+ })++ if base_model_type in ["hunyuan_1_5_upsampler"]:+ ui_defaults.update({+ "video_prompt_type": "V",+ "sliding_window_overlap" : 1,+ })diff --git a/models/hyvideo/modules/embed_layers.py b/models/hyvideo/modules/embed_layers.pyindex f4f6785..3188561 100644--- a/models/hyvideo/modules/embed_layers.py+++ b/models/hyvideo/modules/embed_layers.py@@ -59,6 +59,22 @@ class PatchEmbed(nn.Module):x = self.norm(x)return x, shape+class VisionProjection(torch.nn.Module):++ def __init__(self, input_dim, output_dim):+ super().__init__()++ self.proj = torch.nn.Sequential(+ torch.nn.LayerNorm(input_dim),+ torch.nn.Linear(input_dim, input_dim),+ torch.nn.GELU(),+ torch.nn.Linear(input_dim, output_dim),+ torch.nn.LayerNorm(output_dim)+ )+++ def forward(self, vision_embeds):+ return self.proj(vision_embeds)class TextProjection(nn.Module):"""diff --git a/models/hyvideo/modules/models.py b/models/hyvideo/modules/models.pyindex 4cdce4a..9a266bc 100644--- a/models/hyvideo/modules/models.py+++ b/models/hyvideo/modules/models.py@@ -10,7 +10,7 @@ from diffusers.configuration_utils import ConfigMixin, register_to_configfrom .activation_layers import get_activation_layerfrom .norm_layers import get_norm_layer-from .embed_layers import TimestepEmbedder, PatchEmbed, TextProjection+from .embed_layers import TimestepEmbedder, PatchEmbed, TextProjection, VisionProjectionfrom .attenion import attention, parallel_attention, get_cu_seqlensfrom .posemb_layers import apply_rotary_embfrom .mlp_layers import MLP, MLPEmbedder, FinalLayer@@ -18,6 +18,7 @@ from .modulate_layers import ModulateDiT, modulate, modulate_ , apply_gate, applfrom .token_refiner import SingleTokenRefinerimport numpy as npfrom mmgp import offload+from ..text_encoder.byT5 import ByT5Mapperfrom shared.attention import pay_attentionfrom .audio_adapters import AudioProjNet2, PerceiverAttentionCA@@ -48,7 +49,8 @@ class MMDoubleStreamBlock(nn.Module):qkv_bias: bool = False,dtype: Optional[torch.dtype] = None,device: Optional[torch.device] = None,- attention_mode: str = "sdpa",+ attention_mode: str = "sdpa",+ pre_split_qkv = False,):factory_kwargs = {"device": device, "dtype": dtype}super().__init__()@@ -68,10 +70,15 @@ class MMDoubleStreamBlock(nn.Module):self.img_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, **factory_kwargs)-- self.img_attn_qkv = nn.Linear(- hidden_size, hidden_size * 3, bias=qkv_bias, **factory_kwargs- )+ self.pre_split_qkv = pre_split_qkv+ if pre_split_qkv:+ self.img_attn_q = nn.Linear( hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs )+ self.img_attn_k = nn.Linear( hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs )+ self.img_attn_v = nn.Linear( hidden_size, hidden_size, bias=qkv_bias, **factory_kwargs )+ else:+ self.img_attn_qkv = nn.Linear(+ hidden_size, hidden_size * 3, bias=qkv_bias, **factory_kwargs+ )qk_norm_layer = get_norm_layer(qk_norm_type)self.img_attn_q_norm = (qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs)@@ -108,9 +115,14 @@ class MMDoubleStreamBlock(nn.Module):hidden_size, elementwise_affine=False, eps=1e-6, **factory_kwargs)- self.txt_attn_qkv = nn.Linear(- hidden_size, hidden_size * 3, bias=qkv_bias, **factory_kwargs- )+ if pre_split_qkv:+ self.txt_attn_q = nn.Linear( hidden_size, hidden_size , bias=qkv_bias, **factory_kwargs )+ self.txt_attn_k = nn.Linear( hidden_size, hidden_size , bias=qkv_bias, **factory_kwargs )+ self.txt_attn_v = nn.Linear( hidden_size, hidden_size , bias=qkv_bias, **factory_kwargs )+ else:+ self.txt_attn_qkv = nn.Linear(+ hidden_size, hidden_size * 3, bias=qkv_bias, **factory_kwargs+ )self.txt_attn_q_norm = (qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs)if qk_norm@@ -224,12 +236,19 @@ class MMDoubleStreamBlock(nn.Module):txt_modulated = self.txt_norm1(txt)modulate_(txt_modulated, shift=txt_mod1_shift, scale=txt_mod1_scale )- txt_qkv = self.txt_attn_qkv(txt_modulated)- del txt_modulated- txt_q, txt_k, txt_v = rearrange(- txt_qkv, "B L (K H D) -> K B L H D", K=3, H=self.heads_num- )- del txt_qkv+ if self.pre_split_qkv:+ new_shape = (*txt_modulated.shape[:-1], self.heads_num, -1)+ txt_q = self.txt_attn_q(txt_modulated).reshape(*new_shape)+ txt_k = self.txt_attn_k(txt_modulated).reshape(*new_shape)+ txt_v = self.txt_attn_v(txt_modulated).reshape(*new_shape)+ del txt_modulated+ else:+ txt_qkv = self.txt_attn_qkv(txt_modulated)+ del txt_modulated+ txt_q, txt_k, txt_v = rearrange(+ txt_qkv, "B L (K H D) -> K B L H D", K=3, H=self.heads_num+ )+ del txt_qkv# Apply QK-Norm if needed.self.txt_attn_q_norm.apply_(txt_q).to(txt_v)self.txt_attn_k_norm.apply_(txt_k).to(txt_v)@@ -591,6 +610,15 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):audio_condition: bool = False,avatar = False,custom = False,+ vision_projection = False,+ glyph_byT5_v2 = False,+ use_meanflow = False,+ use_cond_type_embedding = False,+ text_pool_type = True,+ vision_states_dim: int = 1280,+ text_states_dim: int = 4096,+ text_states_dim_2: int = 768,+ pre_split_qkv = False,):factory_kwargs = {"device": device, "dtype": dtype}super().__init__()@@ -615,9 +643,20 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):self.use_attention_mask = use_attention_maskself.text_projection = text_projection- self.text_states_dim = 4096- self.text_states_dim_2 = 768-+ self.text_states_dim = text_states_dim+ self.text_states_dim_2 = text_states_dim_2+ self.vision_states_dim = vision_states_dim+ self.text_pool_type = text_pool_type++ self.glyph_byT5_v2 = glyph_byT5_v2+ if self.glyph_byT5_v2:+ self.byt5_in = ByT5Mapper(+ in_dim=1472,+ out_dim=2048,+ hidden_dim=2048,+ out_dim1=hidden_size,+ use_residual=False+ )if hidden_size % heads_num != 0:raise ValueError(f"Hidden size {hidden_size} must be divisible by heads_num {heads_num}"@@ -635,6 +674,14 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):self.patch_size, self.in_channels, self.hidden_size, **factory_kwargs)+ # Vision projection+ if vision_projection == "linear":+ self.vision_in = VisionProjection(+ input_dim=self.vision_states_dim, output_dim=self.hidden_size+ )+ else:+ self.vision_in = None+# text projectionif self.text_projection == "linear":self.txt_in = TextProjection(@@ -658,8 +705,10 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):)# text modulation- self.vector_in = MLPEmbedder(- self.text_states_dim_2, self.hidden_size, **factory_kwargs+ self.vector_in = (+ MLPEmbedder(+ self.config.text_states_dim_2, self.hidden_size, **factory_kwargs+ ) if self.text_pool_type is not None else None)# guidance modulation@@ -671,6 +720,12 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):else None)+ self.time_r_in = (+ TimestepEmbedder(self.hidden_size, get_activation_layer("silu"), **factory_kwargs)+ if use_meanflow+ else None+ )+# double blocksself.double_blocks = nn.ModuleList([@@ -683,6 +738,7 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):qk_norm_type=qk_norm_type,qkv_bias=qkv_bias,attention_mode = attention_mode,+ pre_split_qkv = pre_split_qkv,**factory_kwargs,)for _ in range(mm_double_blocks_depth)@@ -764,6 +820,17 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):PerceiverAttentionCA(dim=3072, dim_head=1024, heads=33) for _ in range(len(self.double_stream_list) + len(self.single_stream_list))]))+ if use_cond_type_embedding:+ self.cond_type_embedding = nn.Embedding(3, self.hidden_size)+ self.cond_type_embedding.weight.data.fill_(0)+ assert self.glyph_byT5_v2, "text type embedding is only used when glyph_byT5_v2 is True"+ assert vision_projection is not None, "text type embedding is only used when vision_projection is not None"+ # 0: text_encoder feature+ # 1: byt5 feature+ # 2: vision_encoder feature+ else:+ self.cond_type_embedding = None+def lock_layers_dtypes(self, dtype = torch.float32):@@ -848,6 +915,42 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):print(f"Mag Cache, best threshold found:{best_threshold:0.2f} with gain x{num_inference_steps/(target_nb_steps - best_signed_diff):0.2f} for a target of x{speed_factor}")return best_threshold+ def reorder_txt_token(self, byt5_txt, txt, byt5_text_mask, text_mask, zero_feat=False, is_reorder=True):+ if is_reorder:+ reorder_txt = []+ reorder_mask = []+ for i in range(text_mask.shape[0]):+ byt5_text_mask_i = byt5_text_mask[i].bool()+ text_mask_i = text_mask[i].bool()++ byt5_txt_i = byt5_txt[i]+ txt_i = txt[i]+ if zero_feat:+ # When using block mask with approximate computation, set pad to zero to reduce error+ pad_byt5 = torch.zeros_like(byt5_txt_i[~byt5_text_mask_i])+ pad_text = torch.zeros_like(txt_i[~text_mask_i])+ reorder_txt_i = torch.cat(+ [byt5_txt_i[byt5_text_mask_i], txt_i[text_mask_i], pad_byt5, pad_text], dim=0+ )+ else:+ reorder_txt_i = torch.cat(+ [byt5_txt_i[byt5_text_mask_i], txt_i[text_mask_i], byt5_txt_i[~byt5_text_mask_i], txt_i[~text_mask_i]], dim=0+ )+ reorder_mask_i = torch.cat(+ [byt5_text_mask_i[byt5_text_mask_i], text_mask_i[text_mask_i], byt5_text_mask_i[~byt5_text_mask_i], text_mask_i[~text_mask_i]], dim=0+ )++ reorder_txt.append(reorder_txt_i)+ reorder_mask.append(reorder_mask_i)++ reorder_txt = torch.stack(reorder_txt)+ reorder_mask = torch.stack(reorder_mask).to(dtype=torch.int64)+ else:+ reorder_txt = torch.concat([byt5_txt, txt], dim=1)+ reorder_mask = torch.concat([byt5_text_mask, text_mask], dim=1).to(dtype=torch.int64)++ return reorder_txt, reorder_mask+def forward(self,x: torch.Tensor,@@ -870,6 +973,10 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):face_mask = None,audio_strength = None,bg_latents = None,+ vision_states: torch.Tensor = None,+ byt5_text_states=None,+ byt5_text_mask=None,+ timesteps_r=None,) -> Union[torch.Tensor, Dict[str, torch.Tensor]]:img = x@@ -907,12 +1014,13 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):# token_replace_mask_txt = None# text modulation- vec_2 = self.vector_in(text_states_2)- del text_states_2- vec += vec_2- if self.i2v_condition_type == "token_replace":- token_replace_vec += vec_2- del vec_2+ if text_states_2 is not None:+ vec_2 = self.vector_in(text_states_2)+ del text_states_2+ vec += vec_2+ if self.i2v_condition_type == "token_replace":+ token_replace_vec += vec_2+ del vec_2# guidance modulationif self.guidance_embed:@@ -924,6 +1032,9 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):# our timestep_embedding is merged into guidance_in(TimestepEmbedder)vec += self.guidance_in(guidance)+ if timesteps_r is not None:+ vec += self.time_r_in(timesteps_r)+# Embed image and text.img, shape_mask = self.img_in(img)if self.avatar:@@ -946,6 +1057,27 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):f"Unsupported text_projection: {self.text_projection}")+ if self.cond_type_embedding is not None:+ cond_emb = self.cond_type_embedding( torch.zeros_like(txt[:, :, 0], device=text_mask.device, dtype=torch.long) )+ txt += cond_emb++ if self.glyph_byT5_v2:+ byt5_txt = self.byt5_in(byt5_text_states)+ if self.cond_type_embedding is not None:+ cond_emb = self.cond_type_embedding( torch.ones_like(byt5_txt[:, :, 0], device=byt5_txt.device, dtype=torch.long) )+ byt5_txt += cond_emb+ txt, text_mask = self.reorder_txt_token( byt5_txt, txt, byt5_text_mask, text_mask, zero_feat=True )++ if self.vision_in is not None and vision_states is not None:+ extra_encoder_hidden_states = self.vision_in(vision_states).repeat(bsz,1, 1)+ extra_attention_mask = torch.ones( (bsz, extra_encoder_hidden_states.shape[1]), dtype=text_mask.dtype, device=text_mask.device, )+ if self.cond_type_embedding is not None:+ cond_emb = self.cond_type_embedding(+ 2 * torch.ones_like( extra_encoder_hidden_states[:, :, 0], dtype=torch.long, device=extra_encoder_hidden_states.device, )+ )+ extra_encoder_hidden_states += cond_emb+ txt, text_mask = self.reorder_txt_token( extra_encoder_hidden_states, txt, extra_attention_mask, text_mask )+if self.avatar:img += self.before_proj(ref_latents)ref_length = ref_latents_first.shape[-2] # [b s c]@@ -1218,5 +1350,42 @@ HUNYUAN_VIDEO_CONFIG = {'avatar': True,'audio_condition' : True,},-+ 'HYVideo-1_5': {+ "mm_double_blocks_depth": 54,+ "mm_single_blocks_depth": 0,+ "heads_num": 16,+ "hidden_size": 2048,+ 'rope_dim_list': [16, 56, 56],+ 'mlp_width_ratio': 4,+ "text_pool_type": None,+ "text_states_dim": 3584,+ "text_states_dim_2": None,+ "use_attention_mask": True,+ "use_cond_type_embedding": True,+ "use_meanflow": False,+ "vision_projection": "linear",+ "vision_states_dim": 1152,+ "glyph_byT5_v2": True,+ "patch_size": [1, 1, 1],+ "pre_split_qkv": True,+ },+ 'HYVideo-1_5-upsampler': {+ "mm_double_blocks_depth": 54,+ "mm_single_blocks_depth": 0,+ "heads_num": 16,+ "hidden_size": 2048,+ 'rope_dim_list': [16, 56, 56],+ 'mlp_width_ratio': 4,+ "text_pool_type": None,+ "text_states_dim": 3584,+ "text_states_dim_2": None,+ "use_attention_mask": True,+ "use_cond_type_embedding": True,+ "use_meanflow": True,+ "vision_projection": "linear",+ "vision_states_dim": 1152,+ "glyph_byT5_v2": True,+ "patch_size": [1, 1, 1],+ "pre_split_qkv": True,+ },}\ No newline at end of filediff --git a/models/hyvideo/modules/upsample.py b/models/hyvideo/modules/upsample.pynew file mode 100644index 0000000..ee9eab3--- /dev/null+++ b/models/hyvideo/modules/upsample.py@@ -0,0 +1,169 @@+# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE+#+# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any+# output and results therefrom are provided "AS IS" without any express or implied warranties of+# any kind including any warranties of title, merchantability, noninfringement, course of dealing,+# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the+# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of+# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a+# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise+# of rights and permissions under this agreement.+# See the License for the specific language governing permissions and limitations under the License.++from collections.abc import Sequence+from dataclasses import dataclass+from enum import Enum++import torch+import torch.nn as nn+import torch.nn.functional as F+from einops import rearrange+from torch import Tensor+from diffusers.models import ModelMixin+from diffusers.configuration_utils import ConfigMixin, register_to_config++from ..vae.hunyuanvideo_15_vae import (+ CausalConv3d,+ ResnetBlock,+ RMS_norm,+ forward_with_checkpointing,+ swish,+)+++class UpsamplerType(Enum):+ LEARNED = "learned"+ FIXED = "fixed"+ NONE = "none"+ LEARNED_FIXED = "learned_fixed"+++@dataclass+class UpsamplerConfig:+ load_from: str+ enable: bool = False+ hidden_channels: int = 128+ num_blocks: int = 16+ model_type: UpsamplerType = UpsamplerType.NONE+ version: str = "720p"+++class SRResidualCausalBlock3D(nn.Module):+ def __init__(self, channels: int):+ super().__init__()+ self.block = nn.Sequential(+ CausalConv3d(channels, channels, kernel_size=3),+ nn.SiLU(inplace=True),+ CausalConv3d(channels, channels, kernel_size=3),+ nn.SiLU(inplace=True),+ CausalConv3d(channels, channels, kernel_size=3),+ )++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ return x + self.block(x)+++class SRTo720pUpsampler(ModelMixin, ConfigMixin):++ @register_to_config+ def __init__(+ self,+ in_channels: int,+ out_channels: int,+ hidden_channels: int | None = None,+ num_blocks: int = 6,+ global_residual: bool = False,+ ):+ super().__init__()+ if hidden_channels is None:+ hidden_channels = 64+ self.in_conv = CausalConv3d(in_channels, hidden_channels, kernel_size=3)+ self.blocks = nn.ModuleList([SRResidualCausalBlock3D(hidden_channels) for _ in range(num_blocks)])+ self.out_conv = CausalConv3d(hidden_channels, out_channels, kernel_size=3)+ self.global_residual = bool(global_residual)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ residual = x+ y = self.in_conv(x)+ for blk in self.blocks:+ y = blk(y)+ y = self.out_conv(y)+ if self.global_residual and (y.shape == residual.shape):+ y += residual+ return y+++class SRTo1080pUpsampler(ModelMixin, ConfigMixin):++ @register_to_config+ def __init__(+ self,+ z_channels: int,+ out_channels: int,+ block_out_channels: tuple[int, ...],+ num_res_blocks: int = 2,+ is_residual: bool = False,+ ):+ super().__init__()+ self.num_res_blocks = num_res_blocks+ self.block_out_channels = block_out_channels+ self.z_channels = z_channels++ block_in = block_out_channels[0]+ self.conv_in = CausalConv3d(z_channels, block_in, kernel_size=3)++ self.up = nn.ModuleList()+ for i_level, ch in enumerate(block_out_channels):+ block = nn.ModuleList()+ block_out = ch+ for _ in range(self.num_res_blocks + 1):+ block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))+ block_in = block_out+ up = nn.Module()+ up.block = block++ self.up.append(up)++ self.norm_out = RMS_norm(block_in, images=False)+ self.conv_out = CausalConv3d(block_in, out_channels, kernel_size=3)++ self.gradient_checkpointing = False+ self.is_residual = is_residual++ def forward(self, z: Tensor, target_shape: Sequence[int] = None) -> Tensor:+ """+ Args:+ z: (B, C, T, H, W)+ target_shape: (H, W)+ """+ use_checkpointing = bool(self.training and self.gradient_checkpointing)+ if target_shape is not None and z.shape[-2:] != target_shape:+ bsz = z.shape[0]+ z = rearrange(z, "b c f h w -> (b f) c h w")+ z = F.interpolate(z, size=target_shape, mode="bilinear", align_corners=False)+ z = rearrange(z, "(b f) c h w -> b c f h w", b=bsz)++ # z to block_in+ repeats = self.block_out_channels[0] // (self.z_channels)+ h = self.conv_in(z) + z.repeat_interleave(repeats=repeats, dim=1)++ # upsampling+ for i_level in range(len(self.block_out_channels)):+ for i_block in range(self.num_res_blocks + 1):+ x_list= [h]+ del h+ h = self.up[i_level].block[i_block](x_list)+ if hasattr(self.up[i_level], "upsample"):+ x_list= [h]+ del h+ h = self.up[i_level].upsample(x_list)++ # end+ h = self.norm_out(h).to(z.dtype)+ h = swish(h)+ h = self.conv_out(h)+ return hdiff --git a/models/hyvideo/text_encoder/byT5/__init__.py b/models/hyvideo/text_encoder/byT5/__init__.pynew file mode 100644index 0000000..f85b825--- /dev/null+++ b/models/hyvideo/text_encoder/byT5/__init__.py@@ -0,0 +1,248 @@+# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE+#+# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any+# output and results therefrom are provided "AS IS" without any express or implied warranties of+# any kind including any warranties of title, merchantability, noninfringement, course of dealing,+# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the+# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of+# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a+# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise+# of rights and permissions under this agreement.+# See the License for the specific language governing permissions and limitations under the License.++import json+import torch+import torch.nn as nn+from transformers import AutoTokenizer, T5ForConditionalGeneration, T5EncoderModel+import os++def load_glyph_byT5_v2(args, device):+ """+ Loads ByT5 tokenizer and encoder model for glyph encoding.++ Args:+ args (dict): Configuration dictionary containing paths and settings.+ device (str or torch.device): Device to load the model onto.++ Returns:+ dict: Dictionary with keys 'byt5_tokenizer', 'byt5_model', 'byt5_max_length'.+ """+ byt5_tokenizer, byt5_model, byt5_max_length = create_byt5(args, device)+ byt5_model = byt5_model.to(device=device)+ return byt5_tokenizer, byt5_model++ return {+ "byt5_tokenizer": byt5_tokenizer,+ "byt5_model": byt5_model,+ "byt5_max_length": byt5_max_length,+ }+++def create_byt5(args, device):+ """+ Create ByT5 tokenizer and encoder, load weights if provided.++ Args:+ args (dict): Configuration dictionary.+ device (str or torch.device): Device to load the model onto.++ Returns:+ tuple: (byt5_tokenizer, byt5_model, byt5_max_length)+ """+ byt5_max_length = args['byt5_max_length']+ byt5_config = dict(+ byt5_name=args['byT5_google_path'],+ special_token=True,+ color_special_token=True,+ font_special_token=True,+ color_ann_path=args['multilingual_prompt_format_color_path'],+ font_ann_path=args['multilingual_prompt_format_font_path'],+ multilingual=True,+ )+ huggingface_cache_dir = None+ byt5_model, byt5_tokenizer = load_byt5_and_byt5_tokenizer(+ **byt5_config,+ huggingface_cache_dir= os.path.dirname(args["byT5_ckpt_path"]) ,+ device=device,+ )+ from mmgp import safetensors2++ # xsd = torch.load( os.path.dirname( args["byT5_ckpt_path"]) + "/pytorch_model.bin")+ # Load custom checkpoint if provided+ from mmgp import offload+ offload.load_model_data(byt5_model, args['byT5_ckpt_path'], writable_tensors= False)++ if args['byT5_ckpt_path'] is not None and False:+ if "cuda" not in str(device):+ byt5_state_dict = torch.load(args['byT5_ckpt_path'], map_location=device)+ else:+ byt5_state_dict = torch.load(args['byT5_ckpt_path'], map_location=device)+ if 'state_dict' in byt5_state_dict:+ sd = byt5_state_dict["state_dict"]+ newsd = {}+ for k, v in sd.items():+ if k.startswith('module.text_tower.encoder.'):+ newsd[k[len('module.text_tower.encoder.'):]] = v+ byt5_state_dict = newsd+ newsd = {}+ for k, v in byt5_state_dict.items():+ newsd['encoder.'+k] =v+ xsd.update(newsd)+ safetensors2.torch_write_file(xsd ,"combined.safetensors")++ byt5_model.load_state_dict(byt5_state_dict)+ byt5_model.requires_grad_(False)+ return byt5_tokenizer, byt5_model, byt5_max_length+++def add_special_token(+ tokenizer,+ text_encoder,+ add_color,+ add_font,+ color_ann_path,+ font_ann_path,+ multilingual=False,+):+ """+ Add special tokens for color and font to tokenizer and text encoder.++ Args:+ tokenizer: Huggingface tokenizer.+ text_encoder: Huggingface T5 encoder.+ add_color (bool): Whether to add color tokens.+ add_font (bool): Whether to add font tokens.+ color_ann_path (str): Path to color annotation JSON.+ font_ann_path (str): Path to font annotation JSON.+ multilingual (bool): Whether to use multilingual font tokens.+ """+ with open(font_ann_path, 'r') as f:+ idx_font_dict = json.load(f)+ with open(color_ann_path, 'r') as f:+ idx_color_dict = json.load(f)++ if multilingual:+ font_token = [f'<{font_code[:2]}-font-{idx_font_dict[font_code]}>' for font_code in idx_font_dict]+ else:+ font_token = [f'<font-{i}>' for i in range(len(idx_font_dict))]+ color_token = [f'<color-{i}>' for i in range(len(idx_color_dict))]+ additional_special_tokens = []+ if add_color:+ additional_special_tokens += color_token+ if add_font:+ additional_special_tokens += font_token++ tokenizer.add_tokens(additional_special_tokens, special_tokens=True)+ # Set mean_resizing=False to avoid PyTorch LAPACK dependency+ text_encoder.resize_token_embeddings(len(tokenizer), mean_resizing=False)+++def load_byt5_and_byt5_tokenizer(+ byt5_name='google/byt5-small',+ special_token=False,+ color_special_token=False,+ font_special_token=False,+ color_ann_path='assets/color_idx.json',+ font_ann_path='assets/font_idx_512.json',+ huggingface_cache_dir=None,+ multilingual=False,+ device=None,+):+ """+ Load ByT5 encoder and tokenizer from Huggingface, and add special tokens if needed.++ Args:+ byt5_name (str): Model name or path.+ special_token (bool): Whether to add special tokens.+ color_special_token (bool): Whether to add color tokens.+ font_special_token (bool): Whether to add font tokens.+ color_ann_path (str): Path to color annotation JSON.+ font_ann_path (str): Path to font annotation JSON.+ huggingface_cache_dir (str): Huggingface cache directory.+ multilingual (bool): Whether to use multilingual font tokens.+ device (str or torch.device): Device to load the model onto.++ Returns:+ tuple: (byt5_text_encoder, byt5_tokenizer)+ """+ byt5_tokenizer = AutoTokenizer.from_pretrained(+ byt5_name,+ cache_dir=huggingface_cache_dir,+ )+ byt5_text_encoder = T5ForConditionalGeneration.from_pretrained(+ byt5_name,+ cache_dir=huggingface_cache_dir,+ ).get_encoder()++ # byt5_text_encoder = T5EncoderModel.from_pretrained(+ # byt5_name,+ # cache_dir=huggingface_cache_dir,+ # )+++ if "cuda" not in str(device):+ device = torch.device(device)+ else:+ device = torch.device(device)+ byt5_text_encoder = byt5_text_encoder.to(device)++ if special_token:+ add_special_token(+ byt5_tokenizer,+ byt5_text_encoder,+ add_color=color_special_token,+ add_font=font_special_token,+ color_ann_path=color_ann_path,+ font_ann_path=font_ann_path,+ multilingual=multilingual,+ )+ return byt5_text_encoder, byt5_tokenizer+++class ByT5Mapper(nn.Module):+ """+ ByT5Mapper: Maps ByT5 encoder outputs to a new space, with optional residual connection.++ Args:+ in_dim (int): Input dimension (must equal out_dim if use_residual).+ out_dim (int): Output dimension after second linear layer.+ hidden_dim (int): Hidden dimension for intermediate layer.+ out_dim1 (int): Final output dimension.+ use_residual (bool): Whether to use residual connection (default: True).+ """++ def __init__(self, in_dim, out_dim, hidden_dim, out_dim1, use_residual=True):+ super().__init__()+ if use_residual:+ assert in_dim == out_dim+ self.layernorm = nn.LayerNorm(in_dim)+ self.fc1 = nn.Linear(in_dim, hidden_dim)+ self.fc2 = nn.Linear(hidden_dim, out_dim)+ self.fc3 = nn.Linear(out_dim, out_dim1)+ self.use_residual = use_residual+ self.act_fn = nn.GELU()++ def forward(self, x):+ """+ Forward pass for ByT5Mapper.++ Args:+ x (Tensor): Input tensor of shape (..., in_dim).++ Returns:+ Tensor: Output tensor of shape (..., out_dim1).+ """+ residual = x+ x = self.layernorm(x)+ x = self.fc1(x)+ x = self.act_fn(x)+ x = self.fc2(x)+ x2 = self.act_fn(x)+ x2 = self.fc3(x2)+ if self.use_residual:+ x2 = x2 + residual+ return x2diff --git a/models/hyvideo/text_encoder/byT5/format_prompt.py b/models/hyvideo/text_encoder/byT5/format_prompt.pynew file mode 100644index 0000000..ff7b9c5--- /dev/null+++ b/models/hyvideo/text_encoder/byT5/format_prompt.py@@ -0,0 +1,86 @@+# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE+#+# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any+# output and results therefrom are provided "AS IS" without any express or implied warranties of+# any kind including any warranties of title, merchantability, noninfringement, course of dealing,+# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the+# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of+# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a+# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise+# of rights and permissions under this agreement.+# See the License for the specific language governing permissions and limitations under the License.++import json+++def closest_color(requested_color):+ import webcolors++ min_colors = {}+ for key, name in webcolors.CSS3_HEX_TO_NAMES.items():++ r_c, g_c, b_c = webcolors.hex_to_rgb(key)+ rd = (r_c - requested_color[0]) ** 2+ gd = (g_c - requested_color[1]) ** 2+ bd = (b_c - requested_color[2]) ** 2+ min_colors[(rd + gd + bd)] = name+ return min_colors[min(min_colors.keys())]+++def convert_rgb_to_names(rgb_tuple):+ try:+ import webcolors++ color_name = webcolors.rgb_to_name(rgb_tuple)+ except ValueError:+ color_name = closest_color(rgb_tuple)+ return color_name+++class MultilingualPromptFormat:++ def __init__(+ self,+ font_path: str = 'assets/glyph_sdxl_assets/multilingual_10-lang_idx.json',+ color_path: str = 'assets/glyph_sdxl_assets/color_idx.json',+ ):+ with open(font_path, 'r') as f:+ self.font_dict = json.load(f)+ with open(color_path, 'r') as f:+ self.color_dict = json.load(f)++ def format_prompt(self, texts, styles):+ '''+ Text "{text}" in {color}, {type}.+ '''++ prompt = ""+ for text, style in zip(texts, styles):+ text_prompt = f'Text "{text}"'++ attr_list = []++ # format color+ if style["color"] is not None:+ import webcolors++ hex_color = style["color"]+ rgb_color = webcolors.hex_to_rgb(hex_color)+ color_name = convert_rgb_to_names(rgb_color)+ attr_list.append(f"<color-{self.color_dict[color_name]}>")++ # format font+ if style["font-family"] is not None:+ attr_list.append(f"<{style['font-family'][:2]}-font-{self.font_dict[style['font-family']]}>")+ attr_suffix = ", ".join(attr_list)+ text_prompt += " in " + attr_suffix+ text_prompt += ". "+ else:+ text_prompt += ". "++ prompt = prompt + text_prompt+ return promptdiff --git a/models/hyvideo/text_encoder/text_encoder_1_5.py b/models/hyvideo/text_encoder/text_encoder_1_5.pynew file mode 100644index 0000000..a680c8b--- /dev/null+++ b/models/hyvideo/text_encoder/text_encoder_1_5.py@@ -0,0 +1,518 @@+# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE+#+# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any+# output and results therefrom are provided "AS IS" without any express or implied warranties of+# any kind including any warranties of title, merchantability, noninfringement, course of dealing,+# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the+# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of+# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a+# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise+# of rights and permissions under this agreement.+# See the License for the specific language governing permissions and limitations under the License.++import os+from copy import deepcopy+from dataclasses import dataclass+from typing import Optional, Tuple+from shared.utils import files_locator as fl+import torch+import torch.nn as nn+from transformers import Qwen2_5_VLForConditionalGeneration++from transformers import (+ AutoTokenizer,+ AutoModel,+)+from transformers.utils import ModelOutput+++def use_default(value, default):+ """Utility: return value if not None, else default."""+ return value if value is not None else default++# Prompt templates for different models and tasks+++__all__ = [+ "C_SCALE", "PROMPT_TEMPLATE",+ "MODEL_BASE",+]++# =================== Constant Values =====================+# Computation scale factor, 1P = 1_000_000_000_000_000. Tensorboard will display the value in PetaFLOPS to avoid+# overflow error when tensorboard logging values.+C_SCALE = 1_000_000_000_000_000++PROMPT_TEMPLATE_ENCODE_IMAGE_JSON = [+ {"role": "system", "content": "You are a helpful assistant. Describe the image by detailing the following aspects: \+ 1. The main content and theme of the image. \+ 2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects. \+ 3. The background environment, light, style and atmosphere."},+ {"role": "user", "content": "{}"}+]++PROMPT_TEMPLATE_ENCODE_VIDEO_JSON = [+ {"role": "system", "content": "You are a helpful assistant. Describe the video by detailing the following aspects: \+ 1. The main content and theme of the video. \+ 2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects. \+ 3. Actions, events, behaviors temporal relationships, physical movement changes of the objects. \+ 4. background environment, light, style and atmosphere. \+ 5. camera angles, movements, and transitions used in the video."},+ {"role": "user", "content": "{}"}+]++PROMPT_TEMPLATE = {+ "li-dit-encode-image-json": {"template": PROMPT_TEMPLATE_ENCODE_IMAGE_JSON, "crop_start": -1}, # auto-calculate crop_start+ "li-dit-encode-video-json": {"template": PROMPT_TEMPLATE_ENCODE_VIDEO_JSON, "crop_start": -1}, # auto-calculate crop_start+}+++MODEL_BASE = os.getenv("MODEL_BASE", "")+TEXT_ENCODER_PATH = {}+TOKENIZER_PATH = {}++PRECISION_TO_TYPE = {+ 'fp32': torch.float32,+ 'fp16': torch.float16,+ 'bf16': torch.bfloat16,+}+++def load_text_encoder(+ text_encoder_type,+ text_encoder_precision=None,+ text_encoder_path=None,+ logger=None,+ device=None,+):+ if text_encoder_path is None:+ if text_encoder_type not in TEXT_ENCODER_PATH:+ raise ValueError(f"Unsupported text encoder type: {text_encoder_type}")+ text_encoder_path = TEXT_ENCODER_PATH[text_encoder_type]++ from mmgp import offload+ # text_encoder = offload.fast_load_transformers_model(text_encoder_path, forcedConfigPath= os.path.join(os.path.dirname(text_encoder_path), "config.json"))+ text_encoder = offload.fast_load_transformers_model(text_encoder_path, writable_tensors= True , modelClass=Qwen2_5_VLForConditionalGeneration, defaultConfigPath= fl.locate_file(os.path.join("Qwen2.5-VL-7B-Instruct", "config.json")) )++ # text_encoder = AutoModel.from_pretrained(text_encoder_path, low_cpu_mem_usage=True)++ if hasattr(text_encoder, 'language_model'):+ text_encoder = text_encoder.language_model+ text_encoder.final_layer_norm = text_encoder.norm++ # from_pretrained will ensure that the model is in eval mode.+ # if text_encoder_precision is not None:+ # text_encoder = text_encoder.to(dtype=PRECISION_TO_TYPE[text_encoder_precision])++ text_encoder.requires_grad_(False)++ if device is not None:+ text_encoder = text_encoder.to(device)++ return text_encoder, text_encoder_path+++def load_tokenizer(+ tokenizer_type, tokenizer_path=None, padding_side="right", logger=None+):+ processor = None+ if tokenizer_path is None:+ if tokenizer_type not in TOKENIZER_PATH:+ raise ValueError(f"Unsupported tokenizer type: {tokenizer_type}")+ tokenizer_path = TOKENIZER_PATH[tokenizer_type]++ tokenizer = AutoTokenizer.from_pretrained(+ os.path.dirname(tokenizer_path), padding_side=padding_side+ )++ # If the checkpoint ships a chat template separately, load it when missing.+ if getattr(tokenizer, "chat_template", None) in (None, ""):+ jinja_path = os.path.join(os.path.dirname(tokenizer_path), "chat_template.json")+ if os.path.exists(jinja_path):+ with open(jinja_path, "r", encoding="utf-8") as f:+ tokenizer.chat_template = f.read()++ return tokenizer, tokenizer_path, processor+++@dataclass+class TextEncoderModelOutput(ModelOutput):+ """+ Base class for model's outputs that also contains a pooling of the last hidden states.++ Args:+ hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):+ Sequence of hidden-states at the output of the last layer of the model.+ attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):+ Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``:+ hidden_states_list (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed):+ Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, ++ one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.+ Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.+ text_outputs (`list`, *optional*, returned when `return_texts=True` is passed):+ List of decoded texts.+ """++ hidden_state: torch.FloatTensor = None+ attention_mask: Optional[torch.LongTensor] = None+ hidden_states_list: Optional[Tuple[torch.FloatTensor, ...]] = None+ text_outputs: Optional[list] = None+ image_features: Optional[list] = None++class TextEncoder(nn.Module):+ def __init__(+ self,+ text_encoder_type: str,+ max_length: int,+ text_encoder_precision: Optional[str] = None,+ text_encoder_path: Optional[str] = None,+ tokenizer_type: Optional[str] = None,+ tokenizer_path: Optional[str] = None,+ output_key: Optional[str] = None,+ use_attention_mask: bool = True,+ prompt_template: Optional[dict] = None,+ prompt_template_video: Optional[dict] = None,+ hidden_state_skip_layer: Optional[int] = None,+ apply_final_norm: bool = False,+ reproduce: bool = False,+ logger=None,+ i2v_mode = None,+ image_embed_interleave = None,+ device=None,+ ):+ super().__init__()+ self.text_encoder_type = text_encoder_type+ self.max_length = max_length+ self.precision = text_encoder_precision+ self.model_path = text_encoder_path+ self.tokenizer_type = (+ tokenizer_type if tokenizer_type is not None else text_encoder_type+ )+ self.tokenizer_path = (+ tokenizer_path if tokenizer_path is not None else text_encoder_path+ )+ self.use_attention_mask = use_attention_mask+ if prompt_template_video is not None:+ assert (+ use_attention_mask is True+ ), "Attention mask is True required when training videos."+ self.prompt_template = prompt_template+ self.prompt_template_video = prompt_template_video+ self.hidden_state_skip_layer = hidden_state_skip_layer+ self.apply_final_norm = apply_final_norm+ self.reproduce = reproduce+ self.logger = logger++ self.use_template = self.prompt_template is not None+ if self.use_template:+ assert (+ isinstance(self.prompt_template, dict)+ and "template" in self.prompt_template+ ), f"`prompt_template` must be a dictionary with a key 'template', got {self.prompt_template}"+ assert "{}" in str(self.prompt_template["template"]), (+ "`prompt_template['template']` must contain a placeholder `{}` for the input text, "+ f"got {self.prompt_template['template']}"+ )++ self.use_video_template = self.prompt_template_video is not None+ if self.use_video_template:+ if self.prompt_template_video is not None:+ assert (+ isinstance(self.prompt_template_video, dict)+ and "template" in self.prompt_template_video+ ), f"`prompt_template_video` must be a dictionary with a key 'template', got {self.prompt_template_video}"+ assert "{}" in str(self.prompt_template_video["template"]), (+ "`prompt_template_video['template']` must contain a placeholder `{}` for the input text, "+ f"got {self.prompt_template_video['template']}"+ )++ if text_encoder_type != "llm":+ raise ValueError(f"Unsupported text encoder type: {text_encoder_type}")+ self.output_key = output_key or "last_hidden_state"++ self.model, self.model_path = load_text_encoder(+ text_encoder_type=self.text_encoder_type,+ text_encoder_precision=self.precision,+ text_encoder_path=self.model_path,+ logger=self.logger,+ device=device,+ )++ self.tokenizer, self.tokenizer_path, self.processor = load_tokenizer(+ tokenizer_type=self.tokenizer_type,+ tokenizer_path=self.tokenizer_path,+ padding_side="right",+ logger=self.logger,+ )++ # pre-calculate crop_start for image and video+ if self.use_template and self.prompt_template is not None:+ self.text2tokens("a photo of a cat", data_type="image")+ if self.use_video_template and self.prompt_template_video is not None:+ self.text2tokens("a photo of a cat", data_type="video")++ @property+ def dtype(self):+ return self.model.dtype++ @property+ def device(self):+ return self.model.device++ def __repr__(self):+ return f"{self.text_encoder_type} ({self.precision} - {self.model_path})"++ @staticmethod+ def apply_text_to_template(text, template, prevent_empty_text=True):+ """+ Apply text to template.++ Args:+ text (str): Input text.+ template (str or list): Template string or list of chat conversation.+ prevent_empty_text (bool): If Ture, we will prevent the user text from being empty+ by adding a space. Defaults to True.+ """+ if isinstance(template, str):+ # Will send string to tokenizer. Used for llm+ return template.format(text)+ elif isinstance(template, list):+ # For JSON list template format (chat conversation)+ # Create a deep copy to avoid modifying the original template+ template_copy = deepcopy(template)+ for item in template_copy:+ if isinstance(item, dict) and "content" in item:+ # Replace placeholder with text in the content field+ item["content"] = item["content"].format(text if text else (" " if prevent_empty_text else ""))+ return template_copy+ else:+ raise TypeError(f"Unsupported template type: {type(template)}")++ def calculate_crop_start(self, tokenized_input):+ """+ Automatically calculate the crop_start position based on identifying user tokens.++ Args:+ tokenized_input: The output from the tokenizer containing input_ids++ Returns:+ int: The position where the actual prompt content begins (after user markers)+ """+ input_ids = tokenized_input["input_ids"][0].tolist() # Get the first example's tokens++ marker = "<|im_start|>user\n"++ # Tokenize just the marker to get its token IDs+ marker_tokens = self.tokenizer(marker, add_special_tokens=False)["input_ids"]++ # Find the end position of the marker in the input sequence+ for i in range(len(input_ids) - len(marker_tokens) + 1):+ if input_ids[i:i+len(marker_tokens)] == marker_tokens:+ # Return the position after the marker+ return i + len(marker_tokens)++ # If marker not found, try to find based on special tokens+ if hasattr(self.tokenizer, 'special_tokens_map'):+ # Check for user token or any other special token that might indicate user input start+ for token_name, token_value in self.tokenizer.special_tokens_map.items():+ if 'user' in token_name.lower():+ user_token_id = self.tokenizer.convert_tokens_to_ids(token_value)+ if user_token_id in input_ids:+ return input_ids.index(user_token_id) + 1++ # Default fallback: return 0 (no cropping)+ return 0++ def text2tokens(self, text, data_type="image", max_length=300):+ """+ Tokenize the input text.++ Args:+ text (str or list): Input text.+ """+ tokenize_input_type = "str"+ if self.use_template or self.use_video_template:+ if data_type == "image":+ prompt_template = self.prompt_template["template"]+ crop_start = self.prompt_template.get("crop_start", -1)+ elif data_type == "video":+ prompt_template = self.prompt_template_video["template"]+ crop_start = self.prompt_template_video.get("crop_start", -1)+ else:+ raise ValueError(f"Unsupported data type: {data_type}")+ if isinstance(text, (list, tuple)):+ text = [+ self.apply_text_to_template(one_text, prompt_template)+ for one_text in text+ ]+ if isinstance(text[0], list):+ tokenize_input_type = "list"+ elif isinstance(text, str):+ text = self.apply_text_to_template(text, prompt_template)+ if isinstance(text, list):+ tokenize_input_type = "list"+ else:+ raise TypeError(f"Unsupported text type: {type(text)}")++ # First pass: tokenize with arbitrary max_length to find crop_start+ if crop_start == -1:+ # Use temporary max_length for the first pass (large enough)+ temp_kwargs = dict(+ truncation=True,+ max_length=256, # Temporary large value+ padding="max_length",+ return_tensors="pt",+ )++ # First tokenization pass to calculate crop_start+ if tokenize_input_type == "str":+ temp_tokenized = self.tokenizer(+ text,+ return_length=False,+ return_overflowing_tokens=False,+ return_attention_mask=True,+ **temp_kwargs,+ )+ elif tokenize_input_type == "list":+ temp_tokenized = self.tokenizer.apply_chat_template(+ text,+ add_generation_prompt=True,+ tokenize=True,+ return_dict=True,+ **temp_kwargs,+ )++ # Calculate the crop_start from this first pass+ crop_start = self.calculate_crop_start(temp_tokenized)++ # Store the calculated crop_start for future use+ if data_type == "image":+ self.prompt_template["crop_start"] = crop_start+ else:+ self.prompt_template_video["crop_start"] = crop_start+ else:+ crop_start = 0++ # Second pass: tokenize with the proper max_length using the found crop_start+ kwargs = dict(+ truncation=True,+ max_length=max_length + (crop_start if crop_start > 0 else 0),+ padding="max_length",+ return_tensors="pt",+ )++ if tokenize_input_type == "str":+ tokenized_output = self.tokenizer(+ text,+ return_length=False,+ return_overflowing_tokens=False,+ return_attention_mask=True,+ **kwargs,+ )+ elif tokenize_input_type == "list":+ tokenized_output = self.tokenizer.apply_chat_template(+ text,+ add_generation_prompt=True,+ tokenize=True,+ return_dict=True,+ **kwargs,+ )+ else:+ raise ValueError(f"Unsupported tokenize_input_type: {tokenize_input_type}")++ return tokenized_output++ def encode(+ self,+ batch_encoding,+ use_attention_mask=None,+ output_hidden_states=False,+ do_sample=None,+ hidden_state_skip_layer=None,+ return_texts=False,+ data_type="image",+ device=None,+ is_uncond=False+ ):+ """+ Args:+ batch_encoding (dict): Batch encoding from tokenizer.+ use_attention_mask (bool): Whether to use attention mask. If None, use self.use_attention_mask.+ Defaults to None.+ output_hidden_states (bool): Whether to output hidden states. If False, return the value of+ self.output_key. If True, return the entire output. If set self.hidden_state_skip_layer,+ output_hidden_states will be set True. Defaults to False.+ do_sample (bool): Whether to sample from the model. Used for Decoder-Only LLMs. Defaults to None.+ When self.produce is False, do_sample is set to True by default.+ hidden_state_skip_layer (int): Number of hidden states to hidden_state_skip_layer. 0 means the last layer.+ If None, self.output_key will be used. Defaults to None.+ return_texts (bool): Whether to return the decoded texts. Defaults to False.+ """+ device = self.model.device if device is None else device+ use_attention_mask = use_default(use_attention_mask, self.use_attention_mask)+ hidden_state_skip_layer = use_default(+ hidden_state_skip_layer, self.hidden_state_skip_layer+ )+ do_sample = use_default(do_sample, not self.reproduce)++ attention_mask = (+ batch_encoding["attention_mask"].to(device) if use_attention_mask else None+ )+ outputs = self.model(+ input_ids=batch_encoding["input_ids"].to(device),+ attention_mask=attention_mask,+ output_hidden_states=output_hidden_states+ or hidden_state_skip_layer is not None,+ )+ if hidden_state_skip_layer is not None:+ last_hidden_state = outputs.hidden_states[-(hidden_state_skip_layer + 1)]+ # Real last hidden state already has layer norm applied. So here we only apply it+ # for intermediate layers.+ if hidden_state_skip_layer > 0 and self.apply_final_norm:+ last_hidden_state = self.model.final_layer_norm(last_hidden_state)+ else:+ last_hidden_state = outputs[self.output_key]++ # Remove hidden states of instruction tokens, only keep prompt tokens.+ if self.use_template:+ if data_type == "image":+ crop_start = self.prompt_template.get("crop_start", 0)+ elif data_type == "video":+ crop_start = self.prompt_template_video.get("crop_start", 0)+ else:+ raise ValueError(f"Unsupported data type: {data_type}")+ if crop_start > 0:+ last_hidden_state = last_hidden_state[:, crop_start:]+ attention_mask = (+ attention_mask[:, crop_start:] if use_attention_mask else None+ )++ if output_hidden_states:+ return TextEncoderModelOutput(+ last_hidden_state, attention_mask, outputs.hidden_states+ )+ return TextEncoderModelOutput(last_hidden_state, attention_mask)+++ def forward(+ self,+ text,+ use_attention_mask=None,+ output_hidden_states=False,+ do_sample=False,+ hidden_state_skip_layer=None,+ return_texts=False,+ ):+ batch_encoding = self.text2tokens(text, max_length=self.max_length)+ return self.encode(+ batch_encoding,+ use_attention_mask=use_attention_mask,+ output_hidden_states=output_hidden_states,+ do_sample=do_sample,+ hidden_state_skip_layer=hidden_state_skip_layer,+ return_texts=return_texts,+ )diff --git a/models/hyvideo/vae/hunyuanvideo_15_vae.py b/models/hyvideo/vae/hunyuanvideo_15_vae.pynew file mode 100644index 0000000..61a937c--- /dev/null+++ b/models/hyvideo/vae/hunyuanvideo_15_vae.py@@ -0,0 +1,949 @@+# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE+#+# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any+# output and results therefrom are provided "AS IS" without any express or implied warranties of+# any kind including any warranties of title, merchantability, noninfringement, course of dealing,+# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the+# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of+# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a+# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise+# of rights and permissions under this agreement.+# See the License for the specific language governing permissions and limitations under the License.++import math+from dataclasses import dataclass+from typing import Optional, Tuple, Union++import loguru+import numpy as np+import torch+import torch.nn.functional as F+from einops import rearrange+from torch import Tensor, nn+from torch.nn import Conv3d+from torch import distributed as dist++from diffusers.configuration_utils import ConfigMixin, register_to_config+from diffusers.models.autoencoders.vae import BaseOutput, DiagonalGaussianDistribution+from diffusers.models.modeling_outputs import AutoencoderKLOutput+from diffusers.models.modeling_utils import ModelMixin++++@dataclass+class DecoderOutput(BaseOutput):+ sample: torch.FloatTensor+ posterior: Optional[DiagonalGaussianDistribution] = None+++def swish(x: Tensor, inplace=False) -> Tensor:+ """Applies the swish activation function (SiLU) with optional inplace support."""+ return F.silu(x, inplace=inplace)+++def forward_with_checkpointing(module, *inputs, use_checkpointing=False):+ """Forward with optional gradient checkpointing."""+ def create_custom_forward(module):+ def custom_forward(*inputs):+ return module(*inputs)+ return custom_forward++ if use_checkpointing:+ return torch.utils.checkpoint.checkpoint(create_custom_forward(module), *inputs, use_reentrant=False)+ else:+ return module(*inputs)+++# Optimized implementation of CogVideoXSafeConv3d+# https://github.com/huggingface/diffusers/blob/c9ff360966327ace3faad3807dc871a4e5447501/src/diffusers/models/autoencoders/autoencoder_kl_cogvideox.py#L38+class PatchCausalConv3d(nn.Conv3d):+ r"""Causal Conv3d with efficient patch processing for large tensors."""+ def find_split_indices(self, seq_len, part_num):+ ideal_interval = seq_len / part_num+ possible_indices = list(range(0, seq_len, self.stride[0]))+ selected_indices = []++ for i in range(1, part_num):+ closest = min(possible_indices, key=lambda x: abs(x - round(i * ideal_interval)))+ if closest not in selected_indices:+ selected_indices.append(closest)++ merged_indices = []+ prev_idx = 0+ for idx in selected_indices:+ if idx - prev_idx >= self.kernel_size[0]:+ merged_indices.append(idx)+ prev_idx = idx++ return merged_indices++ def forward(self, input):+ T = input.shape[2] # input: NCTHW+ memory_count = torch.prod(torch.tensor(input.shape)).item() * 2 / 1024**3+ if T > self.kernel_size[0] and memory_count > 0.6:+ kernel_size = self.kernel_size[0]+ part_num = int(memory_count / 2) + 1+ split_indices = self.find_split_indices(T, part_num)+ input_chunks = torch.tensor_split(input, split_indices, dim=2) if len(split_indices) > 0 else [input]+ if kernel_size > 1:+ input_chunks = [input_chunks[0]] + [+ torch.cat((input_chunks[i - 1][:, :, -kernel_size + 1:], input_chunks[i]), dim=2)+ for i in range(1, len(input_chunks))+ ]+ output_chunks = []+ for input_chunk in input_chunks:+ output_chunks.append(super().forward(input_chunk))+ output = torch.cat(output_chunks, dim=2)+ return output+ else:+ return super().forward(input)+++++class RMS_norm(nn.Module):+ """Root Mean Square Layer Normalization for Channel-First or Last"""++ def __init__(self, dim, channel_first=True, images=True, bias=False):+ super().__init__()+ broadcastable_dims = (1, 1, 1) if not images else (1, 1)+ shape = (dim, *broadcastable_dims) if channel_first else (dim,)++ self.channel_first = channel_first+ self.scale = dim**0.5+ self.gamma = nn.Parameter(torch.ones(shape))+ self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0.++ def forward(self, x):+ return F.normalize(+ x, dim=(1 if self.channel_first else -1)+ ) * self.scale * self.gamma + self.bias+++class CausalConv3d(nn.Module):+ """Causal Conv3d with configurable padding for temporal axis."""++ def __init__(+ self,+ chan_in,+ chan_out,+ kernel_size: Union[int, Tuple[int, int, int]],+ stride: Union[int, Tuple[int, int, int]] = 1,+ dilation: Union[int, Tuple[int, int, int]] = 1,+ pad_mode='replicate',+ disable_causal=False,+ enable_patch_conv=False,+ **kwargs+ ):+ super().__init__()++ self.pad_mode = pad_mode+ if disable_causal:+ padding = (kernel_size // 2, kernel_size // 2, kernel_size // 2, kernel_size // 2, kernel_size // 2, kernel_size // 2)+ else:+ padding = (kernel_size // 2, kernel_size // 2, kernel_size // 2, kernel_size // 2, kernel_size - 1, 0) # W, H, T+ self.time_causal_padding = padding++ if enable_patch_conv:+ self.conv = PatchCausalConv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)+ else:+ self.conv = nn.Conv3d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)++ def forward(self, x):+ x = F.pad(x, self.time_causal_padding, mode=self.pad_mode)+ return self.conv(x)+++def prepare_causal_attention_mask(n_frame: int, n_hw: int, dtype, device, batch_size: int = None):+ """Prepare a causal attention mask for 3D videos.++ Args:+ n_frame (int): Number of frames (temporal length).+ n_hw (int): Product of height and width.+ dtype: Desired mask dtype.+ device: Device for the mask.+ batch_size (int, optional): If set, expands for batch.++ Returns:+ torch.Tensor: Causal attention mask.+ """+ seq_len = n_frame * n_hw+ mask = torch.full((seq_len, seq_len), float("-inf"), dtype=dtype, device=device)+ for i in range(seq_len):+ i_frame = i // n_hw+ mask[i, : (i_frame + 1) * n_hw] = 0+ if batch_size is not None:+ mask = mask.unsqueeze(0).expand(batch_size, -1, -1)+ return mask+++class AttnBlock(nn.Module):+ """Self-attention block for 3D video tensors."""++ def __init__(self, in_channels: int):+ super().__init__()+ self.in_channels = in_channels++ self.norm = RMS_norm(in_channels, images=False)++ self.q = Conv3d(in_channels, in_channels, kernel_size=1)+ self.k = Conv3d(in_channels, in_channels, kernel_size=1)+ self.v = Conv3d(in_channels, in_channels, kernel_size=1)+ self.proj_out = Conv3d(in_channels, in_channels, kernel_size=1)++ def attention(self, h_: Tensor) -> Tensor:+ h_ = self.norm(h_)+ q = self.q(h_)+ k = self.k(h_)+ v = self.v(h_)++ b, c, f, h, w = q.shape+ q = rearrange(q, "b c f h w -> b 1 (f h w) c").contiguous()+ k = rearrange(k, "b c f h w -> b 1 (f h w) c").contiguous()+ v = rearrange(v, "b c f h w -> b 1 (f h w) c").contiguous()+ attention_mask = prepare_causal_attention_mask(f, h * w, h_.dtype, h_.device, batch_size=b)+ h_ = nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=attention_mask.unsqueeze(1))++ return rearrange(h_, "b 1 (f h w) c -> b c f h w", f=f, h=h, w=w, c=c, b=b)++ def forward(self, x: Tensor) -> Tensor:+ return x + self.proj_out(self.attention(x))+++class ResnetBlock(nn.Module):+ """ResNet-style block for 3D video tensors."""++ def __init__(self, in_channels: int, out_channels: int):+ super().__init__()+ self.in_channels = in_channels+ out_channels = in_channels if out_channels is None else out_channels+ self.out_channels = out_channels++ self.norm1 = RMS_norm(in_channels, images=False)+ self.conv1 = CausalConv3d(in_channels, out_channels, kernel_size=3)++ self.norm2 = RMS_norm(out_channels, images=False)+ self.conv2 = CausalConv3d(out_channels, out_channels, kernel_size=3)+ if self.in_channels != self.out_channels:+ self.nin_shortcut = Conv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)++ def forward(self, x_list):+ x = x_list[0]+ x_list.clear()+ dtype = x.dtype+ h = x+ h = self.norm1(h).to(dtype)+ h = swish(h, inplace=True)+ h = self.conv1(h)++ h = self.norm2(h).to(dtype)+ h = swish(h, inplace=True)+ h = self.conv2(h)++ if self.in_channels != self.out_channels:+ x = self.nin_shortcut(x)+ h += x+ return h+++class Downsample(nn.Module):++ def __init__(self, in_channels: int, out_channels: int, add_temporal_downsample: bool = True):+ super().__init__()+ factor = 2 * 2 * 2 if add_temporal_downsample else 1 * 2 * 2+ assert out_channels % factor == 0+ self.conv = CausalConv3d(in_channels, out_channels // factor, kernel_size=3)+ self.add_temporal_downsample = add_temporal_downsample+ self.group_size = factor * in_channels // out_channels++ def forward(self, x_list):+ x = x_list[0]+ x_list.clear()+ r1 = 2 if self.add_temporal_downsample else 1+ h = self.conv(x)+ if self.add_temporal_downsample:+ h_first = h[:, :, :1, :, :]+ h_first = rearrange(h_first, "b c f (h r2) (w r3) -> b (r2 r3 c) f h w", r2=2, r3=2)+ h_first = torch.cat([h_first, h_first], dim=1)+ h_next = h[:, :, 1:, :, :]+ h_next = rearrange(h_next, "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", r1=r1, r2=2, r3=2)+ h = torch.cat([h_first, h_next], dim=2)+ # shortcut computation+ x_first = x[:, :, :1, :, :]+ x_first = rearrange(x_first, "b c f (h r2) (w r3) -> b (r2 r3 c) f h w", r2=2, r3=2)+ B, C, T, H, W = x_first.shape+ x_first = x_first.view(B, h.shape[1], self.group_size // 2, T, H, W).mean(dim=2)++ x_next = x[:, :, 1:, :, :]+ del x+ x_next = rearrange(x_next, "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", r1=r1, r2=2, r3=2)+ B, C, T, H, W = x_next.shape+ x_next = x_next.view(B, h.shape[1], self.group_size, T, H, W).mean(dim=2)+ shortcut = torch.cat([x_first, x_next], dim=2)+ else:+ h = rearrange(h, "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", r1=r1, r2=2, r3=2)+ shortcut = rearrange(x, "b c (f r1) (h r2) (w r3) -> b (r1 r2 r3 c) f h w", r1=r1, r2=2, r3=2)+ B, C, T, H, W = shortcut.shape+ shortcut = shortcut.view(B, h.shape[1], self.group_size, T, H, W).mean(dim=2)++ h += shortcut+ return h+++++class Upsample(nn.Module):+ """Hierarchical upsampling with temporal/ spatial support."""++ def __init__(self, in_channels: int, out_channels: int, add_temporal_upsample: bool = True):+ super().__init__()+ factor = 2 * 2 * 2 if add_temporal_upsample else 1 * 2 * 2+ self.conv = CausalConv3d(in_channels, out_channels * factor, kernel_size=3)+ self.add_temporal_upsample = add_temporal_upsample+ self.repeats = factor * out_channels // in_channels++ def forward(self, x_list: Tensor):+ x = x_list[0]+ x_list.clear()+ r1 = 2 if self.add_temporal_upsample else 1+ h = self.conv(x)+ if self.add_temporal_upsample:+ h_first = h[:, :, :1, :, :]+ h_first = rearrange(h_first, "b (r2 r3 c) f h w -> b c f (h r2) (w r3)", r2=2, r3=2)+ h_first = h_first[:, : h_first.shape[1] // 2]+ h_next = h[:, :, 1:, :, :]+ h_next = rearrange(h_next, "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", r1=r1, r2=2, r3=2)+ h = torch.cat([h_first, h_next], dim=2)++ # shortcut computation+ x_first = x[:, :, :1, :, :]+ x_first = rearrange(x_first, "b (r2 r3 c) f h w -> b c f (h r2) (w r3)", r2=2, r3=2)+ x_first = x_first.repeat_interleave(repeats=self.repeats // 2, dim=1)++ x_next = x[:, :, 1:, :, :]+ del x+ x_next = rearrange(x_next, "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", r1=r1, r2=2, r3=2)+ x_next = x_next.repeat_interleave(repeats=self.repeats, dim=1)+ shortcut = torch.cat([x_first, x_next], dim=2)++ else:+ h = rearrange(h, "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", r1=r1, r2=2, r3=2)+ shortcut = x.repeat_interleave(repeats=self.repeats, dim=1)+ del x+ shortcut = rearrange(shortcut, "b (r1 r2 r3 c) f h w -> b c (f r1) (h r2) (w r3)", r1=r1, r2=2, r3=2)+ h += shortcut+ return h+++class Encoder(nn.Module):+ """Hierarchical video encoder with temporal and spatial factorization."""++ def __init__(+ self,+ in_channels: int,+ z_channels: int,+ block_out_channels: Tuple[int, ...],+ num_res_blocks: int,+ ffactor_spatial: int,+ ffactor_temporal: int,+ downsample_match_channel: bool = True,+ ):+ super().__init__()+ assert block_out_channels[-1] % (2 * z_channels) == 0++ self.z_channels = z_channels+ self.block_out_channels = block_out_channels+ self.num_res_blocks = num_res_blocks++ # downsampling+ self.conv_in = CausalConv3d(in_channels, block_out_channels[0], kernel_size=3)++ self.down = nn.ModuleList()+ block_in = block_out_channels[0]+ for i_level, ch in enumerate(block_out_channels):+ block = nn.ModuleList()+ block_out = ch+ for _ in range(self.num_res_blocks):+ block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))+ block_in = block_out+ down = nn.Module()+ down.block = block++ add_spatial_downsample = bool(i_level < np.log2(ffactor_spatial))+ add_temporal_downsample = add_spatial_downsample and bool(i_level >= np.log2(ffactor_spatial // ffactor_temporal))+ if add_spatial_downsample or add_temporal_downsample:+ assert i_level < len(block_out_channels) - 1+ block_out = block_out_channels[i_level + 1] if downsample_match_channel else block_in+ down.downsample = Downsample(block_in, block_out, add_temporal_downsample)+ block_in = block_out+ self.down.append(down)++ # middle+ self.mid = nn.Module()+ self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)+ self.mid.attn_1 = AttnBlock(block_in)+ self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)++ # end+ self.norm_out = RMS_norm(block_in, images=False)+ self.conv_out = CausalConv3d(block_in, 2 * z_channels, kernel_size=3)++ self.gradient_checkpointing = False++ def forward(self, x: Tensor) -> Tensor:+ """Forward pass through the encoder."""+ use_checkpointing = bool(self.training and self.gradient_checkpointing)++ # downsampling+ h = self.conv_in(x)+ for i_level in range(len(self.block_out_channels)):+ for i_block in range(self.num_res_blocks):+ x_list = [h]+ del h+ h = self.down[i_level].block[i_block](x_list)+ if hasattr(self.down[i_level], "downsample"):+ x_list = [h]+ del h+ h = self.down[i_level].downsample(x_list)++ # middle+ x_list = [h]+ del h+ h = self.mid.block_1(x_list)+ h = self.mid.attn_1(h)+ x_list = [h]+ del h+ h = self.mid.block_2(x_list)++ # end+ group_size = self.block_out_channels[-1] // (2 * self.z_channels)+ shortcut = rearrange(h, "b (c r) f h w -> b c r f h w", r=group_size).mean(dim=2)+ h = self.norm_out(h)+ h = swish(h, inplace=True)+ h = self.conv_out(h)+ h += shortcut+ return h+++class Decoder(nn.Module):+ """Hierarchical video decoder with upsampling factories."""++ def __init__(+ self,+ z_channels: int,+ out_channels: int,+ block_out_channels: Tuple[int, ...],+ num_res_blocks: int,+ ffactor_spatial: int,+ ffactor_temporal: int,+ upsample_match_channel: bool = True,+ ):+ super().__init__()+ assert block_out_channels[0] % z_channels == 0++ self.z_channels = z_channels+ self.block_out_channels = block_out_channels+ self.num_res_blocks = num_res_blocks++ block_in = block_out_channels[0]+ self.conv_in = CausalConv3d(z_channels, block_in, kernel_size=3)++ # middle+ self.mid = nn.Module()+ self.mid.block_1 = ResnetBlock(in_channels=block_in, out_channels=block_in)+ self.mid.attn_1 = AttnBlock(block_in)+ self.mid.block_2 = ResnetBlock(in_channels=block_in, out_channels=block_in)++ # upsampling+ self.up = nn.ModuleList()+ for i_level, ch in enumerate(block_out_channels):+ block = nn.ModuleList()+ block_out = ch+ for _ in range(self.num_res_blocks + 1):+ block.append(ResnetBlock(in_channels=block_in, out_channels=block_out))+ block_in = block_out+ up = nn.Module()+ up.block = block++ add_spatial_upsample = bool(i_level < np.log2(ffactor_spatial))+ add_temporal_upsample = bool(i_level < np.log2(ffactor_temporal))+ if add_spatial_upsample or add_temporal_upsample:+ assert i_level < len(block_out_channels) - 1+ block_out = block_out_channels[i_level + 1] if upsample_match_channel else block_in+ up.upsample = Upsample(block_in, block_out, add_temporal_upsample)+ block_in = block_out+ self.up.append(up)++ # end+ self.norm_out = RMS_norm(block_in, images=False)+ self.conv_out = CausalConv3d(block_in, out_channels, kernel_size=3)++ self.gradient_checkpointing = False++ def forward(self, z: Tensor) -> Tensor:+ """Forward pass through the decoder."""++ # z to block_in+ repeats = self.block_out_channels[0] // (self.z_channels)+ h = self.conv_in(z) + z.repeat_interleave(repeats=repeats, dim=1)++ # middle+ h_list = [h]+ del h+ h = self.mid.block_1(h_list)+ # h_list = [h]+ # del h+ h = self.mid.attn_1(h)+ h_list = [h]+ del h+ h = self.mid.block_2(h_list)++ # upsampling+ for i_level in range(len(self.block_out_channels)):+ for i_block in range(self.num_res_blocks + 1):+ x_list = [h]+ del h+ h = self.up[i_level].block[i_block](x_list )+ if hasattr(self.up[i_level], "upsample"):+ x_list = [h]+ del h+ h = self.up[i_level].upsample(x_list )++ # end+ h = self.norm_out(h).to(z.dtype)+ h = swish(h, inplace=True)+ h = self.conv_out(h)+ return h+++class AutoencoderKLConv3D(ModelMixin, ConfigMixin):+ """KL regularized 3D Conv VAE with advanced tiling and slicing strategies."""+ _supports_gradient_checkpointing = True++ @register_to_config+ def __init__(+ self,+ in_channels: int,+ out_channels: int,+ latent_channels: int,+ block_out_channels: Tuple[int, ...],+ layers_per_block: int,+ ffactor_spatial: int,+ ffactor_temporal: int,+ sample_size: int,+ sample_tsize: int,+ scaling_factor: float = None,+ shift_factor: Optional[float] = None,+ downsample_match_channel: bool = True,+ upsample_match_channel: bool = True,+ ):+ super().__init__()+ self.ffactor_spatial = ffactor_spatial+ self.ffactor_temporal = ffactor_temporal+ self.scaling_factor = scaling_factor+ self.shift_factor = shift_factor++ self.encoder = Encoder(+ in_channels=in_channels,+ z_channels=latent_channels,+ block_out_channels=block_out_channels,+ num_res_blocks=layers_per_block,+ ffactor_spatial=ffactor_spatial,+ ffactor_temporal=ffactor_temporal,+ downsample_match_channel=downsample_match_channel,+ )+ self.decoder = Decoder(+ z_channels=latent_channels,+ out_channels=out_channels,+ block_out_channels=list(reversed(block_out_channels)),+ num_res_blocks=layers_per_block,+ ffactor_spatial=ffactor_spatial,+ ffactor_temporal=ffactor_temporal,+ upsample_match_channel=upsample_match_channel,+ )++ self.use_slicing = False+ self.use_spatial_tiling = False+ self.use_temporal_tiling = False++ # only relevant if vae tiling is enabled+ self.tile_sample_min_size = sample_size+ self.tile_latent_min_size = sample_size // ffactor_spatial+ self.tile_sample_min_tsize = sample_tsize+ self.tile_latent_min_tsize = sample_tsize // ffactor_temporal+ self.tile_overlap_factor = 0.25++ self._tile_parallelism_enabled = False++ def set_tile_sample_min_size(self, sample_size: int, tile_overlap_factor: float = 0.2):+ self.tile_sample_min_size = sample_size+ self.tile_latent_min_size = sample_size // self.ffactor_spatial+ self.tile_overlap_factor = tile_overlap_factor++ assert (self.tile_latent_min_size * self.tile_overlap_factor).is_integer(), \+ "self.tile_latent_min_size multiplied by tile_overlap_factor must be an integer"+++ def _set_gradient_checkpointing(self, module, value=False):+ """Enable or disable gradient checkpointing on encoder and decoder."""+ if isinstance(module, (Encoder, Decoder)):+ module.gradient_checkpointing = value+++ def enable_temporal_tiling(self, use_tiling: bool = True):+ raise RuntimeError('Temporal tiling is not supported for this VAE.')+ self.use_temporal_tiling = use_tiling++ def disable_temporal_tiling(self):+ self.enable_temporal_tiling(False)++ def enable_spatial_tiling(self, use_tiling: bool = True):+ self.use_spatial_tiling = use_tiling++ def disable_spatial_tiling(self):+ self.enable_spatial_tiling(False)++ def enable_tiling(self, use_tiling: bool = True):+ self.enable_spatial_tiling(use_tiling)++ def disable_tiling(self):+ self.disable_spatial_tiling()++ def enable_slicing(self):+ self.use_slicing = True++ def disable_slicing(self):+ self.use_slicing = False++ def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int):+ """Blend tensor b horizontally into a at blend_extent region."""+ blend_extent = min(a.shape[-1], b.shape[-1], blend_extent)+ for x in range(blend_extent):+ b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, :, x] * (x / blend_extent)+ return b++ def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int):+ """Blend tensor b vertically into a at blend_extent region."""+ blend_extent = min(a.shape[-2], b.shape[-2], blend_extent)+ for y in range(blend_extent):+ b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, :, y, :] * (y / blend_extent)+ return b++ def blend_t(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int):+ """Blend tensor b temporally into a at blend_extent region."""+ blend_extent = min(a.shape[-3], b.shape[-3], blend_extent)+ for x in range(blend_extent):+ b[:, :, x, :, :] = a[:, :, -blend_extent + x, :, :] * (1 - x / blend_extent) + b[:, :, x, :, :] * (x / blend_extent)+ return b++ def spatial_tiled_encode(self, x: torch.Tensor):+ """Tiled spatial encoding for large inputs via overlapping."""+ B, C, T, H, W = x.shape+ overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor))+ blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor)+ row_limit = self.tile_latent_min_size - blend_extent++ rows = []+ for i in range(0, H, overlap_size):+ row = []+ for j in range(0, W, overlap_size):+ tile = x[:, :, :, i: i + self.tile_sample_min_size, j: j + self.tile_sample_min_size]+ tile = self.encoder(tile)+ row.append(tile)+ rows.append(row)+ result_rows = []+ for i, row in enumerate(rows):+ result_row = []+ for j, tile in enumerate(row):+ if i > 0:+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent)+ if j > 0:+ tile = self.blend_h(row[j - 1], tile, blend_extent)+ result_row.append(tile[:, :, :, :row_limit, :row_limit])+ result_rows.append(torch.cat(result_row, dim=-1))+ moments = torch.cat(result_rows, dim=-2)+ return moments++ def temporal_tiled_encode(self, x: torch.Tensor):+ """Tiled temporal encoding for large video sequences."""+ raise RuntimeError('Temporal tiling is not supported for this VAE.')+ B, C, T, H, W = x.shape+ overlap_size = int(self.tile_sample_min_tsize * (1 - self.tile_overlap_factor))+ blend_extent = int(self.tile_latent_min_tsize * self.tile_overlap_factor)+ t_limit = self.tile_latent_min_tsize - blend_extent++ row = []+ for i in range(0, T, overlap_size):+ tile = x[:, :, i: i + self.tile_sample_min_tsize + 1, :, :]+ if self.use_spatial_tiling and (tile.shape[-1] > self.tile_sample_min_size or tile.shape[-2] > self.tile_sample_min_size):+ tile = self.spatial_tiled_encode(tile)+ else:+ tile = self.encoder(tile)+ if i > 0:+ tile = tile[:, :, 1:, :, :]+ row.append(tile)+ result_row = []+ for i, tile in enumerate(row):+ if i > 0:+ tile = self.blend_t(row[i - 1], tile, blend_extent)+ result_row.append(tile[:, :, :t_limit, :, :])+ else:+ result_row.append(tile[:, :, :t_limit + 1, :, :])+ moments = torch.cat(result_row, dim=-3)+ return moments++ def enable_tile_parallelism(self):+ self._tile_parallelism_enabled = True++ def disable_tile_parallelism(self):+ self._tile_parallelism_enabled = False++ def tile_parallel_spatial_tiled_decode(self, z: torch.Tensor):+ B, C, T, H, W = z.shape+ overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor))+ blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor)+ row_limit = self.tile_sample_min_size - blend_extent+++ rank = get_parallel_state().sp_rank+ world_size = get_parallel_state().sp++ num_rows = math.ceil(H / overlap_size)+ num_cols = math.ceil(W / overlap_size)+ total_tiles = num_rows * num_cols+ tiles_per_rank = math.ceil(total_tiles / world_size)+++ my_linear_indices = list(range(rank, total_tiles, world_size))+ decoded_tiles = []+ decoded_metas = []+ H_out_std = self.tile_sample_min_size+ W_out_std = self.tile_sample_min_size+ for lin_idx in my_linear_indices:+ ri = lin_idx // num_cols+ rj = lin_idx % num_cols+ i = ri * overlap_size+ j = rj * overlap_size+ tile = z[:, :, :, i:i + self.tile_latent_min_size, j:j + self.tile_latent_min_size]+ dec = self.decoder(tile)++ pad_h = max(0, H_out_std - dec.shape[-2])+ pad_w = max(0, W_out_std - dec.shape[-1])+ if pad_h > 0 or pad_w > 0:+ dec = F.pad(dec, (0, pad_w, 0, pad_h, 0, 0), "constant", 0)+ decoded_tiles.append(dec)+ decoded_metas.append(torch.tensor([ri, rj, pad_w, pad_h], device=z.device, dtype=torch.int64))++ while len(decoded_tiles) < tiles_per_rank:+ zero_tile = torch.zeros(+ [1, 3, (T - 1) * self.ffactor_temporal + 1, self.tile_sample_min_size, self.tile_sample_min_size],+ device=dec.device,+ dtype=dec.dtype+ )+ decoded_tiles.append(zero_tile)+ meta_tensor = torch.tensor(+ [-1, -1, self.tile_sample_min_size, self.tile_sample_min_size],+ device=z.device,+ dtype=torch.int64+ )+ decoded_metas.append(meta_tensor)++ decoded_tiles = torch.stack(decoded_tiles, dim=0)+ decoded_metas = torch.stack(decoded_metas, dim=0)++ tiles_gather_list = [torch.empty_like(decoded_tiles) for _ in range(world_size)]+ metas_gather_list = [torch.empty_like(decoded_metas) for _ in range(world_size)]++ dist.all_gather(tiles_gather_list, decoded_tiles, group=get_parallel_state().sp_group)+ dist.all_gather(metas_gather_list, decoded_metas, group=get_parallel_state().sp_group)++ if rank != 0:+ return torch.empty(0, device=z.device)++ rows = [[None for _ in range(num_cols)] for _ in range(num_rows)]+ for r in range(world_size):+ gathered_tiles_r = tiles_gather_list[r] # [tiles_per_rank, B, C, T, H, W]+ gathered_metas_r = metas_gather_list[r] # [tiles_per_rank, 4]+ for k in range(gathered_tiles_r.shape[0]):+ ri = int(gathered_metas_r[k][0])+ rj = int(gathered_metas_r[k][1])+ if ri < 0 or rj < 0:+ continue+ if ri < num_rows and rj < num_cols:+ # remove padding+ pad_w = int(gathered_metas_r[k][2])+ pad_h = int(gathered_metas_r[k][3])+ h_end = None if pad_h == 0 else -pad_h+ w_end = None if pad_w == 0 else -pad_w+ rows[ri][rj] = gathered_tiles_r[k][:, :, :, :h_end, :w_end]++ result_rows = []+ for i, row in enumerate(rows):+ result_row = []+ for j, tile in enumerate(row):+ if tile is None:+ continue+ if i > 0:+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent)+ if j > 0:+ tile = self.blend_h(row[j - 1], tile, blend_extent)+ result_row.append(tile[:, :, :, :row_limit, :row_limit])+ result_rows.append(torch.cat(result_row, dim=-1))++ dec = torch.cat(result_rows, dim=-2)+ return dec++ def spatial_tiled_decode(self, z: torch.Tensor):+ if self._tile_parallelism_enabled:+ return self.tile_parallel_spatial_tiled_decode(z)++ B, C, T, H, W = z.shape+ overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor))+ blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor)+ row_limit = self.tile_sample_min_size - blend_extent++ rows = []+ for i in range(0, H, overlap_size):+ row = []+ for j in range(0, W, overlap_size):+ tile = z[:, :, :, i: i + self.tile_latent_min_size, j: j + self.tile_latent_min_size]+ decoded = self.decoder(tile)+ row.append(decoded)+ rows.append(row)++ result_rows = []+ for i, row in enumerate(rows):+ result_row = []+ for j, tile in enumerate(row):+ if i > 0:+ tile = self.blend_v(rows[i - 1][j], tile, blend_extent)+ if j > 0:+ tile = self.blend_h(row[j - 1], tile, blend_extent)+ result_row.append(tile[:, :, :, :row_limit, :row_limit])+ result_rows.append(torch.cat(result_row, dim=-1))+ dec = torch.cat(result_rows, dim=-2)+ return dec++ def temporal_tiled_decode(self, z: torch.Tensor):+ """Tiled temporal decoding for long sequence latents."""+ raise RuntimeError('Temporal tiling is not supported for this VAE.')+ B, C, T, H, W = z.shape+ overlap_size = int(self.tile_latent_min_tsize * (1 - self.tile_overlap_factor))+ blend_extent = int(self.tile_sample_min_tsize * self.tile_overlap_factor)+ t_limit = self.tile_sample_min_tsize - blend_extent+ assert 0 < overlap_size < self.tile_latent_min_tsize++ row = []+ for i in range(0, T, overlap_size):+ tile = z[:, :, i: i + self.tile_latent_min_tsize + 1, :, :]+ if self.use_spatial_tiling and (tile.shape[-1] > self.tile_latent_min_size or tile.shape[-2] > self.tile_latent_min_size):+ decoded = self.spatial_tiled_decode(tile)+ else:+ decoded = self.decoder(tile)+ if i > 0:+ decoded = decoded[:, :, 1:, :, :]+ row.append(decoded)++ result_row = []+ for i, tile in enumerate(row):+ if i > 0:+ tile = self.blend_t(row[i - 1], tile, blend_extent)+ result_row.append(tile[:, :, :t_limit, :, :])+ else:+ result_row.append(tile[:, :, :t_limit + 1, :, :])+ dec = torch.cat(result_row, dim=-3)+ return dec++ def encode(self, x: Tensor, return_dict: bool = True):++ def _encode(x):+ if self.use_temporal_tiling and x.shape[-3] > self.tile_sample_min_tsize:+ return self.temporal_tiled_encode(x)+ if self.use_spatial_tiling and (x.shape[-1] > self.tile_sample_min_size or x.shape[-2] > self.tile_sample_min_size):+ return self.spatial_tiled_encode(x)+ return self.encoder(x)++ assert len(x.shape) == 5 # (B, C, T, H, W)++ if self.use_slicing and x.shape[0] > 1:+ encoded_slices = [_encode(x_slice) for x_slice in x.split(1)]+ h = torch.cat(encoded_slices)+ else:+ h = _encode(x)+ posterior = DiagonalGaussianDistribution(h)++ if not return_dict:+ return (posterior,)++ return AutoencoderKLOutput(latent_dist=posterior)++ def decode(self, z: Tensor, return_dict: bool = True, generator=None):++ def _decode(z):+ if self.use_temporal_tiling and z.shape[-3] > self.tile_latent_min_tsize:+ return self.temporal_tiled_decode(z)+ if self.use_spatial_tiling and (z.shape[-1] > self.tile_latent_min_size or z.shape[-2] > self.tile_latent_min_size):+ return self.spatial_tiled_decode(z)+ return self.decoder(z)++ if self.use_slicing and z.shape[0] > 1:+ decoded_slices = [_decode(z_slice) for z_slice in z.split(1)]+ decoded = torch.cat(decoded_slices)+ else:+ decoded = _decode(z)++ if not return_dict:+ return (decoded,)++ return DecoderOutput(sample=decoded)++ def forward(+ self,+ sample: torch.Tensor,+ sample_posterior: bool = False,+ return_posterior: bool = True,+ return_dict: bool = True+ ):+ """Forward autoencoder pass. Returns both reconstruction and optionally the posterior."""+ posterior = self.encode(sample).latent_dist+ z = posterior.sample() if sample_posterior else posterior.mode()+ dec = self.decode(z).sample+ return DecoderOutput(sample=dec, posterior=posterior) if return_dict else (dec, posterior)++++++ def get_VAE_tile_size(self, vae_config, device_mem_capacity, mixed_precision):+ if mixed_precision:+ device_mem_capacity /= 1.5+ if vae_config == 0:+ if device_mem_capacity >= 24000:+ use_vae_config = 1+ elif device_mem_capacity >= 12000:+ use_vae_config = 2+ else:+ use_vae_config = 3+ else:+ use_vae_config = vae_config++ if use_vae_config == 1:+ sample_size = 256+ tile_overlap_factor = 0.25+ elif use_vae_config == 2:+ sample_size = 160+ tile_overlap_factor = 0.2+ else:+ sample_size = 128+ tile_overlap_factor = 0.25++ VAE_tiling = {+ "tile_sample_min_size" : sample_size,+ "tile_latent_min_size" : int(sample_size / (2 ** (len(self.config.block_out_channels) - 1))),+ "tile_overlap_factor" : tile_overlap_factor+ }+ return VAE_tilingdiff --git a/models/hyvideo/vision_encoder/__init__.py b/models/hyvideo/vision_encoder/__init__.pynew file mode 100644index 0000000..871dd37--- /dev/null+++ b/models/hyvideo/vision_encoder/__init__.py@@ -0,0 +1,241 @@+# Licensed under the TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5/blob/main/LICENSE+#+# Unless and only to the extent required by applicable law, the Tencent Hunyuan works and any+# output and results therefrom are provided "AS IS" without any express or implied warranties of+# any kind including any warranties of title, merchantability, noninfringement, course of dealing,+# usage of trade, or fitness for a particular purpose. You are solely responsible for determining the+# appropriateness of using, reproducing, modifying, performing, displaying or distributing any of+# the Tencent Hunyuan works or outputs and assume any and all risks associated with your or a+# third party's use or distribution of any of the Tencent Hunyuan works or outputs and your exercise+# of rights and permissions under this agreement.+# See the License for the specific language governing permissions and limitations under the License.++from dataclasses import dataclass+from typing import Optional, Tuple+import torch+import torch.nn as nn+from transformers import SiglipImageProcessor, SiglipVisionModel+from transformers.utils import ModelOutput+import numpy as np++PRECISION_TO_TYPE = {+ 'fp32': torch.float32,+ 'fp16': torch.float16,+ 'bf16': torch.bfloat16,+}+++VISION_ENCODER_PATH = {}+++def use_default(value, default):+ return value if value is not None else default+++def load_vision_encoder(+ vision_encoder_type,+ vision_encoder_precision=None,+ vision_encoder_path=None,+ logger=None,+ device=None,+):+ if vision_encoder_path is None:+ vision_encoder_path = VISION_ENCODER_PATH[vision_encoder_type]++ if vision_encoder_type == "siglip":+ vision_encoder = SiglipVisionModel.from_pretrained(+ vision_encoder_path,+ # subfolder='image_encoder'+ )+ else:+ raise ValueError(f"Unsupported vision encoder type: {vision_encoder_type}")++ # from_pretrained will ensure that the model is in eval mode.+ if vision_encoder_precision is not None:+ vision_encoder = vision_encoder.to(dtype=PRECISION_TO_TYPE[vision_encoder_precision])++ vision_encoder.requires_grad_(False)++ if device is not None:+ vision_encoder = vision_encoder.to(device)++ return vision_encoder, vision_encoder_path+++def load_image_processor(+ processor_type,+ processor_path=None,+ logger=None+):+ if processor_path is None:+ processor_path = VISION_ENCODER_PATH[processor_type]++ if processor_type == "siglip":+ processor = SiglipImageProcessor.from_pretrained(+ processor_path,+ # subfolder='feature_extractor'+ )+ else:+ raise ValueError(f"Unsupported processor type: {processor_type}")++ return processor, processor_path+++@dataclass+class VisionEncoderModelOutput(ModelOutput):+ """+ Base class for vision encoder model's outputs.++ Args:+ last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):+ Sequence of hidden-states at the output of the last layer of the model.+ pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*):+ Last layer hidden-state of the first token of the sequence (classification token)+ after further processing through the layers used for the auxiliary pretraining task.+ hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed):+ Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, ++ one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.+ Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.+ """++ last_hidden_state: torch.FloatTensor = None+ pooler_output: Optional[torch.FloatTensor] = None+ hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None+++class VisionEncoder(nn.Module):+ def __init__(+ self,+ vision_encoder_type: str,+ vision_encoder_precision: Optional[str] = None,+ vision_encoder_path: Optional[str] = None,+ processor_type: Optional[str] = None,+ processor_path: Optional[str] = None,+ output_key: Optional[str] = None,+ logger=None,+ device=None,+ ):+ super().__init__()+ self.vision_encoder_type = vision_encoder_type+ self.precision = vision_encoder_precision+ self.model_path = vision_encoder_path+ self.processor_type = (+ processor_type if processor_type is not None else vision_encoder_type+ )+ self.processor_path = (+ processor_path if processor_path is not None else vision_encoder_path+ )+ self.logger = logger++ if "siglip" in vision_encoder_type:+ self.output_key = output_key or "last_hidden_state"+ else:+ raise ValueError(f"Unsupported vision encoder type: {vision_encoder_type}")++ self.model, self.model_path = load_vision_encoder(+ vision_encoder_type=self.vision_encoder_type,+ vision_encoder_precision=self.precision,+ vision_encoder_path=self.model_path,+ logger=self.logger,+ device=device,+ )+ self.dtype = self.model.dtype+ self.device = self.model.device++ self.processor, self.processor_path = load_image_processor(+ processor_type=self.processor_type,+ processor_path=self.processor_path,+ logger=self.logger,+ )++ def __repr__(self):+ return f"{self.vision_encoder_type} ({self.precision} - {self.model_path})"++ def encode_latents_to_images(self, latents, vae, reorg_token=False):+ """+ Convert latents to images using VAE decoder.++ Args:+ latents: Input latents tensor+ vae: VAE model for decoding+ reorg_token: Whether to reorg the token+ Returns:+ images: Decoded images as numpy array+ """+ # Handle both 4D and 5D latents (for video, take first frame)+ first_image_latents = latents[:, :, 0, ...] if len(latents.shape) == 5 else latents+ first_image_latents = 1 / vae.config.scaling_factor * first_image_latents+ first_image = vae.decode(first_image_latents.unsqueeze(2).to(vae.dtype), return_dict=False)[0].cpu()+ first_image = first_image[:, :, 0, :, :]+ first_image = (first_image / 2 + 0.5).clamp(0, 1)+ first_image = (first_image * 255.0).clamp(0, 255.0)+ first_image = first_image.to(torch.uint8).numpy()+ first_image = first_image.transpose(0, 2, 3, 1)++ assert isinstance(first_image, np.ndarray)+ assert first_image.ndim == 4 and first_image.shape[3] == 3+ assert first_image.dtype == np.uint8++ return first_image++ def encode_images(self, images):+ """+ Encode images using the vision encoder.++ Args:+ images: Input images (numpy array or preprocessed tensor)++ Returns:+ VisionEncoderModelOutput with encoded features+ """+ if isinstance(images, np.ndarray):+ # Preprocess images if they're numpy arrays+ preprocessed = self.processor.preprocess(images=images, return_tensors="pt").to(+ device=self.model.device, dtype=self.model.dtype)+ else:+ # Assume already preprocessed+ preprocessed = images++ outputs = self.model(**preprocessed)++ return VisionEncoderModelOutput(+ last_hidden_state=outputs.last_hidden_state,+ pooler_output=outputs.pooler_output if hasattr(outputs, 'pooler_output') else None,+ hidden_states=outputs.hidden_states if hasattr(outputs, 'hidden_states') else None+ )++ def encode_latents(self, latents, vae, reorg_token=False):+ """+ Encode latents by first converting to images, then encoding.+ This is the main function that replaces sigclip_vision_encode.++ Args:+ latents: Input latent tensors+ vae: VAE model for decoding latents to images++ Returns:+ Encoded image features+ """+ # Convert latents to images+ images = self.encode_latents_to_images(latents, vae, reorg_token)++ # Encode images+ outputs = self.encode_images(images)++ return outputs.last_hidden_state++ def forward(self, images):+ """+ Forward pass for direct image encoding.++ Args:+ images: Input images++ Returns:+ VisionEncoderModelOutput with encoded features+ """+ return self.encode_images(images)\ No newline at end of filediff --git a/models/qwen/qwen_handler.py b/models/qwen/qwen_handler.pyindex 597de8b..632181b 100644--- a/models/qwen/qwen_handler.py+++ b/models/qwen/qwen_handler.py@@ -79,7 +79,7 @@ class family_handler():download_def = [{"repoId" : "DeepBeepMeep/Qwen_image","sourceFolderList" : ["", "Qwen2.5-VL-7B-Instruct"],- "fileList" : [ ["qwen_vae.safetensors", "qwen_vae_config.json"], ["merges.txt", "tokenizer_config.json", "config.json", "vocab.json", "video_preprocessor_config.json", "preprocessor_config.json"] + computeList(text_encoder_filename) ]+ "fileList" : [ ["qwen_vae.safetensors", "qwen_vae_config.json"], ["merges.txt", "tokenizer_config.json", "config.json", "vocab.json", "video_preprocessor_config.json", "preprocessor_config.json", "chat_template.json"] + computeList(text_encoder_filename) ]}]download_def += [{diff --git a/models/wan/wan_handler.py b/models/wan/wan_handler.pyindex 20aef4a..77c9e2e 100644--- a/models/wan/wan_handler.py+++ b/models/wan/wan_handler.py@@ -139,6 +139,8 @@ class family_handler():extra_model_def["alpha_class"] = alpha = test_alpha(base_model_type)extra_model_def["wan_5B_class"] = wan_5B = test_wan_5B(base_model_type)extra_model_def["vace_class"] = vace_class = test_vace(base_model_type)+ extra_model_def["color_correction"] = True+if base_model_type in ["vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B"]:extra_model_def["parent_model_type"] = "vace_14B"diff --git a/plugins/wan2gp-about/plugin.py b/plugins/wan2gp-about/plugin.pyindex 9c8d8ee..f48656a 100644--- a/plugins/wan2gp-about/plugin.py+++ b/plugins/wan2gp-about/plugin.py@@ -18,9 +18,9 @@ class AboutPlugin(WAN2GPPlugin):def create_about_ui(self):gr.Markdown("<H2>WanGP - AI Generative Models for the GPU Poor by <B>DeepBeepMeep</B> (<A HREF='https://github.com/deepbeepmeep/Wan2GP'>GitHub</A>)</H2>")gr.Markdown("Many thanks to:")- gr.Markdown("- <B>Alibaba Wan Team</B> for the best open source video generators (https://github.com/Wan-Video/Wan2.1)")+ gr.Markdown("- <B>Alibaba Wan Team</B> for the best open source video generators (https://github.com/Wan-Video/Wan2.1, https://github.com/Wan-Video/Wan2.2)")gr.Markdown("- <B>Alibaba Vace, Multitalk and Fun Teams</B> for their incredible control net models (https://github.com/ali-vilab/VACE), (https://github.com/MeiGen-AI/MultiTalk) and (https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-InP) ")- gr.Markdown("- <B>Tencent</B> for the impressive Hunyuan Video models (https://github.com/Tencent-Hunyuan/HunyuanVideo)")+ gr.Markdown("- <B>Tencent</B> for the impressive Hunyuan Video models (https://github.com/Tencent-Hunyuan/HunyuanVideo, https://github.com/Tencent-Hunyuan/HunyuanVideo-1.5)")gr.Markdown("- <B>Blackforest Labs</B> for the innovative Flux image generators (https://github.com/black-forest-labs/flux)")gr.Markdown("- <B>Alibaba Qwen Team</B> for their state of the art Qwen Image generators (https://github.com/QwenLM/Qwen-Image)")gr.Markdown("- <B>Lightricks</B> for their super fast LTX Video models (https://github.com/Lightricks/LTX-Video)")@@ -35,6 +35,7 @@ class AboutPlugin(WAN2GPPlugin):gr.Markdown("- <B>DepthAnything</B> & <B>Midas</B>: Depth extractors (https://github.com/DepthAnything/Depth-Anything-V2) and (https://github.com/isl-org/MiDaS")gr.Markdown("- <B>Matanyone</B> and <B>SAM2</B>: Mask Generation (https://github.com/pq-yang/MatAnyone) and (https://github.com/facebookresearch/sam2)")gr.Markdown("- <B>Pyannote</B>: speaker diarization (https://github.com/pyannote/pyannote-audio)")+ gr.Markdown("- <B>MMAudio</B>: sound generator (https://github.com/hkchengrex/MMAudio). Due to licensing restriction can be used only for Research work.")gr.Markdown("<BR>Special thanks to the following people for their Contributions & Support:")gr.Markdown("- <B>Tophness</B> : Designed & developped the Queuing Framework, Edit Mode and WanGP PlugIns System")diff --git a/postprocessing/mmaudio/data/av_utils.py b/postprocessing/mmaudio/data/av_utils.pyindex 7866d6c..635c053 100644--- a/postprocessing/mmaudio/data/av_utils.py+++ b/postprocessing/mmaudio/data/av_utils.py@@ -4,6 +4,7 @@ from pathlib import Pathfrom typing import Optionalimport av+import cv2import numpy as npimport torchfrom av import AudioFrame@@ -54,38 +55,46 @@ class ImageInfo:def read_frames(video_path: Path, list_of_fps: list[float], start_sec: float, end_sec: float,need_all_frames: bool) -> tuple[list[np.ndarray], list[np.ndarray], Fraction]:+ cap = cv2.VideoCapture(str(video_path))+ if not cap.isOpened():+ raise RuntimeError(f"Could not open {video_path}")++ fps_val = cap.get(cv2.CAP_PROP_FPS)+ if not fps_val or fps_val <= 0:+ cap.release()+ raise RuntimeError(f"Could not read fps from {video_path}")+ fps = Fraction(fps_val).limit_denominator()++ start_frame = int(start_sec * fps_val)+ end_frame = int(end_sec * fps_val)+ cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame)+output_frames = [[] for _ in list_of_fps]- next_frame_time_for_each_fps = [0.0 for _ in list_of_fps]- time_delta_for_each_fps = [1 / fps for fps in list_of_fps]+ next_frame_time_for_each_fps = [start_sec for _ in list_of_fps]+ time_delta_for_each_fps = [1 / f for f in list_of_fps]all_frames = []- # container = av.open(video_path)- with av.open(video_path) as container:- stream = container.streams.video[0]- fps = stream.guessed_rate- stream.thread_type = 'AUTO'- for packet in container.demux(stream):- for frame in packet.decode():- frame_time = frame.time- if frame_time < start_sec:- continue- if frame_time > end_sec:- break-- frame_np = None- if need_all_frames:- frame_np = frame.to_ndarray(format='rgb24')- all_frames.append(frame_np)-- for i, _ in enumerate(list_of_fps):- this_time = frame_time- while this_time >= next_frame_time_for_each_fps[i]:- if frame_np is None:- frame_np = frame.to_ndarray(format='rgb24')-- output_frames[i].append(frame_np)- next_frame_time_for_each_fps[i] += time_delta_for_each_fps[i]-+ frame_idx = start_frame+ while frame_idx <= end_frame:+ ok, frame_bgr = cap.read()+ if not ok:+ break+ frame_idx += 1+ frame_time = frame_idx / fps_val # seconds+ frame_rgb = None++ if need_all_frames:+ frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)+ all_frames.append(frame_rgb)++ for i, _ in enumerate(list_of_fps):+ while frame_time >= next_frame_time_for_each_fps[i]:+ if frame_rgb is None:+ frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)+ output_frames[i].append(frame_rgb)+ next_frame_time_for_each_fps[i] += time_delta_for_each_fps[i]++ cap.release()output_frames = [np.stack(frames) for frames in output_frames]return output_frames, all_frames, fpsdiff --git a/shared/RGB_factors.py b/shared/RGB_factors.pyindex 8a870b4..5cf9c34 100644--- a/shared/RGB_factors.py+++ b/shared/RGB_factors.py@@ -1,5 +1,5 @@# thanks Comfyui for the rgb factors (https://github.com/comfyanonymous/ComfyUI/blob/master/comfy/latent_formats.py)-def get_rgb_factors(model_family, model_type = None):+def get_rgb_factors(model_family, model_type = None, sub_family = None):if model_family in ["wan", "qwen"]:if model_type =="ti2v_2_2":latent_channels = 48@@ -239,29 +239,67 @@ def get_rgb_factors(model_family, model_type = None):latent_rgb_factors_bias = [-0.0571, -0.1657, -0.2512]elif model_family == "hunyuan":- latent_channels = 16- latent_dimensions = 3- scale_factor = 0.476986- latent_rgb_factors = [- [-0.0395, -0.0331, 0.0445],- [ 0.0696, 0.0795, 0.0518],- [ 0.0135, -0.0945, -0.0282],- [ 0.0108, -0.0250, -0.0765],- [-0.0209, 0.0032, 0.0224],- [-0.0804, -0.0254, -0.0639],- [-0.0991, 0.0271, -0.0669],- [-0.0646, -0.0422, -0.0400],- [-0.0696, -0.0595, -0.0894],- [-0.0799, -0.0208, -0.0375],- [ 0.1166, 0.1627, 0.0962],- [ 0.1165, 0.0432, 0.0407],- [-0.2315, -0.1920, -0.1355],- [-0.0270, 0.0401, -0.0821],- [-0.0616, -0.0997, -0.0727],- [ 0.0249, -0.0469, -0.1703]- ]+ if sub_family == "hunyuan1.5":+ # computed by your very own Deep Beep Meep+ latent_rgb_factors = [+ [0.031652, -0.021645, -0.006670],+ [-0.015073, -0.012539, -0.013397],+ [0.004683, 0.014850, 0.000513],+ [-0.012478, -0.008691, 0.017885],+ [0.046208, 0.069739, 0.062120],+ [0.006511, 0.006715, -0.049985],+ [0.025992, 0.025690, 0.015917],+ [0.013518, 0.002994, 0.019133],+ [0.035577, -0.028987, -0.005096],+ [-0.006799, -0.000338, 0.004619],+ [-0.003666, -0.004885, -0.008002],+ [-0.032795, -0.022699, -0.001088],+ [0.039256, 0.051807, 0.054762],+ [0.011453, 0.010160, -0.039282],+ [0.020320, 0.015219, 0.013882],+ [-0.014381, -0.029567, -0.014211],+ [0.036559, -0.021804, -0.009747],+ [-0.010908, -0.018426, -0.017130],+ [0.011040, 0.028708, 0.020362],+ [-0.007885, -0.009209, 0.015580],+ [0.042282, 0.056767, 0.048389],+ [0.018727, 0.026872, -0.022482],+ [0.016667, 0.023371, 0.022561],+ [-0.016530, -0.026502, -0.005821],+ [0.056069, -0.012952, 0.000057],+ [-0.001732, 0.000677, -0.002098],+ [0.015616, 0.027786, 0.024835],+ [-0.002916, 0.003586, 0.016754],+ [0.046222, 0.049560, 0.044939],+ [0.004627, 0.002370, -0.040525],+ [-0.001537, 0.004215, 0.007437],+ [0.001337, 0.001811, 0.022117],+ ]+ latent_rgb_factors_bias = [0.106222, 0.041891, -0.005932]+ else:+ latent_channels = 16+ latent_dimensions = 3+ scale_factor = 0.476986+ latent_rgb_factors = [+ [-0.0395, -0.0331, 0.0445],+ [ 0.0696, 0.0795, 0.0518],+ [ 0.0135, -0.0945, -0.0282],+ [ 0.0108, -0.0250, -0.0765],+ [-0.0209, 0.0032, 0.0224],+ [-0.0804, -0.0254, -0.0639],+ [-0.0991, 0.0271, -0.0669],+ [-0.0646, -0.0422, -0.0400],+ [-0.0696, -0.0595, -0.0894],+ [-0.0799, -0.0208, -0.0375],+ [ 0.1166, 0.1627, 0.0962],+ [ 0.1165, 0.0432, 0.0407],+ [-0.2315, -0.1920, -0.1355],+ [-0.0270, 0.0401, -0.0821],+ [-0.0616, -0.0997, -0.0727],+ [ 0.0249, -0.0469, -0.1703]+ ]- latent_rgb_factors_bias = [ 0.0259, -0.0192, -0.0761]+ latent_rgb_factors_bias = [ 0.0259, -0.0192, -0.0761]else:latent_rgb_factors_bias = latent_rgb_factors = Nonereturn latent_rgb_factors, latent_rgb_factors_bias\ No newline at end of filediff --git a/shared/tools/sha256_verify.py b/shared/tools/sha256_verify.pynew file mode 100644index 0000000..c02681e--- /dev/null+++ b/shared/tools/sha256_verify.py@@ -0,0 +1,76 @@+import hashlib+from pathlib import Path+++def compute_sha256(file_path, expected_hash=None, chunk_size=8192):+ """+ Compute the SHA256 hash of a file and optionally verify it.++ Args:+ file_path (str or Path): Path to the file to hash+ expected_hash (str, optional): Expected SHA256 hash to verify against+ chunk_size (int): Size of chunks to read (default 8KB for memory efficiency)++ Returns:+ str: The computed SHA256 hash (lowercase hexadecimal)++ Raises:+ FileNotFoundError: If the file doesn't exist+ ValueError: If expected_hash is provided and doesn't match++ Example:+ >>> # Just compute the hash+ >>> hash_value = compute_sha256("model.bin")+ >>> print(f"SHA256: {hash_value}")++ >>> # Compute and verify+ >>> try:+ >>> hash_value = compute_sha256("model.bin",+ >>> expected_hash="abc123...")+ >>> print("File verified successfully!")+ >>> except ValueError as e:+ >>> print(f"Verification failed: {e}")+ """+ file_path = Path(file_path)++ if not file_path.exists():+ raise FileNotFoundError(f"File not found: {file_path}")++ # Create SHA256 hash object+ sha256_hash = hashlib.sha256()++ # Read file in chunks to handle large files efficiently+ with open(file_path, "rb") as f:+ while chunk := f.read(chunk_size):+ sha256_hash.update(chunk)++ # Get the hexadecimal representation+ computed_hash = sha256_hash.hexdigest()++ # Verify if expected hash is provided+ if expected_hash is not None:+ expected_hash = expected_hash.lower().strip()+ if computed_hash != expected_hash:+ raise ValueError(+ f"Hash mismatch!\n"+ f"Expected: {expected_hash}\n"+ f"Computed: {computed_hash}"+ )+ print(f"✓ Hash verified successfully: {computed_hash}")++ return computed_hash+++if __name__ == "__main__":+ # Example usage+ import sys++ file_path = "c:/temp/hunyuan_15_upsampler.safetensors"+ expected_hash = "691dc1b81b49d942e2eb95e6d61b91321e17b868536eaa4e843db6e406390411"++ try:+ hash_value = compute_sha256(file_path, expected_hash)+ print(f"SHA256: {hash_value}")+ except (FileNotFoundError, ValueError) as e:+ print(f"Error: {e}")+ sys.exit(1)diff --git a/wgp.py b/wgp.pyindex 522433a..dd214ba 100644--- a/wgp.py+++ b/wgp.py@@ -82,8 +82,8 @@ global_queue_ref = []AUTOSAVE_FILENAME = "queue.zip"PROMPT_VARS_MAX = 10target_mmgp_version = "3.6.8"-WanGP_version = "9.52"-settings_version = 2.40+WanGP_version = "9.6"+settings_version = 2.41max_source_video_frames = 3000prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer = None, None, None, Noneimage_names_list = ["image_start", "image_end", "image_refs"]@@ -2090,7 +2090,12 @@ def get_lora_dir(model_type):if i2v:return args.lora_dir_hunyuan_i2velse:- return args.lora_dir_hunyuan+ root_lora_dir=args.lora_dir_hunyuan+ if "1_5" in base_model_type:+ lora_dir_1_5 = os.path.join(root_lora_dir, "1.5")+ if os.path.isdir(lora_dir_1_5 ):+ return lora_dir_1_5+ return root_lora_direlif model_family =="qwen":return args.lora_dir_qwenelif model_family =="tts":@@ -5594,10 +5599,11 @@ def generate_video(if video_guide is not None or len(frames_to_inject_parsed) > 0 or model_def.get("forced_guide_mask_inputs", False):any_mask = video_mask is not None or model_def.get("forced_guide_mask_inputs", False)any_guide_padding = model_def.get("pad_guide_video", False)+ dont_cat_preguide = extract_guide_from_window_start or model_def.get("dont_cat_preguide", False) or sparse_video_image is not Nonefrom shared.utils.utils import prepare_video_guide_and_masksrc_videos, src_masks = prepare_video_guide_and_mask( [video_guide_processed] + ([] if video_guide_processed2 is None else [video_guide_processed2]),[video_mask_processed] + ([] if video_guide_processed2 is None else [video_mask_processed2]),- None if extract_guide_from_window_start or model_def.get("dont_cat_preguide", False) or sparse_video_image is not None else pre_video_guide,+ None if dont_cat_preguide else pre_video_guide,image_size, current_video_length, latent_size,any_mask, any_guide_padding, guide_inpaint_color,keep_frames_parsed, frames_to_inject_parsed , outpainting_dims)@@ -5608,7 +5614,7 @@ def generate_video(src_video, src_video2 = src_videossrc_mask, src_mask2 = src_maskssrc_videos = src_masks = None- if src_video is None:+ if src_video is None or window_no >1 and src_video.shape[1] <= sliding_window_overlap and not dont_cat_preguide:abort = Truebreakif src_faces is not None:@@ -9052,7 +9058,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonsliding_window_defaults = model_def.get("sliding_window_defaults", {})sliding_window_size = gr.Slider(5, get_max_frames(257), value=ui_defaults.get("sliding_window_size", 129), step=4, label="Sliding Window Size", interactive=not model_def.get("sliding_window_size_locked"), show_reset_button= False)sliding_window_overlap = gr.Slider(sliding_window_defaults.get("overlap_min", 1), sliding_window_defaults.get("overlap_max", 97), value=ui_defaults.get("sliding_window_overlap",sliding_window_defaults.get("overlap_default", 5)), step=sliding_window_defaults.get("overlap_step", 4), label="Windows Frames Overlap (needed to maintain continuity between windows, a higher value will require more windows)", show_reset_button= False)- sliding_window_color_correction_strength = gr.Slider(0, 1, value=ui_defaults.get("sliding_window_color_correction_strength",0), step=0.01, label="Color Correction Strength (match colors of new window with previous one, 0 = disabled)", visible = True, show_reset_button= False)+ sliding_window_color_correction_strength = gr.Slider(0, 1, value=ui_defaults.get("sliding_window_color_correction_strength",0), step=0.01, label="Color Correction Strength (match colors of new window with previous one, 0 = disabled)", visible = model_def.get("color_correction", False), show_reset_button= False)sliding_window_overlap_noise = gr.Slider(0, 150, value=ui_defaults.get("sliding_window_overlap_noise",20 if vace else 0), step=1, label="Noise to be added to overlapped frames to reduce blur effect" , visible = vace, show_reset_button= False)sliding_window_discard_last_frames = gr.Slider(0, 20, value=ui_defaults.get("sliding_window_discard_last_frames", 0), step=4, label="Discard Last Frames of a Window (that may have bad quality)", visible = True, show_reset_button= False)@@ -9745,18 +9751,53 @@ def create_models_hierarchy(rows):if lt[i]!=prefix_low[i]: return Falsereturn True- def disp(name,mid,ot,lt):- if mid in outliers: return name- rem=ot[L:] if startswith_prefix(lt) else ot[:]- if header_has_last and lt and lt[-1]==p_last and rem and rem[-1].casefold()==p_last:- rem=rem[:-1]- s=' '.join(rem).strip()+ def base_rem(ot, lt):+ return ot[L:] if startswith_prefix(lt) else ot[:]++ def trim_rem(rem, lt):+ out = rem[:]+ if header_has_last and lt and lt[-1] == p_last and out and out[-1].casefold() == p_last:+ out = out[:-1]+ return out++ kid_infos = []+ for name, mid, ot, lt, _ in kids:+ rem_core = base_rem(ot, lt) if mid not in outliers else ot[:]+ kid_infos.append({+ "name": name,+ "mid": mid,+ "ot": ot,+ "lt": lt,+ "outlier": mid in outliers,+ "rem_core": rem_core,+ "rem_trim": trim_rem(rem_core, lt) if mid not in outliers else ot[:],+ "rem_set": {w.casefold() for w in rem_core} if mid not in outliers else set(),+ "rem_trim_set": {w.casefold() for w in (trim_rem(rem_core, lt) if mid not in outliers else ot[:])} if mid not in outliers else set(),+ })++ default_info = next(info for info in kid_infos if info["mid"] == pid)+ other_words = set()+ for info in kid_infos:+ if info["mid"] != pid:+ other_words |= info["rem_set"]+ default_shares = bool(default_info["rem_set"] & other_words)++ def disp(info):+ if info["outlier"]:+ return info["name"]+ if info["mid"] == pid:+ if not default_shares:+ return 'Default'+ rem = info["rem_trim"]+ else:+ rem = info["rem_trim"]+ s = ' '.join(rem).strip()return s if s else 'Default'- entries=[(disp(p_name,pid,p_tok,p_low),pid)]- for name,mid,ot,lt,_ in kids:- if mid==pid: continue- entries.append((disp(name,mid,ot,lt),mid))+ entries=[(disp(default_info),pid)]+ for info in kid_infos:+ if info["mid"]==pid: continue+ entries.append((disp(info), info["mid"]))# Number "Default" for children whose full name == parent's full namep_full=norm(p_name); full_by_mid={mid:name for name,mid,*_ in kids}