Python
wan model refactoring
DeepBeepMeep85ecb51ccd6d
diff --git a/models/wan/any2video.py b/models/wan/any2video.pyindex 8d94f2d..44ec2dd 100644--- a/models/wan/any2video.py+++ b/models/wan/any2video.py@@ -59,23 +59,6 @@ def timestep_transform(t, shift=5.0, num_timesteps=1000 ):new_t = new_t * num_timestepsreturn new_t-def preprocess_sd_with_dtype(dtype, sd):- new_sd = {}- prefix_list = ["model.diffusion_model"]- end_list = [".norm3.bias", ".norm3.weight", ".norm_q.bias", ".norm_q.weight", ".norm_k.bias", ".norm_k.weight" ]- for k,v in sd.items():- for prefix in prefix_list:- if k.startswith(prefix):- k = k[len(prefix)+1:]- break- if v.dtype in (torch.float8_e5m2, torch.float8_e4m3fn):- for endfix in end_list:- if k.endswith(endfix):- v = v.to(dtype)- break- if not k.startswith("vae."):- new_sd[k] = v- return new_sdclass WanAny2V:@@ -159,7 +142,7 @@ class WanAny2V:module_source = model_def.get("module_source", None)module_source2 = model_def.get("module_source2", None)def preprocess_sd(sd):- return preprocess_sd_with_dtype(dtype, sd)+ return WanModel.preprocess_sd_with_dtype(dtype, sd)kwargs= { "modelClass": WanModel,"do_quantize": quantizeTransformer and not save_quantized, "defaultConfigPath": base_config_file , "ignore_unused_weights": ignore_unused_weights, "writable_tensors": False, "default_dtype": dtype, "preprocess_sd": preprocess_sd, "forcedConfigPath": forcedConfigPath, }kwargs_light= { "modelClass": WanModel,"writable_tensors": False, "preprocess_sd": preprocess_sd , "forcedConfigPath" : base_config_file}if module_source is not None:@@ -220,13 +203,8 @@ class WanAny2V:save_quantized_model(self.model2, model_type, model_filename[1], dtype, base_config_file, submodel_no=2)self.sample_neg_prompt = config.sample_neg_prompt- if hasattr(self.model, "vace_blocks"):- self.adapt_vace_model(self.model)- if self.model2 is not None: self.adapt_vace_model(self.model2)-- if hasattr(self.model, "face_adapter"):- self.adapt_animate_model(self.model)- if self.model2 is not None: self.adapt_animate_model(self.model2)+ self.model.apply_post_init_changes()+ if self.model2 is not None: self.model2.apply_post_init_changes()self.num_timesteps = 1000self.use_timestep_transform = True@@ -1157,24 +1135,6 @@ class WanAny2V:return { "x" : videos, "latent_slice" : latent_slice, "BGRA_frames" : BGRA_frames }return videos- def adapt_vace_model(self, model):- modules_dict= { k: m for k, m in model.named_modules()}- for model_layer, vace_layer in model.vace_layers_mapping.items():- module = modules_dict[f"vace_blocks.{vace_layer}"]- target = modules_dict[f"blocks.{model_layer}"]- setattr(target, "vace", module )- delattr(model, "vace_blocks")--- def adapt_animate_model(self, model):- modules_dict= { k: m for k, m in model.named_modules()}- for animate_layer in range(8):- module = modules_dict[f"face_adapter.fuser_blocks.{animate_layer}"]- model_layer = animate_layer * 5- target = modules_dict[f"blocks.{model_layer}"]- setattr(target, "face_adapter_fuser_blocks", module )- delattr(model, "face_adapter")-def get_loras_transformer(self, get_model_recursive_prop, base_model_type, model_type, video_prompt_type, model_mode, **kwargs):if base_model_type == "animate":if "#" in video_prompt_type and "1" in video_prompt_type:diff --git a/models/wan/modules/model.py b/models/wan/modules/model.pyindex 0aca3d0..d711df3 100644--- a/models/wan/modules/model.py+++ b/models/wan/modules/model.py@@ -618,7 +618,7 @@ class WanAttentionBlock(nn.Module):hints_processed.append(self.vace(hint, x, **kwargs) if self.block_id == 0 else self.vace(hint, None, **kwargs))latent_frames = e.shape[0]- e = (self.modulation + e).chunk(6, dim=1)+ e = (self.modulation.weight + e).chunk(6, dim=1)# self-attentionx_mod = self.norm1(x)x_mod = reshape_latent(x_mod , latent_frames)@@ -840,7 +840,7 @@ class Head(nn.Module):dtype = x.dtypelatent_frames = e.shape[0]- e = (self.modulation + e.unsqueeze(1)).chunk(2, dim=1)+ e = (self.modulation.weight + e.unsqueeze(1)).chunk(2, dim=1)x = self.norm(x).to(dtype)x = reshape_latent(x , latent_frames)x *= (1 + e[1])@@ -896,6 +896,24 @@ class WanModel(ModelMixin, ConfigMixin):def release_chipmunk(self):offload.shared_state["_chipmunk_layers"] = None+ @staticmethod+ def preprocess_sd_with_dtype(dtype, sd):+ new_sd = {}+ prefix_list = ["model.diffusion_model"]+ end_list = [".norm3.bias", ".norm3.weight", ".norm_q.bias", ".norm_q.weight", ".norm_k.bias", ".norm_k.weight" ]+ for k,v in sd.items():+ for prefix in prefix_list:+ if k.startswith(prefix):+ k = k[len(prefix)+1:]+ break+ if v.dtype in (torch.float8_e5m2, torch.float8_e4m3fn):+ for endfix in end_list:+ if k.endswith(endfix):+ v = v.to(dtype)+ break+ if not k.startswith("vae."):+ new_sd[k] = v+ return new_sddef preprocess_loras(self, model_type, sd):first = next(iter(sd), None)@@ -1164,6 +1182,46 @@ class WanModel(ModelMixin, ConfigMixin):)+ def adapt_modulation(self, block_name ='blocks'):+ modules_dict= { k: m for k, m in self.named_modules()}+ for k,v in modules_dict[block_name]._modules.items():+ module = torch.nn.Module()+ module.weight = v.modulation+ delattr(v, "modulation")+ v.modulation = module++ if block_name != "blocks": return+ parent_module = modules_dict["head"]+ module = torch.nn.Module()+ module.weight = parent_module.modulation+ delattr(parent_module, "modulation")+ parent_module.modulation = module++ def adapt_vace_model(self):+ self.adapt_modulation("vace_blocks")++ modules_dict= { k: m for k, m in self.named_modules()}+ for model_layer, vace_layer in self.vace_layers_mapping.items():+ module = modules_dict[f"vace_blocks.{vace_layer}"]+ target = modules_dict[f"blocks.{model_layer}"]+ setattr(target, "vace", module )+ delattr(self, "vace_blocks")+++ def adapt_animate_model(self):+ modules_dict= { k: m for k, m in self.named_modules()}+ for animate_layer in range(8):+ module = modules_dict[f"face_adapter.fuser_blocks.{animate_layer}"]+ model_layer = animate_layer * 5+ target = modules_dict[f"blocks.{model_layer}"]+ setattr(target, "face_adapter_fuser_blocks", module )+ delattr(self, "face_adapter")++ def apply_post_init_changes(self):+ self.adapt_modulation()+ if hasattr(self, "vace_blocks"): self.adapt_vace_model()+ if hasattr(self, "face_adapter"): self.adapt_animate_model()+def lock_layers_dtypes(self, hybrid_dtype = None, dtype = torch.float32):from optimum.quanto import QTensor