Skip to content
Python

wan model refactoring

DeepBeepMeep85ecb51ccd6d
diff --git a/models/wan/any2video.py b/models/wan/any2video.py
index 8d94f2d..44ec2dd 100644
--- a/models/wan/any2video.py
+++ b/models/wan/any2video.py
@@ -59,23 +59,6 @@ def timestep_transform(t, shift=5.0, num_timesteps=1000 ):
new_t = new_t * num_timesteps
return new_t
-def preprocess_sd_with_dtype(dtype, sd):
- new_sd = {}
- prefix_list = ["model.diffusion_model"]
- end_list = [".norm3.bias", ".norm3.weight", ".norm_q.bias", ".norm_q.weight", ".norm_k.bias", ".norm_k.weight" ]
- for k,v in sd.items():
- for prefix in prefix_list:
- if k.startswith(prefix):
- k = k[len(prefix)+1:]
- break
- if v.dtype in (torch.float8_e5m2, torch.float8_e4m3fn):
- for endfix in end_list:
- if k.endswith(endfix):
- v = v.to(dtype)
- break
- if not k.startswith("vae."):
- new_sd[k] = v
- return new_sd
class WanAny2V:
@@ -159,7 +142,7 @@ class WanAny2V:
module_source = model_def.get("module_source", None)
module_source2 = model_def.get("module_source2", None)
def preprocess_sd(sd):
- return preprocess_sd_with_dtype(dtype, sd)
+ return WanModel.preprocess_sd_with_dtype(dtype, sd)
kwargs= { "modelClass": WanModel,"do_quantize": quantizeTransformer and not save_quantized, "defaultConfigPath": base_config_file , "ignore_unused_weights": ignore_unused_weights, "writable_tensors": False, "default_dtype": dtype, "preprocess_sd": preprocess_sd, "forcedConfigPath": forcedConfigPath, }
kwargs_light= { "modelClass": WanModel,"writable_tensors": False, "preprocess_sd": preprocess_sd , "forcedConfigPath" : base_config_file}
if module_source is not None:
@@ -220,13 +203,8 @@ class WanAny2V:
save_quantized_model(self.model2, model_type, model_filename[1], dtype, base_config_file, submodel_no=2)
self.sample_neg_prompt = config.sample_neg_prompt
- if hasattr(self.model, "vace_blocks"):
- self.adapt_vace_model(self.model)
- if self.model2 is not None: self.adapt_vace_model(self.model2)
-
- if hasattr(self.model, "face_adapter"):
- self.adapt_animate_model(self.model)
- if self.model2 is not None: self.adapt_animate_model(self.model2)
+ self.model.apply_post_init_changes()
+ if self.model2 is not None: self.model2.apply_post_init_changes()
self.num_timesteps = 1000
self.use_timestep_transform = True
@@ -1157,24 +1135,6 @@ class WanAny2V:
return { "x" : videos, "latent_slice" : latent_slice, "BGRA_frames" : BGRA_frames }
return videos
- def adapt_vace_model(self, model):
- modules_dict= { k: m for k, m in model.named_modules()}
- for model_layer, vace_layer in model.vace_layers_mapping.items():
- module = modules_dict[f"vace_blocks.{vace_layer}"]
- target = modules_dict[f"blocks.{model_layer}"]
- setattr(target, "vace", module )
- delattr(model, "vace_blocks")
-
-
- def adapt_animate_model(self, model):
- modules_dict= { k: m for k, m in model.named_modules()}
- for animate_layer in range(8):
- module = modules_dict[f"face_adapter.fuser_blocks.{animate_layer}"]
- model_layer = animate_layer * 5
- target = modules_dict[f"blocks.{model_layer}"]
- setattr(target, "face_adapter_fuser_blocks", module )
- delattr(model, "face_adapter")
-
def get_loras_transformer(self, get_model_recursive_prop, base_model_type, model_type, video_prompt_type, model_mode, **kwargs):
if base_model_type == "animate":
if "#" in video_prompt_type and "1" in video_prompt_type:
diff --git a/models/wan/modules/model.py b/models/wan/modules/model.py
index 0aca3d0..d711df3 100644
--- a/models/wan/modules/model.py
+++ b/models/wan/modules/model.py
@@ -618,7 +618,7 @@ class WanAttentionBlock(nn.Module):
hints_processed.append(self.vace(hint, x, **kwargs) if self.block_id == 0 else self.vace(hint, None, **kwargs))
latent_frames = e.shape[0]
- e = (self.modulation + e).chunk(6, dim=1)
+ e = (self.modulation.weight + e).chunk(6, dim=1)
# self-attention
x_mod = self.norm1(x)
x_mod = reshape_latent(x_mod , latent_frames)
@@ -840,7 +840,7 @@ class Head(nn.Module):
dtype = x.dtype
latent_frames = e.shape[0]
- e = (self.modulation + e.unsqueeze(1)).chunk(2, dim=1)
+ e = (self.modulation.weight + e.unsqueeze(1)).chunk(2, dim=1)
x = self.norm(x).to(dtype)
x = reshape_latent(x , latent_frames)
x *= (1 + e[1])
@@ -896,6 +896,24 @@ class WanModel(ModelMixin, ConfigMixin):
def release_chipmunk(self):
offload.shared_state["_chipmunk_layers"] = None
+ @staticmethod
+ def preprocess_sd_with_dtype(dtype, sd):
+ new_sd = {}
+ prefix_list = ["model.diffusion_model"]
+ end_list = [".norm3.bias", ".norm3.weight", ".norm_q.bias", ".norm_q.weight", ".norm_k.bias", ".norm_k.weight" ]
+ for k,v in sd.items():
+ for prefix in prefix_list:
+ if k.startswith(prefix):
+ k = k[len(prefix)+1:]
+ break
+ if v.dtype in (torch.float8_e5m2, torch.float8_e4m3fn):
+ for endfix in end_list:
+ if k.endswith(endfix):
+ v = v.to(dtype)
+ break
+ if not k.startswith("vae."):
+ new_sd[k] = v
+ return new_sd
def preprocess_loras(self, model_type, sd):
first = next(iter(sd), None)
@@ -1164,6 +1182,46 @@ class WanModel(ModelMixin, ConfigMixin):
)
+ def adapt_modulation(self, block_name ='blocks'):
+ modules_dict= { k: m for k, m in self.named_modules()}
+ for k,v in modules_dict[block_name]._modules.items():
+ module = torch.nn.Module()
+ module.weight = v.modulation
+ delattr(v, "modulation")
+ v.modulation = module
+
+ if block_name != "blocks": return
+ parent_module = modules_dict["head"]
+ module = torch.nn.Module()
+ module.weight = parent_module.modulation
+ delattr(parent_module, "modulation")
+ parent_module.modulation = module
+
+ def adapt_vace_model(self):
+ self.adapt_modulation("vace_blocks")
+
+ modules_dict= { k: m for k, m in self.named_modules()}
+ for model_layer, vace_layer in self.vace_layers_mapping.items():
+ module = modules_dict[f"vace_blocks.{vace_layer}"]
+ target = modules_dict[f"blocks.{model_layer}"]
+ setattr(target, "vace", module )
+ delattr(self, "vace_blocks")
+
+
+ def adapt_animate_model(self):
+ modules_dict= { k: m for k, m in self.named_modules()}
+ for animate_layer in range(8):
+ module = modules_dict[f"face_adapter.fuser_blocks.{animate_layer}"]
+ model_layer = animate_layer * 5
+ target = modules_dict[f"blocks.{model_layer}"]
+ setattr(target, "face_adapter_fuser_blocks", module )
+ delattr(self, "face_adapter")
+
+ def apply_post_init_changes(self):
+ self.adapt_modulation()
+ if hasattr(self, "vace_blocks"): self.adapt_vace_model()
+ if hasattr(self, "face_adapter"): self.adapt_animate_model()
+
def lock_layers_dtypes(self, hybrid_dtype = None, dtype = torch.float32):
from optimum.quanto import QTensor