Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Properly calculate inner dim for t5 model.

comfyanonymousd0f3752e332a
diff --git a/comfy/text_encoders/t5.py b/comfy/text_encoders/t5.py
index 38d8d5234..7405528e2 100644
--- a/comfy/text_encoders/t5.py
+++ b/comfy/text_encoders/t5.py
@@ -227,8 +227,9 @@ class T5(torch.nn.Module):
super().__init__()
self.num_layers = config_dict["num_layers"]
model_dim = config_dict["d_model"]
+ inner_dim = config_dict["d_kv"] * config_dict["num_heads"]
- self.encoder = T5Stack(self.num_layers, model_dim, model_dim, config_dict["d_ff"], config_dict["dense_act_fn"], config_dict["is_gated_act"], config_dict["num_heads"], config_dict["model_type"] != "umt5", dtype, device, operations)
+ self.encoder = T5Stack(self.num_layers, model_dim, inner_dim, config_dict["d_ff"], config_dict["dense_act_fn"], config_dict["is_gated_act"], config_dict["num_heads"], config_dict["model_type"] != "umt5", dtype, device, operations)
self.dtype = dtype
self.shared = operations.Embedding(config_dict["vocab_size"], model_dim, device=device, dtype=dtype)