Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Support 512 siglip model.

comfyanonymous3bfe4e527665
diff --git a/comfy/clip_vision.py b/comfy/clip_vision.py
index 87d32a66e..11bc57789 100644
--- a/comfy/clip_vision.py
+++ b/comfy/clip_vision.py
@@ -110,9 +110,13 @@ def load_clipvision_from_sd(sd, prefix="", convert_keys=False):
elif "vision_model.encoder.layers.30.layer_norm1.weight" in sd:
json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_h.json")
elif "vision_model.encoder.layers.22.layer_norm1.weight" in sd:
+ embed_shape = sd["vision_model.embeddings.position_embedding.weight"].shape[0]
if sd["vision_model.encoder.layers.0.layer_norm1.weight"].shape[0] == 1152:
- json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_siglip_384.json")
- elif sd["vision_model.embeddings.position_embedding.weight"].shape[0] == 577:
+ if embed_shape == 729:
+ json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_siglip_384.json")
+ elif embed_shape == 1024:
+ json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_siglip_512.json")
+ elif embed_shape == 577:
if "multi_modal_projector.linear_1.bias" in sd:
json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_vitl_336_llava.json")
else:
diff --git a/comfy/clip_vision_siglip_512.json b/comfy/clip_vision_siglip_512.json
new file mode 100644
index 000000000..7fb93ce15
--- /dev/null
+++ b/comfy/clip_vision_siglip_512.json
@@ -0,0 +1,13 @@
+{
+ "num_channels": 3,
+ "hidden_act": "gelu_pytorch_tanh",
+ "hidden_size": 1152,
+ "image_size": 512,
+ "intermediate_size": 4304,
+ "model_type": "siglip_vision_model",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 27,
+ "patch_size": 16,
+ "image_mean": [0.5, 0.5, 0.5],
+ "image_std": [0.5, 0.5, 0.5]
+}