The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Properly disable weight initialization in clip models.
comfyanonymousbb1f45d6e862
diff --git a/comfy/clip_vision.py b/comfy/clip_vision.pyindex a95707e46..7a59ef6e2 100644--- a/comfy/clip_vision.py+++ b/comfy/clip_vision.py@@ -2,12 +2,14 @@ from transformers import CLIPVisionModelWithProjection, CLIPVisionConfig, CLIPImfrom .utils import load_torch_file, transformers_convertimport osimport torch+import comfy.opsclass ClipVisionModel():def __init__(self, json_config):config = CLIPVisionConfig.from_json_file(json_config)- with modeling_utils.no_init_weights():- self.model = CLIPVisionModelWithProjection(config)+ with comfy.ops.use_comfy_ops():+ with modeling_utils.no_init_weights():+ self.model = CLIPVisionModelWithProjection(config)self.processor = CLIPImageProcessor(crop_size=224,do_center_crop=True,do_convert_rgb=True,diff --git a/comfy/ops.py b/comfy/ops.pyindex c39b994ab..2e72030bd 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -1,4 +1,5 @@import torch+from contextlib import contextmanagerclass Linear(torch.nn.Module):def __init__(self, in_features: int, out_features: int, bias: bool = True,@@ -19,3 +20,13 @@ class Linear(torch.nn.Module):class Conv2d(torch.nn.Conv2d):def reset_parameters(self):return None+++@contextmanager+def use_comfy_ops(): # Kind of an ugly hack but I can't think of a better way+ old_torch_nn_linear = torch.nn.Linear+ torch.nn.Linear = Linear+ try:+ yield+ finally:+ torch.nn.Linear = old_torch_nn_lineardiff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.pyindex 0df3d9d91..c2d4df092 100644--- a/comfy/sd1_clip.py+++ b/comfy/sd1_clip.py@@ -1,6 +1,7 @@import osfrom transformers import CLIPTokenizer, CLIPTextModel, CLIPTextConfig, modeling_utils+import comfy.opsimport torchimport tracebackimport zipfile@@ -38,8 +39,9 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder):if textmodel_json_config is None:textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_clip_config.json")config = CLIPTextConfig.from_json_file(textmodel_json_config)- with modeling_utils.no_init_weights():- self.transformer = CLIPTextModel(config)+ with comfy.ops.use_comfy_ops():+ with modeling_utils.no_init_weights():+ self.transformer = CLIPTextModel(config)self.device = deviceself.max_length = max_length