The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add model_options for text encoder.
comfyanonymousfca42836f26d
diff --git a/comfy/sd.py b/comfy/sd.pyindex edd0b51d8..cae7812e3 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -62,7 +62,7 @@ def load_lora_for_models(model, clip, lora, strength_model, strength_clip):class CLIP:- def __init__(self, target=None, embedding_directory=None, no_init=False, tokenizer_data={}, parameters=0):+ def __init__(self, target=None, embedding_directory=None, no_init=False, tokenizer_data={}, parameters=0, model_options={}):if no_init:returnparams = target.params.copy()@@ -71,9 +71,14 @@ class CLIP:load_device = model_management.text_encoder_device()offload_device = model_management.text_encoder_offload_device()- dtype = model_management.text_encoder_dtype(load_device)+ dtype = model_options.get("dtype", None)+ if dtype is None:+ dtype = model_management.text_encoder_dtype(load_device)+params['dtype'] = dtypeparams['device'] = model_management.text_encoder_initial_device(load_device, offload_device, parameters * model_management.dtype_size(dtype))+ params['model_options'] = model_options+self.cond_stage_model = clip(**(params))for dt in self.cond_stage_model.dtypes:@@ -394,7 +399,7 @@ class CLIPType(Enum):HUNYUAN_DIT = 5FLUX = 6-def load_clip(ckpt_paths, embedding_directory=None, clip_type=CLIPType.STABLE_DIFFUSION):+def load_clip(ckpt_paths, embedding_directory=None, clip_type=CLIPType.STABLE_DIFFUSION, model_options={}):clip_data = []for p in ckpt_paths:clip_data.append(comfy.utils.load_torch_file(p, safe_load=True))@@ -464,7 +469,7 @@ def load_clip(ckpt_paths, embedding_directory=None, clip_type=CLIPType.STABLE_DIfor c in clip_data:parameters += comfy.utils.calculate_parameters(c)- clip = CLIP(clip_target, embedding_directory=embedding_directory, parameters=parameters)+ clip = CLIP(clip_target, embedding_directory=embedding_directory, parameters=parameters, model_options=model_options)for c in clip_data:m, u = clip.load_sd(c)if len(m) > 0:@@ -506,14 +511,14 @@ def load_checkpoint(config_path=None, ckpt_path=None, output_vae=True, output_clreturn (model, clip, vae)-def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True, model_options={}):+def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True, model_options={}, te_model_options={}):sd = comfy.utils.load_torch_file(ckpt_path)- out = load_state_dict_guess_config(sd, output_vae, output_clip, output_clipvision, embedding_directory, output_model, model_options)+ out = load_state_dict_guess_config(sd, output_vae, output_clip, output_clipvision, embedding_directory, output_model, model_options, te_model_options=te_model_options)if out is None:raise RuntimeError("ERROR: Could not detect model type of: {}".format(ckpt_path))return out-def load_state_dict_guess_config(sd, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True, model_options={}):+def load_state_dict_guess_config(sd, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True, model_options={}, te_model_options={}):clip = Noneclipvision = Nonevae = None@@ -563,7 +568,7 @@ def load_state_dict_guess_config(sd, output_vae=True, output_clip=True, output_cclip_sd = model_config.process_clip_state_dict(sd)if len(clip_sd) > 0:parameters = comfy.utils.calculate_parameters(clip_sd)- clip = CLIP(clip_target, embedding_directory=embedding_directory, tokenizer_data=clip_sd, parameters=parameters)+ clip = CLIP(clip_target, embedding_directory=embedding_directory, tokenizer_data=clip_sd, parameters=parameters, model_options=te_model_options)m, u = clip.load_sd(clip_sd, full_model=True)if len(m) > 0:m_filter = list(filter(lambda a: ".logit_scale" not in a and ".transformer.text_projection.weight" not in a, m))diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.pyindex e65cab285..dc8413b7b 100644--- a/comfy/sd1_clip.py+++ b/comfy/sd1_clip.py@@ -84,7 +84,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):def __init__(self, version="openai/clip-vit-large-patch14", device="cpu", max_length=77,freeze=True, layer="last", layer_idx=None, textmodel_json_config=None, dtype=None, model_class=comfy.clip_model.CLIPTextModel,special_tokens={"start": 49406, "end": 49407, "pad": 49407}, layer_norm_hidden_state=True, enable_attention_masks=False, zero_out_masked=False,- return_projected_pooled=True, return_attention_masks=False): # clip-vit-base-patch32+ return_projected_pooled=True, return_attention_masks=False, model_options={}): # clip-vit-base-patch32super().__init__()assert layer in self.LAYERS@@ -94,7 +94,11 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):with open(textmodel_json_config) as f:config = json.load(f)- self.operations = comfy.ops.manual_cast+ operations = model_options.get("custom_operations", None)+ if operations is None:+ operations = comfy.ops.manual_cast++ self.operations = operationsself.transformer = model_class(config, dtype, device, self.operations)self.num_layers = self.transformer.num_layers@@ -553,7 +557,7 @@ class SD1Tokenizer:return {}class SD1ClipModel(torch.nn.Module):- def __init__(self, device="cpu", dtype=None, clip_name="l", clip_model=SDClipModel, name=None, **kwargs):+ def __init__(self, device="cpu", dtype=None, model_options={}, clip_name="l", clip_model=SDClipModel, name=None, **kwargs):super().__init__()if name is not None:@@ -563,7 +567,7 @@ class SD1ClipModel(torch.nn.Module):self.clip_name = clip_nameself.clip = "clip_{}".format(self.clip_name)- setattr(self, self.clip, clip_model(device=device, dtype=dtype, **kwargs))+ setattr(self, self.clip, clip_model(device=device, dtype=dtype, model_options=model_options, **kwargs))self.dtypes = set()if dtype is not None:diff --git a/comfy/sdxl_clip.py b/comfy/sdxl_clip.pyindex 6e6b87d62..860900ccd 100644--- a/comfy/sdxl_clip.py+++ b/comfy/sdxl_clip.py@@ -3,14 +3,14 @@ import torchimport osclass SDXLClipG(sd1_clip.SDClipModel):- def __init__(self, device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, dtype=None):+ def __init__(self, device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, dtype=None, model_options={}):if layer == "penultimate":layer="hidden"layer_idx=-2textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_config_bigg.json")super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype,- special_tokens={"start": 49406, "end": 49407, "pad": 0}, layer_norm_hidden_state=False)+ special_tokens={"start": 49406, "end": 49407, "pad": 0}, layer_norm_hidden_state=False, model_options=model_options)def load_sd(self, sd):return super().load_sd(sd)@@ -38,10 +38,10 @@ class SDXLTokenizer:return {}class SDXLClipModel(torch.nn.Module):- def __init__(self, device="cpu", dtype=None):+ def __init__(self, device="cpu", dtype=None, model_options={}):super().__init__()- self.clip_l = sd1_clip.SDClipModel(layer="hidden", layer_idx=-2, device=device, dtype=dtype, layer_norm_hidden_state=False)- self.clip_g = SDXLClipG(device=device, dtype=dtype)+ self.clip_l = sd1_clip.SDClipModel(layer="hidden", layer_idx=-2, device=device, dtype=dtype, layer_norm_hidden_state=False, model_options=model_options)+ self.clip_g = SDXLClipG(device=device, dtype=dtype, model_options=model_options)self.dtypes = set([dtype])def set_clip_options(self, options):@@ -66,8 +66,8 @@ class SDXLClipModel(torch.nn.Module):return self.clip_l.load_sd(sd)class SDXLRefinerClipModel(sd1_clip.SD1ClipModel):- def __init__(self, device="cpu", dtype=None):- super().__init__(device=device, dtype=dtype, clip_name="g", clip_model=SDXLClipG)+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(device=device, dtype=dtype, clip_name="g", clip_model=SDXLClipG, model_options=model_options)class StableCascadeClipGTokenizer(sd1_clip.SDTokenizer):@@ -79,14 +79,14 @@ class StableCascadeTokenizer(sd1_clip.SD1Tokenizer):super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="g", tokenizer=StableCascadeClipGTokenizer)class StableCascadeClipG(sd1_clip.SDClipModel):- def __init__(self, device="cpu", max_length=77, freeze=True, layer="hidden", layer_idx=-1, dtype=None):+ def __init__(self, device="cpu", max_length=77, freeze=True, layer="hidden", layer_idx=-1, dtype=None, model_options={}):textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_config_bigg.json")super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype,- special_tokens={"start": 49406, "end": 49407, "pad": 49407}, layer_norm_hidden_state=False, enable_attention_masks=True)+ special_tokens={"start": 49406, "end": 49407, "pad": 49407}, layer_norm_hidden_state=False, enable_attention_masks=True, model_options=model_options)def load_sd(self, sd):return super().load_sd(sd)class StableCascadeClipModel(sd1_clip.SD1ClipModel):- def __init__(self, device="cpu", dtype=None):- super().__init__(device=device, dtype=dtype, clip_name="g", clip_model=StableCascadeClipG)+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(device=device, dtype=dtype, clip_name="g", clip_model=StableCascadeClipG, model_options=model_options)diff --git a/comfy/text_encoders/aura_t5.py b/comfy/text_encoders/aura_t5.pyindex 8500c7b70..e9ad45a7f 100644--- a/comfy/text_encoders/aura_t5.py+++ b/comfy/text_encoders/aura_t5.py@@ -4,9 +4,9 @@ import comfy.text_encoders.t5import osclass PT5XlModel(sd1_clip.SDClipModel):- def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None):+ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_options={}):textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_config_xl.json")- super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 2, "pad": 1}, model_class=comfy.text_encoders.t5.T5, enable_attention_masks=True, zero_out_masked=True)+ super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 2, "pad": 1}, model_class=comfy.text_encoders.t5.T5, enable_attention_masks=True, zero_out_masked=True, model_options=model_options)class PT5XlTokenizer(sd1_clip.SDTokenizer):def __init__(self, embedding_directory=None, tokenizer_data={}):@@ -18,5 +18,5 @@ class AuraT5Tokenizer(sd1_clip.SD1Tokenizer):super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="pile_t5xl", tokenizer=PT5XlTokenizer)class AuraT5Model(sd1_clip.SD1ClipModel):- def __init__(self, device="cpu", dtype=None, **kwargs):- super().__init__(device=device, dtype=dtype, name="pile_t5xl", clip_model=PT5XlModel, **kwargs)+ def __init__(self, device="cpu", dtype=None, model_options={}, **kwargs):+ super().__init__(device=device, dtype=dtype, model_options=model_options, name="pile_t5xl", clip_model=PT5XlModel, **kwargs)diff --git a/comfy/text_encoders/flux.py b/comfy/text_encoders/flux.pyindex ee26f560d..91d1f249d 100644--- a/comfy/text_encoders/flux.py+++ b/comfy/text_encoders/flux.py@@ -6,9 +6,9 @@ import torchimport osclass T5XXLModel(sd1_clip.SDClipModel):- def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None):+ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_options={}):textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_config_xxl.json")- super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 1, "pad": 0}, model_class=comfy.text_encoders.t5.T5)+ super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 1, "pad": 0}, model_class=comfy.text_encoders.t5.T5, model_options=model_options)class T5XXLTokenizer(sd1_clip.SDTokenizer):def __init__(self, embedding_directory=None, tokenizer_data={}):@@ -35,11 +35,11 @@ class FluxTokenizer:class FluxClipModel(torch.nn.Module):- def __init__(self, dtype_t5=None, device="cpu", dtype=None):+ def __init__(self, dtype_t5=None, device="cpu", dtype=None, model_options={}):super().__init__()dtype_t5 = comfy.model_management.pick_weight_dtype(dtype_t5, dtype, device)- self.clip_l = sd1_clip.SDClipModel(device=device, dtype=dtype, return_projected_pooled=False)- self.t5xxl = T5XXLModel(device=device, dtype=dtype_t5)+ self.clip_l = sd1_clip.SDClipModel(device=device, dtype=dtype, return_projected_pooled=False, model_options=model_options)+ self.t5xxl = T5XXLModel(device=device, dtype=dtype_t5, model_options=model_options)self.dtypes = set([dtype, dtype_t5])def set_clip_options(self, options):@@ -66,6 +66,6 @@ class FluxClipModel(torch.nn.Module):def flux_clip(dtype_t5=None):class FluxClipModel_(FluxClipModel):- def __init__(self, device="cpu", dtype=None):- super().__init__(dtype_t5=dtype_t5, device=device, dtype=dtype)+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(dtype_t5=dtype_t5, device=device, dtype=dtype, model_options=model_options)return FluxClipModel_diff --git a/comfy/text_encoders/hydit.py b/comfy/text_encoders/hydit.pyindex 9dfa288b1..7cb790f45 100644--- a/comfy/text_encoders/hydit.py+++ b/comfy/text_encoders/hydit.py@@ -7,9 +7,9 @@ import osimport torchclass HyditBertModel(sd1_clip.SDClipModel):- def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None):+ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_options={}):textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hydit_clip.json")- super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"start": 101, "end": 102, "pad": 0}, model_class=BertModel, enable_attention_masks=True, return_attention_masks=True)+ super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"start": 101, "end": 102, "pad": 0}, model_class=BertModel, enable_attention_masks=True, return_attention_masks=True, model_options=model_options)class HyditBertTokenizer(sd1_clip.SDTokenizer):def __init__(self, embedding_directory=None, tokenizer_data={}):@@ -18,9 +18,9 @@ class HyditBertTokenizer(sd1_clip.SDTokenizer):class MT5XLModel(sd1_clip.SDClipModel):- def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None):+ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_options={}):textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "mt5_config_xl.json")- super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 1, "pad": 0}, model_class=comfy.text_encoders.t5.T5, enable_attention_masks=True, return_attention_masks=True)+ super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 1, "pad": 0}, model_class=comfy.text_encoders.t5.T5, enable_attention_masks=True, return_attention_masks=True, model_options=model_options)class MT5XLTokenizer(sd1_clip.SDTokenizer):def __init__(self, embedding_directory=None, tokenizer_data={}):@@ -50,10 +50,10 @@ class HyditTokenizer:return {"mt5xl.spiece_model": self.mt5xl.state_dict()["spiece_model"]}class HyditModel(torch.nn.Module):- def __init__(self, device="cpu", dtype=None):+ def __init__(self, device="cpu", dtype=None, model_options={}):super().__init__()- self.hydit_clip = HyditBertModel(dtype=dtype)- self.mt5xl = MT5XLModel(dtype=dtype)+ self.hydit_clip = HyditBertModel(dtype=dtype, model_options=model_options)+ self.mt5xl = MT5XLModel(dtype=dtype, model_options=model_options)self.dtypes = set()if dtype is not None:diff --git a/comfy/text_encoders/sa_t5.py b/comfy/text_encoders/sa_t5.pyindex 189f8c181..7778ce47a 100644--- a/comfy/text_encoders/sa_t5.py+++ b/comfy/text_encoders/sa_t5.py@@ -4,9 +4,9 @@ import comfy.text_encoders.t5import osclass T5BaseModel(sd1_clip.SDClipModel):- def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None):+ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_options={}):textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_config_base.json")- super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 1, "pad": 0}, model_class=comfy.text_encoders.t5.T5, enable_attention_masks=True, zero_out_masked=True)+ super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, model_options=model_options, special_tokens={"end": 1, "pad": 0}, model_class=comfy.text_encoders.t5.T5, enable_attention_masks=True, zero_out_masked=True)class T5BaseTokenizer(sd1_clip.SDTokenizer):def __init__(self, embedding_directory=None, tokenizer_data={}):@@ -18,5 +18,5 @@ class SAT5Tokenizer(sd1_clip.SD1Tokenizer):super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="t5base", tokenizer=T5BaseTokenizer)class SAT5Model(sd1_clip.SD1ClipModel):- def __init__(self, device="cpu", dtype=None, **kwargs):- super().__init__(device=device, dtype=dtype, name="t5base", clip_model=T5BaseModel, **kwargs)+ def __init__(self, device="cpu", dtype=None, model_options={}, **kwargs):+ super().__init__(device=device, dtype=dtype, model_options=model_options, name="t5base", clip_model=T5BaseModel, **kwargs)diff --git a/comfy/text_encoders/sd2_clip.py b/comfy/text_encoders/sd2_clip.pyindex 8ea8e1cd0..0c98cd853 100644--- a/comfy/text_encoders/sd2_clip.py+++ b/comfy/text_encoders/sd2_clip.py@@ -2,13 +2,13 @@ from comfy import sd1_clipimport osclass SD2ClipHModel(sd1_clip.SDClipModel):- def __init__(self, arch="ViT-H-14", device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, dtype=None):+ def __init__(self, arch="ViT-H-14", device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, dtype=None, model_options={}):if layer == "penultimate":layer="hidden"layer_idx=-2textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd2_clip_config.json")- super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"start": 49406, "end": 49407, "pad": 0})+ super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"start": 49406, "end": 49407, "pad": 0}, model_options=model_options)class SD2ClipHTokenizer(sd1_clip.SDTokenizer):def __init__(self, tokenizer_path=None, embedding_directory=None, tokenizer_data={}):@@ -19,5 +19,5 @@ class SD2Tokenizer(sd1_clip.SD1Tokenizer):super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="h", tokenizer=SD2ClipHTokenizer)class SD2ClipModel(sd1_clip.SD1ClipModel):- def __init__(self, device="cpu", dtype=None, **kwargs):- super().__init__(device=device, dtype=dtype, clip_name="h", clip_model=SD2ClipHModel, **kwargs)+ def __init__(self, device="cpu", dtype=None, model_options={}, **kwargs):+ super().__init__(device=device, dtype=dtype, model_options=model_options, clip_name="h", clip_model=SD2ClipHModel, **kwargs)diff --git a/comfy/text_encoders/sd3_clip.py b/comfy/text_encoders/sd3_clip.pyindex 549c068e9..83e8fa1f3 100644--- a/comfy/text_encoders/sd3_clip.py+++ b/comfy/text_encoders/sd3_clip.py@@ -8,9 +8,9 @@ import comfy.model_managementimport loggingclass T5XXLModel(sd1_clip.SDClipModel):- def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None):+ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_options={}):textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_config_xxl.json")- super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 1, "pad": 0}, model_class=comfy.text_encoders.t5.T5)+ super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 1, "pad": 0}, model_class=comfy.text_encoders.t5.T5, model_options=model_options)class T5XXLTokenizer(sd1_clip.SDTokenizer):def __init__(self, embedding_directory=None, tokenizer_data={}):@@ -38,24 +38,24 @@ class SD3Tokenizer:return {}class SD3ClipModel(torch.nn.Module):- def __init__(self, clip_l=True, clip_g=True, t5=True, dtype_t5=None, device="cpu", dtype=None):+ def __init__(self, clip_l=True, clip_g=True, t5=True, dtype_t5=None, device="cpu", dtype=None, model_options={}):super().__init__()self.dtypes = set()if clip_l:- self.clip_l = sd1_clip.SDClipModel(layer="hidden", layer_idx=-2, device=device, dtype=dtype, layer_norm_hidden_state=False, return_projected_pooled=False)+ self.clip_l = sd1_clip.SDClipModel(layer="hidden", layer_idx=-2, device=device, dtype=dtype, layer_norm_hidden_state=False, return_projected_pooled=False, model_options=model_options)self.dtypes.add(dtype)else:self.clip_l = Noneif clip_g:- self.clip_g = sdxl_clip.SDXLClipG(device=device, dtype=dtype)+ self.clip_g = sdxl_clip.SDXLClipG(device=device, dtype=dtype, model_options=model_options)self.dtypes.add(dtype)else:self.clip_g = Noneif t5:dtype_t5 = comfy.model_management.pick_weight_dtype(dtype_t5, dtype, device)- self.t5xxl = T5XXLModel(device=device, dtype=dtype_t5)+ self.t5xxl = T5XXLModel(device=device, dtype=dtype_t5, model_options=model_options)self.dtypes.add(dtype_t5)else:self.t5xxl = None@@ -132,6 +132,6 @@ class SD3ClipModel(torch.nn.Module):def sd3_clip(clip_l=True, clip_g=True, t5=True, dtype_t5=None):class SD3ClipModel_(SD3ClipModel):- def __init__(self, device="cpu", dtype=None):- super().__init__(clip_l=clip_l, clip_g=clip_g, t5=t5, dtype_t5=dtype_t5, device=device, dtype=dtype)+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(clip_l=clip_l, clip_g=clip_g, t5=t5, dtype_t5=dtype_t5, device=device, dtype=dtype, model_options=model_options)return SD3ClipModel_