The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Cleanup and fix issues with text encoder quants. (#10872)
comfyanonymous25022e0b0965
diff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex cf1b0d441..6551ced5a 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -231,7 +231,6 @@ class ModelPatcher:self.object_patches_backup = {}self.weight_wrapper_patches = {}self.model_options = {"transformer_options":{}}- self.model_size()self.load_device = load_deviceself.offload_device = offload_deviceself.weight_inplace_update = weight_inplace_update@@ -286,7 +285,7 @@ class ModelPatcher:return self.model.lowvram_patch_counterdef clone(self):- n = self.__class__(self.model, self.load_device, self.offload_device, self.size, weight_inplace_update=self.weight_inplace_update)+ n = self.__class__(self.model, self.load_device, self.offload_device, self.model_size(), weight_inplace_update=self.weight_inplace_update)n.patches = {}for k in self.patches:n.patches[k] = self.patches[k][:]diff --git a/comfy/ops.py b/comfy/ops.pyindex 640622fd1..af185ec24 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -540,113 +540,115 @@ if CUBLAS_IS_AVAILABLE:# ==============================================================================from .quant_ops import QuantizedTensor, QUANT_ALGOS-class MixedPrecisionOps(disable_weight_init):- _layer_quant_config = {}- _compute_dtype = torch.bfloat16-- class Linear(torch.nn.Module, CastWeightBiasOp):- def __init__(- self,- in_features: int,- out_features: int,- bias: bool = True,- device=None,- dtype=None,- ) -> None:- super().__init__()-- self.factory_kwargs = {"device": device, "dtype": MixedPrecisionOps._compute_dtype}- # self.factory_kwargs = {"device": device, "dtype": dtype}-- self.in_features = in_features- self.out_features = out_features- if bias:- self.bias = torch.nn.Parameter(torch.empty(out_features, **self.factory_kwargs))- else:- self.register_parameter("bias", None)- self.tensor_class = None+def mixed_precision_ops(layer_quant_config={}, compute_dtype=torch.bfloat16, full_precision_mm=False):+ class MixedPrecisionOps(manual_cast):+ _layer_quant_config = layer_quant_config+ _compute_dtype = compute_dtype+ _full_precision_mm = full_precision_mm++ class Linear(torch.nn.Module, CastWeightBiasOp):+ def __init__(+ self,+ in_features: int,+ out_features: int,+ bias: bool = True,+ device=None,+ dtype=None,+ ) -> None:+ super().__init__()++ self.factory_kwargs = {"device": device, "dtype": MixedPrecisionOps._compute_dtype}+ # self.factory_kwargs = {"device": device, "dtype": dtype}++ self.in_features = in_features+ self.out_features = out_features+ if bias:+ self.bias = torch.nn.Parameter(torch.empty(out_features, **self.factory_kwargs))+ else:+ self.register_parameter("bias", None)- def reset_parameters(self):- return None+ self.tensor_class = None+ self._full_precision_mm = MixedPrecisionOps._full_precision_mm- def _load_from_state_dict(self, state_dict, prefix, local_metadata,- strict, missing_keys, unexpected_keys, error_msgs):+ def reset_parameters(self):+ return None- device = self.factory_kwargs["device"]- layer_name = prefix.rstrip('.')- weight_key = f"{prefix}weight"- weight = state_dict.pop(weight_key, None)- if weight is None:- raise ValueError(f"Missing weight for layer {layer_name}")+ def _load_from_state_dict(self, state_dict, prefix, local_metadata,+ strict, missing_keys, unexpected_keys, error_msgs):- manually_loaded_keys = [weight_key]+ device = self.factory_kwargs["device"]+ layer_name = prefix.rstrip('.')+ weight_key = f"{prefix}weight"+ weight = state_dict.pop(weight_key, None)+ if weight is None:+ raise ValueError(f"Missing weight for layer {layer_name}")- if layer_name not in MixedPrecisionOps._layer_quant_config:- self.weight = torch.nn.Parameter(weight.to(device=device, dtype=MixedPrecisionOps._compute_dtype), requires_grad=False)- else:- quant_format = MixedPrecisionOps._layer_quant_config[layer_name].get("format", None)- if quant_format is None:- raise ValueError(f"Unknown quantization format for layer {layer_name}")-- qconfig = QUANT_ALGOS[quant_format]- self.layout_type = qconfig["comfy_tensor_layout"]-- weight_scale_key = f"{prefix}weight_scale"- layout_params = {- 'scale': state_dict.pop(weight_scale_key, None),- 'orig_dtype': MixedPrecisionOps._compute_dtype,- 'block_size': qconfig.get("group_size", None),- }- if layout_params['scale'] is not None:- manually_loaded_keys.append(weight_scale_key)-- self.weight = torch.nn.Parameter(- QuantizedTensor(weight.to(device=device), self.layout_type, layout_params),- requires_grad=False- )-- for param_name in qconfig["parameters"]:- param_key = f"{prefix}{param_name}"- _v = state_dict.pop(param_key, None)- if _v is None:- continue- setattr(self, param_name, torch.nn.Parameter(_v.to(device=device), requires_grad=False))- manually_loaded_keys.append(param_key)-- super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs)-- for key in manually_loaded_keys:- if key in missing_keys:- missing_keys.remove(key)-- def _forward(self, input, weight, bias):- return torch.nn.functional.linear(input, weight, bias)+ manually_loaded_keys = [weight_key]- def forward_comfy_cast_weights(self, input):- weight, bias, offload_stream = cast_bias_weight(self, input, offloadable=True)- x = self._forward(input, weight, bias)- uncast_bias_weight(self, weight, bias, offload_stream)- return x+ if layer_name not in MixedPrecisionOps._layer_quant_config:+ self.weight = torch.nn.Parameter(weight.to(device=device, dtype=MixedPrecisionOps._compute_dtype), requires_grad=False)+ else:+ quant_format = MixedPrecisionOps._layer_quant_config[layer_name].get("format", None)+ if quant_format is None:+ raise ValueError(f"Unknown quantization format for layer {layer_name}")++ qconfig = QUANT_ALGOS[quant_format]+ self.layout_type = qconfig["comfy_tensor_layout"]++ weight_scale_key = f"{prefix}weight_scale"+ layout_params = {+ 'scale': state_dict.pop(weight_scale_key, None),+ 'orig_dtype': MixedPrecisionOps._compute_dtype,+ 'block_size': qconfig.get("group_size", None),+ }+ if layout_params['scale'] is not None:+ manually_loaded_keys.append(weight_scale_key)++ self.weight = torch.nn.Parameter(+ QuantizedTensor(weight.to(device=device), self.layout_type, layout_params),+ requires_grad=False+ )++ for param_name in qconfig["parameters"]:+ param_key = f"{prefix}{param_name}"+ _v = state_dict.pop(param_key, None)+ if _v is None:+ continue+ setattr(self, param_name, torch.nn.Parameter(_v.to(device=device), requires_grad=False))+ manually_loaded_keys.append(param_key)++ super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs)++ for key in manually_loaded_keys:+ if key in missing_keys:+ missing_keys.remove(key)++ def _forward(self, input, weight, bias):+ return torch.nn.functional.linear(input, weight, bias)- def forward(self, input, *args, **kwargs):- run_every_op()+ def forward_comfy_cast_weights(self, input):+ weight, bias, offload_stream = cast_bias_weight(self, input, offloadable=True)+ x = self._forward(input, weight, bias)+ uncast_bias_weight(self, weight, bias, offload_stream)+ return x- if self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0:- return self.forward_comfy_cast_weights(input, *args, **kwargs)- if (getattr(self, 'layout_type', None) is not None and- getattr(self, 'input_scale', None) is not None and- not isinstance(input, QuantizedTensor)):- input = QuantizedTensor.from_float(input, self.layout_type, scale=self.input_scale, dtype=self.weight.dtype)- return self._forward(input, self.weight, self.bias)+ def forward(self, input, *args, **kwargs):+ run_every_op()+ if self._full_precision_mm or self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0:+ return self.forward_comfy_cast_weights(input, *args, **kwargs)+ if (getattr(self, 'layout_type', None) is not None and+ getattr(self, 'input_scale', None) is not None and+ not isinstance(input, QuantizedTensor)):+ input = QuantizedTensor.from_float(input, self.layout_type, scale=self.input_scale, dtype=self.weight.dtype)+ return self._forward(input, self.weight, self.bias)+ return MixedPrecisionOpsdef pick_operations(weight_dtype, compute_dtype, load_device=None, disable_fast_fp8=False, fp8_optimizations=False, scaled_fp8=None, model_config=None):if model_config and hasattr(model_config, 'layer_quant_config') and model_config.layer_quant_config:- MixedPrecisionOps._layer_quant_config = model_config.layer_quant_config- MixedPrecisionOps._compute_dtype = compute_dtypelogging.info(f"Using mixed precision operations: {len(model_config.layer_quant_config)} quantized layers")- return MixedPrecisionOps+ return mixed_precision_ops(model_config.layer_quant_config, compute_dtype)fp8_compute = comfy.model_management.supports_fp8_compute(load_device)if scaled_fp8 is not None:diff --git a/comfy/quant_ops.py b/comfy/quant_ops.pyindex 1d058bece..905b4729e 100644--- a/comfy/quant_ops.py+++ b/comfy/quant_ops.py@@ -338,6 +338,18 @@ def generic_copy_(func, args, kwargs):return func(*args, **kwargs)+@register_generic_util(torch.ops.aten.to.dtype)+def generic_to_dtype(func, args, kwargs):+ """Handle .to(dtype) calls - dtype conversion only."""+ src = args[0]+ if isinstance(src, QuantizedTensor):+ # For dtype-only conversion, just change the orig_dtype, no real cast is needed+ target_dtype = args[1] if len(args) > 1 else kwargs.get('dtype')+ src._layout_params["orig_dtype"] = target_dtype+ return src+ return func(*args, **kwargs)++@register_generic_util(torch.ops.aten._has_compatible_shallow_copy_type.default)def generic_has_compatible_shallow_copy_type(func, args, kwargs):return Truediff --git a/comfy/sd.py b/comfy/sd.pyindex dc0905ada..b6df0bd61 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -917,7 +917,12 @@ class CLIPType(Enum):def load_clip(ckpt_paths, embedding_directory=None, clip_type=CLIPType.STABLE_DIFFUSION, model_options={}):clip_data = []for p in ckpt_paths:- clip_data.append(comfy.utils.load_torch_file(p, safe_load=True))+ sd, metadata = comfy.utils.load_torch_file(p, safe_load=True, return_metadata=True)+ if metadata is not None:+ quant_metadata = metadata.get("_quantization_metadata", None)+ if quant_metadata is not None:+ sd["_quantization_metadata"] = quant_metadata+ clip_data.append(sd)return load_text_encoder_state_dicts(clip_data, embedding_directory=embedding_directory, clip_type=clip_type, model_options=model_options)@@ -1142,6 +1147,8 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipparameters = 0for c in clip_data:+ if "_quantization_metadata" in c:+ c.pop("_quantization_metadata")parameters += comfy.utils.calculate_parameters(c)tokenizer_data, model_options = comfy.text_encoders.long_clipl.model_options_long_clip(c, tokenizer_data, model_options)diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.pyindex 3066de2d7..8f509bab1 100644--- a/comfy/sd1_clip.py+++ b/comfy/sd1_clip.py@@ -109,13 +109,23 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):operations = model_options.get("custom_operations", None)scaled_fp8 = None+ quantization_metadata = model_options.get("quantization_metadata", None)if operations is None:- scaled_fp8 = model_options.get("scaled_fp8", None)- if scaled_fp8 is not None:- operations = comfy.ops.scaled_fp8_ops(fp8_matrix_mult=False, override_dtype=scaled_fp8)+ layer_quant_config = None+ if quantization_metadata is not None:+ layer_quant_config = json.loads(quantization_metadata).get("layers", None)++ if layer_quant_config is not None:+ operations = comfy.ops.mixed_precision_ops(layer_quant_config, dtype, full_precision_mm=True)+ logging.info(f"Using MixedPrecisionOps for text encoder: {len(layer_quant_config)} quantized layers")else:- operations = comfy.ops.manual_cast+ # Fallback to scaled_fp8_ops for backward compatibility+ scaled_fp8 = model_options.get("scaled_fp8", None)+ if scaled_fp8 is not None:+ operations = comfy.ops.scaled_fp8_ops(fp8_matrix_mult=False, override_dtype=scaled_fp8)+ else:+ operations = comfy.ops.manual_castself.operations = operationsself.transformer = model_class(config, dtype, device, self.operations)diff --git a/comfy/text_encoders/hunyuan_video.py b/comfy/text_encoders/hunyuan_video.pyindex 557094f49..0110517bb 100644--- a/comfy/text_encoders/hunyuan_video.py+++ b/comfy/text_encoders/hunyuan_video.py@@ -18,6 +18,9 @@ def llama_detect(state_dict, prefix=""):if scaled_fp8_key in state_dict:out["llama_scaled_fp8"] = state_dict[scaled_fp8_key].dtype+ if "_quantization_metadata" in state_dict:+ out["llama_quantization_metadata"] = state_dict["_quantization_metadata"]+return outdiff --git a/tests-unit/comfy_quant/test_mixed_precision.py b/tests-unit/comfy_quant/test_mixed_precision.pyindex f8d1fd04e..63361309f 100644--- a/tests-unit/comfy_quant/test_mixed_precision.py+++ b/tests-unit/comfy_quant/test_mixed_precision.py@@ -37,11 +37,8 @@ class TestMixedPrecisionOps(unittest.TestCase):def test_all_layers_standard(self):"""Test that model with no quantization works normally"""- # Configure no quantization- ops.MixedPrecisionOps._layer_quant_config = {}-# Create model- model = SimpleModel(operations=ops.MixedPrecisionOps)+ model = SimpleModel(operations=ops.mixed_precision_ops({}))# Initialize weights manuallymodel.layer1.weight = torch.nn.Parameter(torch.randn(20, 10, dtype=torch.bfloat16))@@ -76,7 +73,6 @@ class TestMixedPrecisionOps(unittest.TestCase):"params": {}}}- ops.MixedPrecisionOps._layer_quant_config = layer_quant_config# Create state dict with mixed precisionfp8_weight1 = torch.randn(20, 10, dtype=torch.float32).to(torch.float8_e4m3fn)@@ -99,7 +95,7 @@ class TestMixedPrecisionOps(unittest.TestCase):}# Create model and load state dict (strict=False because custom loading pops keys)- model = SimpleModel(operations=ops.MixedPrecisionOps)+ model = SimpleModel(operations=ops.mixed_precision_ops(layer_quant_config))model.load_state_dict(state_dict, strict=False)# Verify weights are wrapped in QuantizedTensor@@ -132,7 +128,6 @@ class TestMixedPrecisionOps(unittest.TestCase):"params": {}}}- ops.MixedPrecisionOps._layer_quant_config = layer_quant_config# Create and load modelfp8_weight = torch.randn(20, 10, dtype=torch.float32).to(torch.float8_e4m3fn)@@ -146,7 +141,7 @@ class TestMixedPrecisionOps(unittest.TestCase):"layer3.bias": torch.randn(40, dtype=torch.bfloat16),}- model = SimpleModel(operations=ops.MixedPrecisionOps)+ model = SimpleModel(operations=ops.mixed_precision_ops(layer_quant_config))model.load_state_dict(state_dict1, strict=False)# Save state dict@@ -170,7 +165,6 @@ class TestMixedPrecisionOps(unittest.TestCase):"params": {}}}- ops.MixedPrecisionOps._layer_quant_config = layer_quant_config# Create and load modelfp8_weight = torch.randn(20, 10, dtype=torch.float32).to(torch.float8_e4m3fn)@@ -184,7 +178,7 @@ class TestMixedPrecisionOps(unittest.TestCase):"layer3.bias": torch.randn(40, dtype=torch.bfloat16),}- model = SimpleModel(operations=ops.MixedPrecisionOps)+ model = SimpleModel(operations=ops.mixed_precision_ops(layer_quant_config))model.load_state_dict(state_dict, strict=False)# Add a weight function (simulating LoRA)@@ -210,7 +204,6 @@ class TestMixedPrecisionOps(unittest.TestCase):"params": {}}}- ops.MixedPrecisionOps._layer_quant_config = layer_quant_config# Create state dictstate_dict = {@@ -223,7 +216,7 @@ class TestMixedPrecisionOps(unittest.TestCase):}# Load should raise KeyError for unknown format in QUANT_FORMAT_MIXINS- model = SimpleModel(operations=ops.MixedPrecisionOps)+ model = SimpleModel(operations=ops.mixed_precision_ops(layer_quant_config))with self.assertRaises(KeyError):model.load_state_dict(state_dict, strict=False)