The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Initial ops changes to use comfy_kitchen: Initial nvfp4 checkpoint support. (#11635)
comfyanonymous6da00dd899e3
diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.ymlindex 419873ad8..9160242e9 100644--- a/.github/workflows/test-build.yml+++ b/.github/workflows/test-build.yml@@ -18,7 +18,7 @@ jobs:strategy:fail-fast: falsematrix:- python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]+ python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]steps:- uses: actions/checkout@v4- name: Set up Python ${{ matrix.python-version }}diff --git a/.github/workflows/test-launch.yml b/.github/workflows/test-launch.ymlindex fd70aff23..ef0d3f123 100644--- a/.github/workflows/test-launch.yml+++ b/.github/workflows/test-launch.yml@@ -32,7 +32,9 @@ jobs:working-directory: ComfyUI- name: Check for unhandled exceptions in server logrun: |- if grep -qE "Exception|Error" console_output.log; then+ grep -v "Found comfy_kitchen backend triton: {'available': False, 'disabled': True, 'unavailable_reason': \"ImportError: No module named 'triton'\", 'capabilities': \[\]}" console_output.log | grep -v "Found comfy_kitchen backend triton: {'available': False, 'disabled': False, 'unavailable_reason': \"ImportError: No module named 'triton'\", 'capabilities': \[\]}" > console_output_filtered.log+ cat console_output_filtered.log+ if grep -qE "Exception|Error" console_output_filtered.log; thenecho "Unhandled exception/error found in server log."exit 1fidiff --git a/comfy/model_management.py b/comfy/model_management.pyindex 7f5a8aee9..22f4de044 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -1156,7 +1156,7 @@ def pin_memory(tensor):if not tensor.is_contiguous():return False- size = tensor.numel() * tensor.element_size()+ size = tensor.nbytesif (TOTAL_PINNED_MEMORY + size) > MAX_PINNED_MEMORY:return False@@ -1183,7 +1183,7 @@ def unpin_memory(tensor):return Falseptr = tensor.data_ptr()- size = tensor.numel() * tensor.element_size()+ size = tensor.nbytessize_stored = PINNED_MEMORY.get(ptr, None)if size_stored is None:diff --git a/comfy/ops.py b/comfy/ops.pyindex 16889bb82..f5e1e9230 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -79,7 +79,7 @@ def cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype=None, ofif input is not None:if dtype is None:if isinstance(input, QuantizedTensor):- dtype = input._layout_params["orig_dtype"]+ dtype = input.params.orig_dtypeelse:dtype = input.dtypeif bias_dtype is None:@@ -412,26 +412,34 @@ def fp8_linear(self, input):return Noneinput_dtype = input.dtype+ input_shape = input.shape+ tensor_3d = input.ndim == 3- if input.ndim == 3 or input.ndim == 2:- w, bias, offload_stream = cast_bias_weight(self, input, dtype=dtype, bias_dtype=input_dtype, offloadable=True)- scale_weight = torch.ones((), device=input.device, dtype=torch.float32)+ if tensor_3d:+ input = input.reshape(-1, input_shape[2])- scale_input = torch.ones((), device=input.device, dtype=torch.float32)- input = torch.clamp(input, min=-448, max=448, out=input)- layout_params_weight = {'scale': scale_input, 'orig_dtype': input_dtype}- quantized_input = QuantizedTensor(input.to(dtype).contiguous(), "TensorCoreFP8Layout", layout_params_weight)+ if input.ndim != 2:+ return None+ w, bias, offload_stream = cast_bias_weight(self, input, dtype=dtype, bias_dtype=input_dtype, offloadable=True)+ scale_weight = torch.ones((), device=input.device, dtype=torch.float32)++ scale_input = torch.ones((), device=input.device, dtype=torch.float32)+ input = torch.clamp(input, min=-448, max=448, out=input)+ input_fp8 = input.to(dtype).contiguous()+ layout_params_input = TensorCoreFP8Layout.Params(scale=scale_input, orig_dtype=input_dtype, orig_shape=tuple(input_fp8.shape))+ quantized_input = QuantizedTensor(input_fp8, TensorCoreFP8Layout, layout_params_input)- # Wrap weight in QuantizedTensor - this enables unified dispatch- # Call F.linear - __torch_dispatch__ routes to fp8_linear handler in quant_ops.py!- layout_params_weight = {'scale': scale_weight, 'orig_dtype': input_dtype}- quantized_weight = QuantizedTensor(w, "TensorCoreFP8Layout", layout_params_weight)- o = torch.nn.functional.linear(quantized_input, quantized_weight, bias)+ # Wrap weight in QuantizedTensor - this enables unified dispatch+ # Call F.linear - __torch_dispatch__ routes to fp8_linear handler in quant_ops.py!+ layout_params_weight = TensorCoreFP8Layout.Params(scale=scale_weight, orig_dtype=input_dtype, orig_shape=tuple(w.shape))+ quantized_weight = QuantizedTensor(w, TensorCoreFP8Layout, layout_params_weight)+ o = torch.nn.functional.linear(quantized_input, quantized_weight, bias)- uncast_bias_weight(self, w, bias, offload_stream)- return o+ uncast_bias_weight(self, w, bias, offload_stream)+ if tensor_3d:+ o = o.reshape((input_shape[0], input_shape[1], w.shape[0]))- return None+ return oclass fp8_ops(manual_cast):class Linear(manual_cast.Linear):@@ -477,7 +485,12 @@ if CUBLAS_IS_AVAILABLE:# ==============================================================================# Mixed Precision Operations# ==============================================================================-from .quant_ops import QuantizedTensor, QUANT_ALGOS+from .quant_ops import (+ QuantizedTensor,+ QUANT_ALGOS,+ TensorCoreFP8Layout,+ get_layout_class,+)def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precision_mm=False):@@ -497,14 +510,15 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec) -> None:super().__init__()- if dtype is None:- dtype = MixedPrecisionOps._compute_dtype-- self.factory_kwargs = {"device": device, "dtype": dtype}+ self.factory_kwargs = {"device": device, "dtype": MixedPrecisionOps._compute_dtype}+ # self.factory_kwargs = {"device": device, "dtype": dtype}self.in_features = in_featuresself.out_features = out_features- self._has_bias = bias+ if bias:+ self.bias = torch.nn.Parameter(torch.empty(out_features, **self.factory_kwargs))+ else:+ self.register_parameter("bias", None)self.tensor_class = Noneself._full_precision_mm = MixedPrecisionOps._full_precision_mm@@ -512,6 +526,16 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precdef reset_parameters(self):return None+ def _load_scale_param(self, state_dict, prefix, param_name, device, manually_loaded_keys, dtype=None):+ key = f"{prefix}{param_name}"+ value = state_dict.pop(key, None)+ if value is not None:+ value = value.to(device=device)+ if dtype is not None:+ value = value.view(dtype=dtype)+ manually_loaded_keys.append(key)+ return value+def _load_from_state_dict(self, state_dict, prefix, local_metadata,strict, missing_keys, unexpected_keys, error_msgs):@@ -529,14 +553,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_preclayer_conf = json.loads(layer_conf.numpy().tobytes())if layer_conf is None:- dtype = self.factory_kwargs["dtype"]- self.weight = torch.nn.Parameter(weight.to(device=device, dtype=dtype), requires_grad=False)- if dtype != MixedPrecisionOps._compute_dtype:- self.comfy_cast_weights = True- if self._has_bias:- self.bias = torch.nn.Parameter(torch.empty(self.out_features, device=device, dtype=dtype))- else:- self.register_parameter("bias", None)+ self.weight = torch.nn.Parameter(weight.to(device=device, dtype=MixedPrecisionOps._compute_dtype), requires_grad=False)else:self.quant_format = layer_conf.get("format", None)if not self._full_precision_mm:@@ -547,31 +564,46 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precqconfig = QUANT_ALGOS[self.quant_format]self.layout_type = qconfig["comfy_tensor_layout"]-- weight_scale_key = f"{prefix}weight_scale"- scale = state_dict.pop(weight_scale_key, None)- if scale is not None:- scale = scale.to(device)- layout_params = {- 'scale': scale,- 'orig_dtype': MixedPrecisionOps._compute_dtype,- 'block_size': qconfig.get("group_size", None),- }-- if scale is not None:- manually_loaded_keys.append(weight_scale_key)+ layout_cls = get_layout_class(self.layout_type)++ # Load format-specific parameters+ if self.quant_format in ["float8_e4m3fn", "float8_e5m2"]:+ # FP8: single tensor scale+ scale = self._load_scale_param(state_dict, prefix, "weight_scale", device, manually_loaded_keys)++ params = layout_cls.Params(+ scale=scale,+ orig_dtype=MixedPrecisionOps._compute_dtype,+ orig_shape=(self.out_features, self.in_features),+ )++ elif self.quant_format == "nvfp4":+ # NVFP4: tensor_scale (weight_scale_2) + block_scale (weight_scale)+ tensor_scale = self._load_scale_param(state_dict, prefix, "weight_scale_2", device, manually_loaded_keys)+ block_scale = self._load_scale_param(state_dict, prefix, "weight_scale", device, manually_loaded_keys,+ dtype=torch.float8_e4m3fn)++ if tensor_scale is None or block_scale is None:+ raise ValueError(f"Missing NVFP4 scales for layer {layer_name}")++ params = layout_cls.Params(+ scale=tensor_scale,+ block_scale=block_scale,+ orig_dtype=MixedPrecisionOps._compute_dtype,+ orig_shape=(self.out_features, self.in_features),+ )+ else:+ raise ValueError(f"Unsupported quantization format: {self.quant_format}")self.weight = torch.nn.Parameter(- QuantizedTensor(weight.to(device=device, dtype=qconfig.get("storage_t", None)), self.layout_type, layout_params),+ QuantizedTensor(weight.to(device=device, dtype=qconfig["storage_t"]), self.layout_type, params),requires_grad=False)- if self._has_bias:- self.bias = torch.nn.Parameter(torch.empty(self.out_features, device=device, dtype=MixedPrecisionOps._compute_dtype))- else:- self.register_parameter("bias", None)-for param_name in qconfig["parameters"]:+ if param_name in {"weight_scale", "weight_scale_2"}:+ continue # Already handled above+param_key = f"{prefix}{param_name}"_v = state_dict.pop(param_key, None)if _v is None:@@ -588,7 +620,15 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precdef state_dict(self, *args, destination=None, prefix="", **kwargs):sd = super().state_dict(*args, destination=destination, prefix=prefix, **kwargs)if isinstance(self.weight, QuantizedTensor):- sd["{}weight_scale".format(prefix)] = self.weight._layout_params['scale']+ layout_cls = self.weight._layout_cls++ # Check if it's any FP8 variant (E4M3 or E5M2)+ if layout_cls in ("TensorCoreFP8E4M3Layout", "TensorCoreFP8E5M2Layout", "TensorCoreFP8Layout"):+ sd["{}weight_scale".format(prefix)] = self.weight._params.scale+ elif layout_cls == "TensorCoreNVFP4Layout":+ sd["{}weight_scale_2".format(prefix)] = self.weight._params.scale+ sd["{}weight_scale".format(prefix)] = self.weight._params.block_scale+quant_conf = {"format": self.quant_format}if self._full_precision_mm:quant_conf["full_precision_matrix_mult"] = True@@ -607,12 +647,33 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precdef forward(self, input, *args, **kwargs):run_every_op()+ input_shape = input.shape+ tensor_3d = input.ndim == 3+if self._full_precision_mm or self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0:return self.forward_comfy_cast_weights(input, *args, **kwargs)+if (getattr(self, 'layout_type', None) is not None andnot isinstance(input, QuantizedTensor)):- input = QuantizedTensor.from_float(input, self.layout_type, scale=getattr(self, 'input_scale', None), dtype=self.weight.dtype)- return self._forward(input, self.weight, self.bias)++ # Reshape 3D tensors to 2D for quantization (needed for NVFP4 and others)+ if tensor_3d:+ input = input.reshape(-1, input_shape[2])++ if input.ndim != 2:+ # Fall back to comfy_cast_weights for non-2D tensors+ return self.forward_comfy_cast_weights(input.reshape(input_shape), *args, **kwargs)++ # dtype is now implicit in the layout class+ input = QuantizedTensor.from_float(input, self.layout_type, scale=getattr(self, 'input_scale', None))++ output = self._forward(input, self.weight, self.bias)++ # Reshape output back to 3D if input was 3D+ if tensor_3d:+ output = output.reshape((input_shape[0], input_shape[1], self.weight.shape[0]))++ return outputdef convert_weight(self, weight, inplace=False, **kwargs):if isinstance(weight, QuantizedTensor):@@ -622,7 +683,8 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precdef set_weight(self, weight, inplace_update=False, seed=None, return_weight=False, **kwargs):if getattr(self, 'layout_type', None) is not None:- weight = QuantizedTensor.from_float(weight, self.layout_type, scale="recalculate", dtype=self.weight.dtype, stochastic_rounding=seed, inplace_ops=True)+ # dtype is now implicit in the layout class+ weight = QuantizedTensor.from_float(weight, self.layout_type, scale="recalculate", stochastic_rounding=seed, inplace_ops=True)else:weight = weight.to(self.weight.dtype)if return_weight:diff --git a/comfy/quant_ops.py b/comfy/quant_ops.pyindex cd96541d7..cd737726f 100644--- a/comfy/quant_ops.py+++ b/comfy/quant_ops.py@@ -1,580 +1,133 @@import torchimport logging-from typing import Tuple, Dict-import comfy.float--_LAYOUT_REGISTRY = {}-_GENERIC_UTILS = {}---def register_layout_op(torch_op, layout_type):- """- Decorator to register a layout-specific operation handler.- Args:- torch_op: PyTorch operation (e.g., torch.ops.aten.linear.default)- layout_type: Layout class (e.g., TensorCoreFP8Layout)- Example:- @register_layout_op(torch.ops.aten.linear.default, TensorCoreFP8Layout)- def fp8_linear(func, args, kwargs):- # FP8-specific linear implementation- ...- """- def decorator(handler_func):- if torch_op not in _LAYOUT_REGISTRY:- _LAYOUT_REGISTRY[torch_op] = {}- _LAYOUT_REGISTRY[torch_op][layout_type] = handler_func- return handler_func- return decorator---def register_generic_util(torch_op):- """- Decorator to register a generic utility that works for all layouts.- Args:- torch_op: PyTorch operation (e.g., torch.ops.aten.detach.default)-- Example:- @register_generic_util(torch.ops.aten.detach.default)- def generic_detach(func, args, kwargs):- # Works for any layout- ...- """- def decorator(handler_func):- _GENERIC_UTILS[torch_op] = handler_func- return handler_func- return decorator---def _get_layout_from_args(args):- for arg in args:- if isinstance(arg, QuantizedTensor):- return arg._layout_type- elif isinstance(arg, (list, tuple)):- for item in arg:- if isinstance(item, QuantizedTensor):- return item._layout_type- return None---def _move_layout_params_to_device(params, device):- new_params = {}- for k, v in params.items():- if isinstance(v, torch.Tensor):- new_params[k] = v.to(device=device)- else:- new_params[k] = v- return new_params---def _copy_layout_params(params):- new_params = {}- for k, v in params.items():- if isinstance(v, torch.Tensor):- new_params[k] = v.clone()- else:- new_params[k] = v- return new_params--def _copy_layout_params_inplace(src, dst, non_blocking=False):- for k, v in src.items():- if isinstance(v, torch.Tensor):- dst[k].copy_(v, non_blocking=non_blocking)- else:- dst[k] = v--class QuantizedLayout:- """- Base class for quantization layouts.-- A layout encapsulates the format-specific logic for quantization/dequantization- and provides a uniform interface for extracting raw tensors needed for computation.-- New quantization formats should subclass this and implement the required methods.- """- @classmethod- def quantize(cls, tensor, **kwargs) -> Tuple[torch.Tensor, Dict]:- raise NotImplementedError(f"{cls.__name__} must implement quantize()")-- @staticmethod- def dequantize(qdata, **layout_params) -> torch.Tensor:- raise NotImplementedError("TensorLayout must implement dequantize()")-- @classmethod- def get_plain_tensors(cls, qtensor) -> torch.Tensor:- raise NotImplementedError(f"{cls.__name__} must implement get_plain_tensors()")---class QuantizedTensor(torch.Tensor):- """- Universal quantized tensor that works with any layout.-- This tensor subclass uses a pluggable layout system to support multiple- quantization formats (FP8, INT4, INT8, etc.) without code duplication.-- The layout_type determines format-specific behavior, while common operations- (detach, clone, to) are handled generically.-- Attributes:- _qdata: The quantized tensor data- _layout_type: Layout class (e.g., TensorCoreFP8Layout)- _layout_params: Dict with layout-specific params (scale, zero_point, etc.)- """-- @staticmethod- def __new__(cls, qdata, layout_type, layout_params):- """- Create a quantized tensor.-- Args:- qdata: The quantized data tensor- layout_type: Layout class (subclass of QuantizedLayout)- layout_params: Dict with layout-specific parameters- """- return torch.Tensor._make_wrapper_subclass(cls, qdata.shape, device=qdata.device, dtype=qdata.dtype, requires_grad=False)-- def __init__(self, qdata, layout_type, layout_params):- self._qdata = qdata- self._layout_type = layout_type- self._layout_params = layout_params- def __repr__(self):- layout_name = self._layout_type- param_str = ", ".join(f"{k}={v}" for k, v in list(self._layout_params.items())[:2])- return f"QuantizedTensor(shape={self.shape}, layout={layout_name}, {param_str})"-- @property- def layout_type(self):- return self._layout_type-- def __tensor_flatten__(self):- """- Tensor flattening protocol for proper device movement.- """- inner_tensors = ["_qdata"]- ctx = {- "layout_type": self._layout_type,- }-- tensor_params = {}- non_tensor_params = {}- for k, v in self._layout_params.items():- if isinstance(v, torch.Tensor):- tensor_params[k] = v- else:- non_tensor_params[k] = v-- ctx["tensor_param_keys"] = list(tensor_params.keys())- ctx["non_tensor_params"] = non_tensor_params-- for k, v in tensor_params.items():- attr_name = f"_layout_param_{k}"- object.__setattr__(self, attr_name, v)- inner_tensors.append(attr_name)-- return inner_tensors, ctx-- @staticmethod- def __tensor_unflatten__(inner_tensors, ctx, outer_size, outer_stride):- """- Tensor unflattening protocol for proper device movement.- Reconstructs the QuantizedTensor after device movement.- """- layout_type = ctx["layout_type"]- layout_params = dict(ctx["non_tensor_params"])-- for key in ctx["tensor_param_keys"]:- attr_name = f"_layout_param_{key}"- layout_params[key] = inner_tensors[attr_name]-- return QuantizedTensor(inner_tensors["_qdata"], layout_type, layout_params)-- @classmethod- def from_float(cls, tensor, layout_type, **quantize_kwargs) -> 'QuantizedTensor':- qdata, layout_params = LAYOUTS[layout_type].quantize(tensor, **quantize_kwargs)- return cls(qdata, layout_type, layout_params)-- def dequantize(self) -> torch.Tensor:- return LAYOUTS[self._layout_type].dequantize(self._qdata, **self._layout_params)-- @classmethod- def __torch_dispatch__(cls, func, types, args=(), kwargs=None):- kwargs = kwargs or {}-- # Step 1: Check generic utilities first (detach, clone, to, etc.)- if func in _GENERIC_UTILS:- return _GENERIC_UTILS[func](func, args, kwargs)-- # Step 2: Check layout-specific handlers (linear, matmul, etc.)- layout_type = _get_layout_from_args(args)- if layout_type and func in _LAYOUT_REGISTRY:- handler = _LAYOUT_REGISTRY[func].get(layout_type)- if handler:- return handler(func, args, kwargs)-- # Step 3: Fallback to dequantization- if isinstance(args[0] if args else None, QuantizedTensor):- logging.info(f"QuantizedTensor: Unhandled operation {func}, falling back to dequantization. kwargs={kwargs}")- return cls._dequant_and_fallback(func, args, kwargs)+try:+ import comfy_kitchen as ck+ from comfy_kitchen.tensor import (+ QuantizedTensor,+ QuantizedLayout,+ TensorCoreFP8Layout as _CKFp8Layout,+ TensorCoreNVFP4Layout, # Direct import, no wrapper needed+ register_layout_op,+ register_layout_class,+ get_layout_class,+ )+ _CK_AVAILABLE = True+ ck.registry.disable("triton")+ for k, v in ck.list_backends().items():+ logging.info(f"Found comfy_kitchen backend {k}: {v}")+except ImportError as e:+ logging.error(f"Failed to import comfy_kitchen, Error: {e}, fp8 and fp4 support will not be available.")+ _CK_AVAILABLE = False- @classmethod- def _dequant_and_fallback(cls, func, args, kwargs):- def dequant_arg(arg):- if isinstance(arg, QuantizedTensor):- return arg.dequantize()- elif isinstance(arg, (list, tuple)):- return type(arg)(dequant_arg(a) for a in arg)- return arg+ class QuantizedTensor:+ pass- new_args = dequant_arg(args)- new_kwargs = dequant_arg(kwargs)- return func(*new_args, **new_kwargs)+ class _CKFp8Layout:+ pass- def data_ptr(self):- return self._qdata.data_ptr()+ class TensorCoreNVFP4Layout:+ pass- def is_pinned(self):- return self._qdata.is_pinned()+ def register_layout_class(name, cls):+ pass- def is_contiguous(self, *arg, **kwargs):- return self._qdata.is_contiguous(*arg, **kwargs)+ def get_layout_class(name):+ return None- def storage(self):- return self._qdata.storage()+import comfy.float# ==============================================================================-# Generic Utilities (Layout-Agnostic Operations)+# FP8 Layouts with Comfy-Specific Extensions# ==============================================================================-def _create_transformed_qtensor(qt, transform_fn):- new_data = transform_fn(qt._qdata)- new_params = _copy_layout_params(qt._layout_params)- return QuantizedTensor(new_data, qt._layout_type, new_params)---def _handle_device_transfer(qt, target_device, target_dtype=None, target_layout=None, op_name="to"):- if target_layout is not None and target_layout != torch.strided:- logging.warning(- f"QuantizedTensor: layout change requested to {target_layout}, "- f"but not supported. Ignoring layout."- )-- # Handle device transfer- current_device = qt._qdata.device- if target_device is not None:- # Normalize device for comparison- if isinstance(target_device, str):- target_device = torch.device(target_device)- if isinstance(current_device, str):- current_device = torch.device(current_device)-- if target_device != current_device:- logging.debug(f"QuantizedTensor.{op_name}: Moving from {current_device} to {target_device}")- new_q_data = qt._qdata.to(device=target_device)- new_params = _move_layout_params_to_device(qt._layout_params, target_device)- if target_dtype is not None:- new_params["orig_dtype"] = target_dtype- new_qt = QuantizedTensor(new_q_data, qt._layout_type, new_params)- logging.debug(f"QuantizedTensor.{op_name}: Created new tensor on {target_device}")- return new_qt-- logging.debug(f"QuantizedTensor.{op_name}: No device change needed, returning original")- return qt---@register_generic_util(torch.ops.aten.detach.default)-def generic_detach(func, args, kwargs):- """Detach operation - creates a detached copy of the quantized tensor."""- qt = args[0]- if isinstance(qt, QuantizedTensor):- return _create_transformed_qtensor(qt, lambda x: x.detach())- return func(*args, **kwargs)---@register_generic_util(torch.ops.aten.clone.default)-def generic_clone(func, args, kwargs):- """Clone operation - creates a deep copy of the quantized tensor."""- qt = args[0]- if isinstance(qt, QuantizedTensor):- return _create_transformed_qtensor(qt, lambda x: x.clone())- return func(*args, **kwargs)---@register_generic_util(torch.ops.aten._to_copy.default)-def generic_to_copy(func, args, kwargs):- """Device/dtype transfer operation - handles .to(device) calls."""- qt = args[0]- if isinstance(qt, QuantizedTensor):- return _handle_device_transfer(- qt,- target_device=kwargs.get('device', None),- target_dtype=kwargs.get('dtype', None),- op_name="_to_copy"- )- return func(*args, **kwargs)+class _TensorCoreFP8LayoutBase(_CKFp8Layout):+ FP8_DTYPE = None # Must be overridden in subclass--@register_generic_util(torch.ops.aten.to.dtype_layout)-def generic_to_dtype_layout(func, args, kwargs):- """Handle .to(device) calls using the dtype_layout variant."""- qt = args[0]- if isinstance(qt, QuantizedTensor):- return _handle_device_transfer(- qt,- target_device=kwargs.get('device', None),- target_dtype=kwargs.get('dtype', None),- target_layout=kwargs.get('layout', None),- op_name="to"- )- return func(*args, **kwargs)---@register_generic_util(torch.ops.aten.copy_.default)-def generic_copy_(func, args, kwargs):- qt_dest = args[0]- src = args[1]- non_blocking = args[2] if len(args) > 2 else False- if isinstance(qt_dest, QuantizedTensor):- if isinstance(src, QuantizedTensor):- # Copy from another quantized tensor- qt_dest._qdata.copy_(src._qdata, non_blocking=non_blocking)- qt_dest._layout_type = src._layout_type- orig_dtype = qt_dest._layout_params["orig_dtype"]- _copy_layout_params_inplace(src._layout_params, qt_dest._layout_params, non_blocking=non_blocking)- qt_dest._layout_params["orig_dtype"] = orig_dtype- else:- # Copy from regular tensor - just copy raw data- qt_dest._qdata.copy_(src)- return qt_dest- return func(*args, **kwargs)---@register_generic_util(torch.ops.aten.to.dtype)-def generic_to_dtype(func, args, kwargs):- """Handle .to(dtype) calls - dtype conversion only."""- src = args[0]- if isinstance(src, QuantizedTensor):- # For dtype-only conversion, just change the orig_dtype, no real cast is needed- target_dtype = args[1] if len(args) > 1 else kwargs.get('dtype')- src._layout_params["orig_dtype"] = target_dtype- return src- return func(*args, **kwargs)---@register_generic_util(torch.ops.aten._has_compatible_shallow_copy_type.default)-def generic_has_compatible_shallow_copy_type(func, args, kwargs):- return True---@register_generic_util(torch.ops.aten.empty_like.default)-def generic_empty_like(func, args, kwargs):- """Empty_like operation - creates an empty tensor with the same quantized structure."""- qt = args[0]- if isinstance(qt, QuantizedTensor):- # Create empty tensor with same shape and dtype as the quantized data- hp_dtype = kwargs.pop('dtype', qt._layout_params["orig_dtype"])- new_qdata = torch.empty_like(qt._qdata, **kwargs)-- # Handle device transfer for layout params- target_device = kwargs.get('device', new_qdata.device)- new_params = _move_layout_params_to_device(qt._layout_params, target_device)-- # Update orig_dtype if dtype is specified- new_params['orig_dtype'] = hp_dtype-- return QuantizedTensor(new_qdata, qt._layout_type, new_params)- return func(*args, **kwargs)--# ==============================================================================-# FP8 Layout + Operation Handlers-# ==============================================================================-class TensorCoreFP8Layout(QuantizedLayout):- """- Storage format:- - qdata: FP8 tensor (torch.float8_e4m3fn or torch.float8_e5m2)- - scale: Scalar tensor (float32) for dequantization- - orig_dtype: Original dtype before quantization (for casting back)- """@classmethod- def quantize(cls, tensor, scale=None, dtype=torch.float8_e4m3fn, stochastic_rounding=0, inplace_ops=False):+ def quantize(cls, tensor, scale=None, stochastic_rounding=0, inplace_ops=False):+ if cls.FP8_DTYPE is None:+ raise NotImplementedError(f"{cls.__name__} must define FP8_DTYPE")+orig_dtype = tensor.dtype+ orig_shape = tuple(tensor.shape)if isinstance(scale, str) and scale == "recalculate":- scale = torch.amax(tensor.abs()).to(dtype=torch.float32) / torch.finfo(dtype).max+ scale = torch.amax(tensor.abs()).to(dtype=torch.float32) / torch.finfo(cls.FP8_DTYPE).maxif tensor.dtype not in [torch.float32, torch.bfloat16]: # Prevent scale from being too smalltensor_info = torch.finfo(tensor.dtype)scale = (1.0 / torch.clamp((1.0 / scale), min=tensor_info.min, max=tensor_info.max))- if scale is not None:- if not isinstance(scale, torch.Tensor):- scale = torch.tensor(scale)- scale = scale.to(device=tensor.device, dtype=torch.float32)+ if scale is None:+ scale = torch.ones((), device=tensor.device, dtype=torch.float32)+ if not isinstance(scale, torch.Tensor):+ scale = torch.tensor(scale, device=tensor.device, dtype=torch.float32)+ if stochastic_rounding > 0:if inplace_ops:tensor *= (1.0 / scale).to(tensor.dtype)else:tensor = tensor * (1.0 / scale).to(tensor.dtype)+ qdata = comfy.float.stochastic_rounding(tensor, dtype=cls.FP8_DTYPE, seed=stochastic_rounding)else:- scale = torch.ones((), device=tensor.device, dtype=torch.float32)-- if stochastic_rounding > 0:- tensor = comfy.float.stochastic_rounding(tensor, dtype=dtype, seed=stochastic_rounding)- else:- lp_amax = torch.finfo(dtype).max- torch.clamp(tensor, min=-lp_amax, max=lp_amax, out=tensor)- tensor = tensor.to(dtype, memory_format=torch.contiguous_format)-- layout_params = {- 'scale': scale,- 'orig_dtype': orig_dtype- }- return tensor, layout_params+ qdata = ck.quantize_per_tensor_fp8(tensor, scale, cls.FP8_DTYPE)- @staticmethod- def dequantize(qdata, scale, orig_dtype, **kwargs):- plain_tensor = torch.ops.aten._to_copy.default(qdata, dtype=orig_dtype)- plain_tensor.mul_(scale)- return plain_tensor+ params = cls.Params(scale=scale.float(), orig_dtype=orig_dtype, orig_shape=orig_shape)+ return qdata, params- @classmethod- def get_plain_tensors(cls, qtensor):- return qtensor._qdata, qtensor._layout_params['scale']-QUANT_ALGOS = {- "float8_e4m3fn": {- "storage_t": torch.float8_e4m3fn,- "parameters": {"weight_scale", "input_scale"},- "comfy_tensor_layout": "TensorCoreFP8Layout",- },-}+class TensorCoreFP8E4M3Layout(_TensorCoreFP8LayoutBase):+ FP8_DTYPE = torch.float8_e4m3fn-LAYOUTS = {- "TensorCoreFP8Layout": TensorCoreFP8Layout,-}+class TensorCoreFP8E5M2Layout(_TensorCoreFP8LayoutBase):+ FP8_DTYPE = torch.float8_e5m2-@register_layout_op(torch.ops.aten.linear.default, "TensorCoreFP8Layout")-def fp8_linear(func, args, kwargs):- input_tensor = args[0]- weight = args[1]- bias = args[2] if len(args) > 2 else None- if isinstance(input_tensor, QuantizedTensor) and isinstance(weight, QuantizedTensor):- plain_input, scale_a = TensorCoreFP8Layout.get_plain_tensors(input_tensor)- plain_weight, scale_b = TensorCoreFP8Layout.get_plain_tensors(weight)+# Backward compatibility alias - default to E4M3+TensorCoreFP8Layout = TensorCoreFP8E4M3Layout- out_dtype = kwargs.get("out_dtype")- if out_dtype is None:- out_dtype = input_tensor._layout_params['orig_dtype']- weight_t = plain_weight.t()-- tensor_2d = False- if len(plain_input.shape) == 2:- tensor_2d = True- plain_input = plain_input.unsqueeze(1)-- input_shape = plain_input.shape- if len(input_shape) != 3:- return None-- try:- output = torch._scaled_mm(- plain_input.reshape(-1, input_shape[2]).contiguous(),- weight_t,- bias=bias,- scale_a=scale_a,- scale_b=scale_b,- out_dtype=out_dtype,- )-- if isinstance(output, tuple): # TODO: remove when we drop support for torch 2.4- output = output[0]-- if not tensor_2d:- output = output.reshape((-1, input_shape[1], weight.shape[0]))-- if output.dtype in [torch.float8_e4m3fn, torch.float8_e5m2]:- output_scale = scale_a * scale_b- output_params = {- 'scale': output_scale,- 'orig_dtype': input_tensor._layout_params['orig_dtype']- }- return QuantizedTensor(output, "TensorCoreFP8Layout", output_params)- else:- return output-- except Exception as e:- raise RuntimeError(f"FP8 _scaled_mm failed, falling back to dequantization: {e}")-- # Case 2: DQ Fallback- if isinstance(weight, QuantizedTensor):- weight = weight.dequantize()- if isinstance(input_tensor, QuantizedTensor):- input_tensor = input_tensor.dequantize()-- return torch.nn.functional.linear(input_tensor, weight, bias)--def fp8_mm_(input_tensor, weight, bias=None, out_dtype=None):- if out_dtype is None:- out_dtype = input_tensor._layout_params['orig_dtype']-- plain_input, scale_a = TensorCoreFP8Layout.get_plain_tensors(input_tensor)- plain_weight, scale_b = TensorCoreFP8Layout.get_plain_tensors(weight)-- output = torch._scaled_mm(- plain_input.contiguous(),- plain_weight,- bias=bias,- scale_a=scale_a,- scale_b=scale_b,- out_dtype=out_dtype,- )-- if isinstance(output, tuple): # TODO: remove when we drop support for torch 2.4- output = output[0]- return output--@register_layout_op(torch.ops.aten.addmm.default, "TensorCoreFP8Layout")-def fp8_addmm(func, args, kwargs):- input_tensor = args[1]- weight = args[2]- bias = args[0]-- if isinstance(input_tensor, QuantizedTensor) and isinstance(weight, QuantizedTensor):- return fp8_mm_(input_tensor, weight, bias=bias, out_dtype=kwargs.get("out_dtype", None))-- a = list(args)- if isinstance(args[0], QuantizedTensor):- a[0] = args[0].dequantize()- if isinstance(args[1], QuantizedTensor):- a[1] = args[1].dequantize()- if isinstance(args[2], QuantizedTensor):- a[2] = args[2].dequantize()+# ==============================================================================+# Registry+# ==============================================================================- return func(*a, **kwargs)+register_layout_class("TensorCoreFP8Layout", TensorCoreFP8Layout)+register_layout_class("TensorCoreFP8E4M3Layout", TensorCoreFP8E4M3Layout)+register_layout_class("TensorCoreFP8E5M2Layout", TensorCoreFP8E5M2Layout)+register_layout_class("TensorCoreNVFP4Layout", TensorCoreNVFP4Layout)-@register_layout_op(torch.ops.aten.mm.default, "TensorCoreFP8Layout")-def fp8_mm(func, args, kwargs):- input_tensor = args[0]- weight = args[1]+QUANT_ALGOS = {+ "float8_e4m3fn": {+ "storage_t": torch.float8_e4m3fn,+ "parameters": {"weight_scale", "input_scale"},+ "comfy_tensor_layout": "TensorCoreFP8E4M3Layout",+ },+ "float8_e5m2": {+ "storage_t": torch.float8_e5m2,+ "parameters": {"weight_scale", "input_scale"},+ "comfy_tensor_layout": "TensorCoreFP8E5M2Layout",+ },+ "nvfp4": {+ "storage_t": torch.uint8,+ "parameters": {"weight_scale", "weight_scale_2", "input_scale"},+ "comfy_tensor_layout": "TensorCoreNVFP4Layout",+ "group_size": 16,+ },+}- if isinstance(input_tensor, QuantizedTensor) and isinstance(weight, QuantizedTensor):- return fp8_mm_(input_tensor, weight, bias=None, out_dtype=kwargs.get("out_dtype", None))- a = list(args)- if isinstance(args[0], QuantizedTensor):- a[0] = args[0].dequantize()- if isinstance(args[1], QuantizedTensor):- a[1] = args[1].dequantize()- return func(*a, **kwargs)+# ==============================================================================+# Re-exports for backward compatibility+# ==============================================================================-@register_layout_op(torch.ops.aten.view.default, "TensorCoreFP8Layout")-@register_layout_op(torch.ops.aten.t.default, "TensorCoreFP8Layout")-def fp8_func(func, args, kwargs):- input_tensor = args[0]- if isinstance(input_tensor, QuantizedTensor):- plain_input, scale_a = TensorCoreFP8Layout.get_plain_tensors(input_tensor)- ar = list(args)- ar[0] = plain_input- return QuantizedTensor(func(*ar, **kwargs), "TensorCoreFP8Layout", input_tensor._layout_params)- return func(*args, **kwargs)+__all__ = [+ "QuantizedTensor",+ "QuantizedLayout",+ "TensorCoreFP8Layout",+ "TensorCoreFP8E4M3Layout",+ "TensorCoreFP8E5M2Layout",+ "TensorCoreNVFP4Layout",+ "QUANT_ALGOS",+ "register_layout_op",+]diff --git a/requirements.txt b/requirements.txtindex 3a05799eb..0ee152032 100644--- a/requirements.txt+++ b/requirements.txt@@ -21,6 +21,7 @@ psutilalembicSQLAlchemyav>=14.2.0+comfy-kitchen>=0.2.0#non essential dependencies:kornia>=0.7.1diff --git a/tests-unit/comfy_quant/test_mixed_precision.py b/tests-unit/comfy_quant/test_mixed_precision.pyindex 3a54941e6..7b2eac940 100644--- a/tests-unit/comfy_quant/test_mixed_precision.py+++ b/tests-unit/comfy_quant/test_mixed_precision.py@@ -103,18 +103,18 @@ class TestMixedPrecisionOps(unittest.TestCase):# Verify weights are wrapped in QuantizedTensorself.assertIsInstance(model.layer1.weight, QuantizedTensor)- self.assertEqual(model.layer1.weight._layout_type, "TensorCoreFP8Layout")+ self.assertEqual(model.layer1.weight._layout_cls, "TensorCoreFP8E4M3Layout")# Layer 2 should NOT be quantizedself.assertNotIsInstance(model.layer2.weight, QuantizedTensor)# Layer 3 should be quantizedself.assertIsInstance(model.layer3.weight, QuantizedTensor)- self.assertEqual(model.layer3.weight._layout_type, "TensorCoreFP8Layout")+ self.assertEqual(model.layer3.weight._layout_cls, "TensorCoreFP8E4M3Layout")# Verify scales were loaded- self.assertEqual(model.layer1.weight._layout_params['scale'].item(), 2.0)- self.assertEqual(model.layer3.weight._layout_params['scale'].item(), 1.5)+ self.assertEqual(model.layer1.weight._params.scale.item(), 2.0)+ self.assertEqual(model.layer3.weight._params.scale.item(), 1.5)# Forward passinput_tensor = torch.randn(5, 10, dtype=torch.bfloat16)@@ -154,8 +154,8 @@ class TestMixedPrecisionOps(unittest.TestCase):# Verify layer1.weight is a QuantizedTensor with scale preservedself.assertIsInstance(state_dict2["layer1.weight"], QuantizedTensor)- self.assertEqual(state_dict2["layer1.weight"]._layout_params['scale'].item(), 3.0)- self.assertEqual(state_dict2["layer1.weight"]._layout_type, "TensorCoreFP8Layout")+ self.assertEqual(state_dict2["layer1.weight"]._params.scale.item(), 3.0)+ self.assertEqual(state_dict2["layer1.weight"]._layout_cls, "TensorCoreFP8E4M3Layout")# Verify non-quantized layers are standard tensorsself.assertNotIsInstance(state_dict2["layer2.weight"], QuantizedTensor)diff --git a/tests-unit/comfy_quant/test_quant_registry.py b/tests-unit/comfy_quant/test_quant_registry.pydeleted file mode 100644index 9cb54ede8..000000000--- a/tests-unit/comfy_quant/test_quant_registry.py+++ /dev/null@@ -1,190 +0,0 @@-import unittest-import torch-import sys-import os--# Add comfy to path-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))--def has_gpu():- return torch.cuda.is_available()--from comfy.cli_args import args-if not has_gpu():- args.cpu = True--from comfy.quant_ops import QuantizedTensor, TensorCoreFP8Layout---class TestQuantizedTensor(unittest.TestCase):- """Test the QuantizedTensor subclass with FP8 layout"""-- def test_creation(self):- """Test creating a QuantizedTensor with TensorCoreFP8Layout"""- fp8_data = torch.randn(256, 128, dtype=torch.float32).to(torch.float8_e4m3fn)- scale = torch.tensor(2.0)- layout_params = {'scale': scale, 'orig_dtype': torch.bfloat16}-- qt = QuantizedTensor(fp8_data, "TensorCoreFP8Layout", layout_params)-- self.assertIsInstance(qt, QuantizedTensor)- self.assertEqual(qt.shape, (256, 128))- self.assertEqual(qt.dtype, torch.float8_e4m3fn)- self.assertEqual(qt._layout_params['scale'], scale)- self.assertEqual(qt._layout_params['orig_dtype'], torch.bfloat16)- self.assertEqual(qt._layout_type, "TensorCoreFP8Layout")-- def test_dequantize(self):- """Test explicit dequantization"""-- fp8_data = torch.ones(10, 20, dtype=torch.float32).to(torch.float8_e4m3fn)- scale = torch.tensor(3.0)- layout_params = {'scale': scale, 'orig_dtype': torch.float32}-- qt = QuantizedTensor(fp8_data, "TensorCoreFP8Layout", layout_params)- dequantized = qt.dequantize()-- self.assertEqual(dequantized.dtype, torch.float32)- self.assertTrue(torch.allclose(dequantized, torch.ones(10, 20) * 3.0, rtol=0.1))-- def test_from_float(self):- """Test creating QuantizedTensor from float tensor"""- float_tensor = torch.randn(64, 32, dtype=torch.float32)- scale = torch.tensor(1.5)-- qt = QuantizedTensor.from_float(- float_tensor,- "TensorCoreFP8Layout",- scale=scale,- dtype=torch.float8_e4m3fn- )-- self.assertIsInstance(qt, QuantizedTensor)- self.assertEqual(qt.dtype, torch.float8_e4m3fn)- self.assertEqual(qt.shape, (64, 32))-- # Verify dequantization gives approximately original values- dequantized = qt.dequantize()- mean_rel_error = ((dequantized - float_tensor).abs() / (float_tensor.abs() + 1e-6)).mean()- self.assertLess(mean_rel_error, 0.1)---class TestGenericUtilities(unittest.TestCase):- """Test generic utility operations"""-- def test_detach(self):- """Test detach operation on quantized tensor"""- fp8_data = torch.randn(10, 20, dtype=torch.float32).to(torch.float8_e4m3fn)- scale = torch.tensor(1.5)- layout_params = {'scale': scale, 'orig_dtype': torch.float32}- qt = QuantizedTensor(fp8_data, "TensorCoreFP8Layout", layout_params)-- # Detach should return a new QuantizedTensor- qt_detached = qt.detach()-- self.assertIsInstance(qt_detached, QuantizedTensor)- self.assertEqual(qt_detached.shape, qt.shape)- self.assertEqual(qt_detached._layout_type, "TensorCoreFP8Layout")-- def test_clone(self):- """Test clone operation on quantized tensor"""- fp8_data = torch.randn(10, 20, dtype=torch.float32).to(torch.float8_e4m3fn)- scale = torch.tensor(1.5)- layout_params = {'scale': scale, 'orig_dtype': torch.float32}- qt = QuantizedTensor(fp8_data, "TensorCoreFP8Layout", layout_params)-- # Clone should return a new QuantizedTensor- qt_cloned = qt.clone()-- self.assertIsInstance(qt_cloned, QuantizedTensor)- self.assertEqual(qt_cloned.shape, qt.shape)- self.assertEqual(qt_cloned._layout_type, "TensorCoreFP8Layout")-- # Verify it's a deep copy- self.assertIsNot(qt_cloned._qdata, qt._qdata)-- @unittest.skipUnless(has_gpu(), "GPU not available")- def test_to_device(self):- """Test device transfer"""- fp8_data = torch.randn(10, 20, dtype=torch.float32).to(torch.float8_e4m3fn)- scale = torch.tensor(1.5)- layout_params = {'scale': scale, 'orig_dtype': torch.float32}- qt = QuantizedTensor(fp8_data, "TensorCoreFP8Layout", layout_params)-- # Moving to same device should work (CPU to CPU)- qt_cpu = qt.to('cpu')-- self.assertIsInstance(qt_cpu, QuantizedTensor)- self.assertEqual(qt_cpu.device.type, 'cpu')- self.assertEqual(qt_cpu._layout_params['scale'].device.type, 'cpu')---class TestTensorCoreFP8Layout(unittest.TestCase):- """Test the TensorCoreFP8Layout implementation"""-- def test_quantize(self):- """Test quantization method"""- float_tensor = torch.randn(32, 64, dtype=torch.float32)- scale = torch.tensor(1.5)-- qdata, layout_params = TensorCoreFP8Layout.quantize(- float_tensor,- scale=scale,- dtype=torch.float8_e4m3fn- )-- self.assertEqual(qdata.dtype, torch.float8_e4m3fn)- self.assertEqual(qdata.shape, float_tensor.shape)- self.assertIn('scale', layout_params)- self.assertIn('orig_dtype', layout_params)- self.assertEqual(layout_params['orig_dtype'], torch.float32)-- def test_dequantize(self):- """Test dequantization method"""- float_tensor = torch.ones(10, 20, dtype=torch.float32) * 3.0- scale = torch.tensor(1.0)-- qdata, layout_params = TensorCoreFP8Layout.quantize(- float_tensor,- scale=scale,- dtype=torch.float8_e4m3fn- )-- dequantized = TensorCoreFP8Layout.dequantize(qdata, **layout_params)-- # Should approximately match original- self.assertTrue(torch.allclose(dequantized, float_tensor, rtol=0.1, atol=0.1))---class TestFallbackMechanism(unittest.TestCase):- """Test fallback for unsupported operations"""-- def test_unsupported_op_dequantizes(self):- """Test that unsupported operations fall back to dequantization"""- # Set seed for reproducibility- torch.manual_seed(42)-- # Create quantized tensor- a_fp32 = torch.randn(10, 20, dtype=torch.float32)- scale = torch.tensor(1.0)- a_q = QuantizedTensor.from_float(- a_fp32,- "TensorCoreFP8Layout",- scale=scale,- dtype=torch.float8_e4m3fn- )-- # Call an operation that doesn't have a registered handler- # For example, torch.abs- result = torch.abs(a_q)-- # Should work via fallback (dequantize → abs → return)- self.assertNotIsInstance(result, QuantizedTensor)- expected = torch.abs(a_fp32)- # FP8 introduces quantization error, so use loose tolerance- mean_error = (result - expected).abs().mean()- self.assertLess(mean_error, 0.05, f"Mean error {mean_error:.4f} is too large")---if __name__ == "__main__":- unittest.main()