Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

ops: implement lora requanting for non QuantizedTensor fp8 (#12668)

rattuse721e24136b5
diff --git a/comfy/ops.py b/comfy/ops.py
index 98fec1e1d..6ee6075fb 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -167,17 +167,15 @@ def cast_bias_weight_with_vbar(s, dtype, device, bias_dtype, non_blocking, compu
x = to_dequant(x, dtype)
if not resident and lowvram_fn is not None:
x = to_dequant(x, dtype if compute_dtype is None else compute_dtype)
- #FIXME: this is not accurate, we need to be sensitive to the compute dtype
x = lowvram_fn(x)
- if (isinstance(orig, QuantizedTensor) and
- (want_requant and len(fns) == 0 or update_weight)):
+ if (want_requant and len(fns) == 0 or update_weight):
seed = comfy.utils.string_to_seed(s.seed_key)
- y = QuantizedTensor.from_float(x, s.layout_type, scale="recalculate", stochastic_rounding=seed)
- if want_requant and len(fns) == 0:
- #The layer actually wants our freshly saved QT
- x = y
- elif update_weight:
- y = comfy.float.stochastic_rounding(x, orig.dtype, seed = comfy.utils.string_to_seed(s.seed_key))
+ if isinstance(orig, QuantizedTensor):
+ y = QuantizedTensor.from_float(x, s.layout_type, scale="recalculate", stochastic_rounding=seed)
+ else:
+ y = comfy.float.stochastic_rounding(x, orig.dtype, seed=seed)
+ if want_requant and len(fns) == 0:
+ x = y
if update_weight:
orig.copy_(y)
for f in fns:
@@ -617,7 +615,8 @@ def fp8_linear(self, input):
if input.ndim != 2:
return None
- w, bias, offload_stream = cast_bias_weight(self, input, dtype=dtype, bias_dtype=input_dtype, offloadable=True)
+ lora_compute_dtype=comfy.model_management.lora_compute_dtype(input.device)
+ w, bias, offload_stream = cast_bias_weight(self, input, dtype=dtype, bias_dtype=input_dtype, offloadable=True, compute_dtype=lora_compute_dtype, want_requant=True)
scale_weight = torch.ones((), device=input.device, dtype=torch.float32)
scale_input = torch.ones((), device=input.device, dtype=torch.float32)