The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add --fp16-intermediates to use fp16 for intermediate values between nodes (#12953)
comfyanonymousc711b8f43792
diff --git a/comfy/cli_args.py b/comfy/cli_args.pyindex e9832acaf..0a0bf2f30 100644--- a/comfy/cli_args.py+++ b/comfy/cli_args.py@@ -83,6 +83,8 @@ fpte_group.add_argument("--fp16-text-enc", action="store_true", help="Store textfpte_group.add_argument("--fp32-text-enc", action="store_true", help="Store text encoder weights in fp32.")fpte_group.add_argument("--bf16-text-enc", action="store_true", help="Store text encoder weights in bf16.")+parser.add_argument("--fp16-intermediates", action="store_true", help="Experimental: Use fp16 for intermediate tensors between nodes instead of fp32.")+parser.add_argument("--force-channels-last", action="store_true", help="Force channels last format when inferencing the models.")parser.add_argument("--directml", type=int, nargs="?", metavar="DIRECTML_DEVICE", const=-1, help="Use torch-directml.")diff --git a/comfy/model_management.py b/comfy/model_management.pyindex bb77cff47..442d5a40a 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -1050,6 +1050,12 @@ def intermediate_device():else:return torch.device("cpu")+def intermediate_dtype():+ if args.fp16_intermediates:+ return torch.float16+ else:+ return torch.float32+def vae_device():if args.cpu_vae:return torch.device("cpu")diff --git a/comfy/sd.py b/comfy/sd.pyindex adcd67767..4d427bb9a 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -871,13 +871,16 @@ class VAE:pixels = torch.nn.functional.pad(pixels, (0, self.output_channels - pixels.shape[-1]), mode=mode, value=value)return pixels+ def vae_output_dtype(self):+ return model_management.intermediate_dtype()+def decode_tiled_(self, samples, tile_x=64, tile_y=64, overlap = 16):steps = samples.shape[0] * comfy.utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x, tile_y, overlap)steps += samples.shape[0] * comfy.utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x // 2, tile_y * 2, overlap)steps += samples.shape[0] * comfy.utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x * 2, tile_y // 2, overlap)pbar = comfy.utils.ProgressBar(steps)- decode_fn = lambda a: self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)).float()+ decode_fn = lambda a: self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype())output = self.process_output((comfy.utils.tiled_scale(samples, decode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = self.upscale_ratio, output_device=self.output_device, pbar = pbar) +comfy.utils.tiled_scale(samples, decode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = self.upscale_ratio, output_device=self.output_device, pbar = pbar) +@@ -887,16 +890,16 @@ class VAE:def decode_tiled_1d(self, samples, tile_x=256, overlap=32):if samples.ndim == 3:- decode_fn = lambda a: self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)).float()+ decode_fn = lambda a: self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype())else:og_shape = samples.shapesamples = samples.reshape((og_shape[0], og_shape[1] * og_shape[2], -1))- decode_fn = lambda a: self.first_stage_model.decode(a.reshape((-1, og_shape[1], og_shape[2], a.shape[-1])).to(self.vae_dtype).to(self.device)).float()+ decode_fn = lambda a: self.first_stage_model.decode(a.reshape((-1, og_shape[1], og_shape[2], a.shape[-1])).to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype())return self.process_output(comfy.utils.tiled_scale_multidim(samples, decode_fn, tile=(tile_x,), overlap=overlap, upscale_amount=self.upscale_ratio, out_channels=self.output_channels, output_device=self.output_device))def decode_tiled_3d(self, samples, tile_t=999, tile_x=32, tile_y=32, overlap=(1, 8, 8)):- decode_fn = lambda a: self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)).float()+ decode_fn = lambda a: self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype())return self.process_output(comfy.utils.tiled_scale_multidim(samples, decode_fn, tile=(tile_t, tile_x, tile_y), overlap=overlap, upscale_amount=self.upscale_ratio, out_channels=self.output_channels, index_formulas=self.upscale_index_formula, output_device=self.output_device))def encode_tiled_(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64):@@ -905,7 +908,7 @@ class VAE:steps += pixel_samples.shape[0] * comfy.utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x * 2, tile_y // 2, overlap)pbar = comfy.utils.ProgressBar(steps)- encode_fn = lambda a: self.first_stage_model.encode((self.process_input(a)).to(self.vae_dtype).to(self.device)).float()+ encode_fn = lambda a: self.first_stage_model.encode((self.process_input(a)).to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype())samples = comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x, tile_y, overlap, upscale_amount = (1/self.downscale_ratio), out_channels=self.latent_channels, output_device=self.output_device, pbar=pbar)samples += comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x * 2, tile_y // 2, overlap, upscale_amount = (1/self.downscale_ratio), out_channels=self.latent_channels, output_device=self.output_device, pbar=pbar)samples += comfy.utils.tiled_scale(pixel_samples, encode_fn, tile_x // 2, tile_y * 2, overlap, upscale_amount = (1/self.downscale_ratio), out_channels=self.latent_channels, output_device=self.output_device, pbar=pbar)@@ -914,7 +917,7 @@ class VAE:def encode_tiled_1d(self, samples, tile_x=256 * 2048, overlap=64 * 2048):if self.latent_dim == 1:- encode_fn = lambda a: self.first_stage_model.encode((self.process_input(a)).to(self.vae_dtype).to(self.device)).float()+ encode_fn = lambda a: self.first_stage_model.encode((self.process_input(a)).to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype())out_channels = self.latent_channelsupscale_amount = 1 / self.downscale_ratioelse:@@ -923,7 +926,7 @@ class VAE:tile_x = tile_x // extra_channel_sizeoverlap = overlap // extra_channel_sizeupscale_amount = 1 / self.downscale_ratio- encode_fn = lambda a: self.first_stage_model.encode((self.process_input(a)).to(self.vae_dtype).to(self.device)).reshape(1, out_channels, -1).float()+ encode_fn = lambda a: self.first_stage_model.encode((self.process_input(a)).to(self.vae_dtype).to(self.device)).reshape(1, out_channels, -1).to(dtype=self.vae_output_dtype())out = comfy.utils.tiled_scale_multidim(samples, encode_fn, tile=(tile_x,), overlap=overlap, upscale_amount=upscale_amount, out_channels=out_channels, output_device=self.output_device)if self.latent_dim == 1:@@ -932,7 +935,7 @@ class VAE:return out.reshape(samples.shape[0], self.latent_channels, extra_channel_size, -1)def encode_tiled_3d(self, samples, tile_t=9999, tile_x=512, tile_y=512, overlap=(1, 64, 64)):- encode_fn = lambda a: self.first_stage_model.encode((self.process_input(a)).to(self.vae_dtype).to(self.device)).float()+ encode_fn = lambda a: self.first_stage_model.encode((self.process_input(a)).to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype())return comfy.utils.tiled_scale_multidim(samples, encode_fn, tile=(tile_t, tile_x, tile_y), overlap=overlap, upscale_amount=self.downscale_ratio, out_channels=self.latent_channels, downscale=True, index_formulas=self.downscale_index_formula, output_device=self.output_device)def decode(self, samples_in, vae_options={}):@@ -950,9 +953,9 @@ class VAE:for x in range(0, samples_in.shape[0], batch_number):samples = samples_in[x:x+batch_number].to(self.vae_dtype).to(self.device)- out = self.process_output(self.first_stage_model.decode(samples, **vae_options).to(self.output_device).float())+ out = self.process_output(self.first_stage_model.decode(samples, **vae_options).to(self.output_device).to(dtype=self.vae_output_dtype()))if pixel_samples is None:- pixel_samples = torch.empty((samples_in.shape[0],) + tuple(out.shape[1:]), device=self.output_device)+ pixel_samples = torch.empty((samples_in.shape[0],) + tuple(out.shape[1:]), device=self.output_device, dtype=self.vae_output_dtype())pixel_samples[x:x+batch_number] = outexcept Exception as e:model_management.raise_non_oom(e)@@ -1025,9 +1028,9 @@ class VAE:samples = Nonefor x in range(0, pixel_samples.shape[0], batch_number):pixels_in = self.process_input(pixel_samples[x:x + batch_number]).to(self.vae_dtype).to(self.device)- out = self.first_stage_model.encode(pixels_in).to(self.output_device).float()+ out = self.first_stage_model.encode(pixels_in).to(self.output_device).to(dtype=self.vae_output_dtype())if samples is None:- samples = torch.empty((pixel_samples.shape[0],) + tuple(out.shape[1:]), device=self.output_device)+ samples = torch.empty((pixel_samples.shape[0],) + tuple(out.shape[1:]), device=self.output_device, dtype=self.vae_output_dtype())samples[x:x + batch_number] = outexcept Exception as e:diff --git a/nodes.py b/nodes.pyindex eb63f9d44..1e19a8223 100644--- a/nodes.py+++ b/nodes.py@@ -1724,6 +1724,8 @@ class LoadImage:output_masks = []w, h = None, None+ dtype = comfy.model_management.intermediate_dtype()+for i in ImageSequence.Iterator(img):i = node_helpers.pillow(ImageOps.exif_transpose, i)@@ -1748,8 +1750,8 @@ class LoadImage:mask = 1. - torch.from_numpy(mask)else:mask = torch.zeros((64,64), dtype=torch.float32, device="cpu")- output_images.append(image)- output_masks.append(mask.unsqueeze(0))+ output_images.append(image.to(dtype=dtype))+ output_masks.append(mask.unsqueeze(0).to(dtype=dtype))if img.format == "MPO":break # ignore all frames except the first one for MPO format