The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
--gpu-only now keeps the VAE on the device.
comfyanonymous1c1b0e7299fa
diff --git a/comfy/model_management.py b/comfy/model_management.pyindex ecbcabb0a..e44c9e8a5 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -349,6 +349,15 @@ def text_encoder_device():else:return torch.device("cpu")+def vae_device():+ return get_torch_device()++def vae_offload_device():+ if args.gpu_only or vram_state == VRAMState.SHARED:+ return get_torch_device()+ else:+ return torch.device("cpu")+def get_autocast_device(dev):if hasattr(dev, 'type'):return dev.typediff --git a/comfy/sd.py b/comfy/sd.pyindex 08d68c5f8..3d79c7c04 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -605,8 +605,9 @@ class VAE:self.first_stage_model.load_state_dict(sd, strict=False)if device is None:- device = model_management.get_torch_device()+ device = model_management.vae_device()self.device = device+ self.offload_device = model_management.vae_offload_device()def decode_tiled_(self, samples, tile_x=64, tile_y=64, overlap = 16):steps = samples.shape[0] * utils.get_tiled_scale_steps(samples.shape[3], samples.shape[2], tile_x, tile_y, overlap)@@ -651,7 +652,7 @@ class VAE:print("Warning: Ran out of memory when regular VAE decoding, retrying with tiled VAE decoding.")pixel_samples = self.decode_tiled_(samples_in)- self.first_stage_model = self.first_stage_model.cpu()+ self.first_stage_model = self.first_stage_model.to(self.offload_device)pixel_samples = pixel_samples.cpu().movedim(1,-1)return pixel_samples@@ -659,7 +660,7 @@ class VAE:model_management.unload_model()self.first_stage_model = self.first_stage_model.to(self.device)output = self.decode_tiled_(samples, tile_x, tile_y, overlap)- self.first_stage_model = self.first_stage_model.cpu()+ self.first_stage_model = self.first_stage_model.to(self.offload_device)return output.movedim(1,-1)def encode(self, pixel_samples):@@ -679,7 +680,7 @@ class VAE:print("Warning: Ran out of memory when regular VAE encoding, retrying with tiled VAE encoding.")samples = self.encode_tiled_(pixel_samples)- self.first_stage_model = self.first_stage_model.cpu()+ self.first_stage_model = self.first_stage_model.to(self.offload_device)return samplesdef encode_tiled(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64):@@ -687,7 +688,7 @@ class VAE:self.first_stage_model = self.first_stage_model.to(self.device)pixel_samples = pixel_samples.movedim(-1,1)samples = self.encode_tiled_(pixel_samples, tile_x=tile_x, tile_y=tile_y, overlap=overlap)- self.first_stage_model = self.first_stage_model.cpu()+ self.first_stage_model = self.first_stage_model.to(self.offload_device)return samplesdef get_sd(self):