The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
upscalers: convert latent_upsampler model to DynamicVram (#15063)
rattusf8a3fd9d7983
diff --git a/comfy/ldm/lightricks/latent_upsampler.py b/comfy/ldm/lightricks/latent_upsampler.pyindex 78ed7653f..6a4beb1bf 100644--- a/comfy/ldm/lightricks/latent_upsampler.py+++ b/comfy/ldm/lightricks/latent_upsampler.py@@ -97,11 +97,11 @@ class SpatialRationalResampler(nn.Module):For dims==3, work per-frame for spatial scaling (temporal axis untouched)."""- def __init__(self, mid_channels: int, scale: float):+ def __init__(self, mid_channels: int, scale: float, operations):super().__init__()self.scale = float(scale)self.num, self.den = _rational_for_scale(self.scale)- self.conv = nn.Conv2d(+ self.conv = operations.Conv2d(mid_channels, (self.num**2) * mid_channels, kernel_size=3, padding=1)self.pixel_shuffle = PixelShuffleND(2, upscale_factors=(self.num, self.num))@@ -119,18 +119,18 @@ class SpatialRationalResampler(nn.Module):class ResBlock(nn.Module):def __init__(- self, channels: int, mid_channels: Optional[int] = None, dims: int = 3+ self, channels: int, operations, mid_channels: Optional[int] = None, dims: int = 3):super().__init__()if mid_channels is None:mid_channels = channels- Conv = nn.Conv2d if dims == 2 else nn.Conv3d+ Conv = operations.Conv2d if dims == 2 else operations.Conv3dself.conv1 = Conv(channels, mid_channels, kernel_size=3, padding=1)- self.norm1 = nn.GroupNorm(32, mid_channels)+ self.norm1 = operations.GroupNorm(32, mid_channels)self.conv2 = Conv(mid_channels, channels, kernel_size=3, padding=1)- self.norm2 = nn.GroupNorm(32, channels)+ self.norm2 = operations.GroupNorm(32, channels)self.activation = nn.SiLU()def forward(self, x: torch.Tensor) -> torch.Tensor:@@ -159,6 +159,7 @@ class LatentUpsampler(nn.Module):def __init__(self,+ operations,in_channels: int = 128,mid_channels: int = 512,num_blocks_per_stage: int = 4,@@ -179,34 +180,34 @@ class LatentUpsampler(nn.Module):self.spatial_scale = float(spatial_scale)self.rational_resampler = rational_resampler- Conv = nn.Conv2d if dims == 2 else nn.Conv3d+ Conv = operations.Conv2d if dims == 2 else operations.Conv3dself.initial_conv = Conv(in_channels, mid_channels, kernel_size=3, padding=1)- self.initial_norm = nn.GroupNorm(32, mid_channels)+ self.initial_norm = operations.GroupNorm(32, mid_channels)self.initial_activation = nn.SiLU()self.res_blocks = nn.ModuleList(- [ResBlock(mid_channels, dims=dims) for _ in range(num_blocks_per_stage)]+ [ResBlock(mid_channels, dims=dims, operations=operations) for _ in range(num_blocks_per_stage)])if spatial_upsample and temporal_upsample:self.upsampler = nn.Sequential(- nn.Conv3d(mid_channels, 8 * mid_channels, kernel_size=3, padding=1),+ operations.Conv3d(mid_channels, 8 * mid_channels, kernel_size=3, padding=1),PixelShuffleND(3),)elif spatial_upsample:if rational_resampler:self.upsampler = SpatialRationalResampler(- mid_channels=mid_channels, scale=self.spatial_scale+ mid_channels=mid_channels, scale=self.spatial_scale, operations=operations)else:self.upsampler = nn.Sequential(- nn.Conv2d(mid_channels, 4 * mid_channels, kernel_size=3, padding=1),+ operations.Conv2d(mid_channels, 4 * mid_channels, kernel_size=3, padding=1),PixelShuffleND(2),)elif temporal_upsample:self.upsampler = nn.Sequential(- nn.Conv3d(mid_channels, 2 * mid_channels, kernel_size=3, padding=1),+ operations.Conv3d(mid_channels, 2 * mid_channels, kernel_size=3, padding=1),PixelShuffleND(1),)else:@@ -215,11 +216,14 @@ class LatentUpsampler(nn.Module):)self.post_upsample_res_blocks = nn.ModuleList(- [ResBlock(mid_channels, dims=dims) for _ in range(num_blocks_per_stage)]+ [ResBlock(mid_channels, dims=dims, operations=operations) for _ in range(num_blocks_per_stage)])self.final_conv = Conv(mid_channels, in_channels, kernel_size=3, padding=1)+ def get_dtype(self):+ return getattr(self.initial_conv, "weight_comfy_model_dtype", self.initial_conv.weight.dtype)+def forward(self, latent: torch.Tensor) -> torch.Tensor:b, c, f, h, w = latent.shape@@ -266,7 +270,7 @@ class LatentUpsampler(nn.Module):return x@classmethod- def from_config(cls, config):+ def from_config(cls, config, operations):return cls(in_channels=config.get("in_channels", 4),mid_channels=config.get("mid_channels", 128),@@ -276,6 +280,7 @@ class LatentUpsampler(nn.Module):temporal_upsample=config.get("temporal_upsample", False),spatial_scale=config.get("spatial_scale", 2.0),rational_resampler=config.get("rational_resampler", False),+ operations=operations,)def config(self):diff --git a/comfy_extras/nodes_hunyuan.py b/comfy_extras/nodes_hunyuan.pyindex 8df2c8908..ce2997245 100644--- a/comfy_extras/nodes_hunyuan.py+++ b/comfy_extras/nodes_hunyuan.py@@ -2,6 +2,8 @@ import nodesimport node_helpersimport torchimport comfy.model_management+import comfy.model_patcher+import comfy.opsfrom typing_extensions import overridefrom comfy_api.latest import ComfyExtension, iofrom comfy.ldm.hunyuan_video.upsampler import HunyuanVideo15SRModel@@ -217,8 +219,11 @@ class LatentUpscaleModelLoader(io.ComfyNode):model.load_sd(sd)elif "post_upsample_res_blocks.0.conv2.bias" in sd:config = json.loads(metadata["config"])- model = LatentUpsampler.from_config(config).to(dtype=comfy.model_management.vae_dtype(allowed_dtypes=[torch.bfloat16, torch.float32]))- model.load_state_dict(sd)+ model = LatentUpsampler.from_config(config, operations=comfy.ops.disable_weight_init).to(dtype=comfy.model_management.vae_dtype(allowed_dtypes=[torch.bfloat16, torch.float32]))+ comfy.model_management.archive_model_dtypes(model)+ model_patcher = comfy.model_patcher.CoreModelPatcher(model, load_device=comfy.model_management.get_torch_device(), offload_device=comfy.model_management.unet_offload_device())+ model.load_state_dict(sd, assign=model_patcher.is_dynamic())+ model = model_patcherreturn io.NodeOutput(model)diff --git a/comfy_extras/nodes_lt_upsampler.py b/comfy_extras/nodes_lt_upsampler.pyindex ef36109d1..7e7975495 100644--- a/comfy_extras/nodes_lt_upsampler.py+++ b/comfy_extras/nodes_lt_upsampler.py@@ -38,26 +38,20 @@ class LTXVLatentUpsampler(IO.ComfyNode):Returns:tuple: Tuple containing the upsampled latent"""- device = model_management.get_torch_device()- memory_required = model_management.module_size(upscale_model)-- model_dtype = next(upscale_model.parameters()).dtype+ device = upscale_model.load_device+ model = upscale_model.model+ model_dtype = upscale_model.model_dtype()latents = samples["samples"]input_dtype = latents.dtype- memory_required += math.prod(latents.shape) * 3000.0 # TODO: more accurate- model_management.free_memory(memory_required, device)-- try:- upscale_model.to(device) # TODO: use the comfy model management system.+ memory_required = math.prod(latents.shape) * 3000.0 # TODO: more accurate+ model_management.load_models_gpu([upscale_model], memory_required=memory_required)- latents = latents.to(dtype=model_dtype, device=device)+ latents = latents.to(dtype=model_dtype, device=device)- """Upsample latents without tiling."""- latents = vae.first_stage_model.per_channel_statistics.un_normalize(latents)- upsampled_latents = upscale_model(latents)- finally:- upscale_model.cpu()+ """Upsample latents without tiling."""+ latents = vae.first_stage_model.per_channel_statistics.un_normalize(latents)+ upsampled_latents = model(latents)upsampled_latents = vae.first_stage_model.per_channel_statistics.normalize(upsampled_latentsdiff --git a/comfy_extras/nodes_upscale_model.py b/comfy_extras/nodes_upscale_model.pyindex 1cf5a5d01..a4d692955 100644--- a/comfy_extras/nodes_upscale_model.py+++ b/comfy_extras/nodes_upscale_model.py@@ -7,6 +7,7 @@ import folder_pathsfrom typing_extensions import overridefrom comfy_api.latest import ComfyExtension, ioimport comfy.model_management+import comfy.model_patchertry:from spandrel_extra_arches import EXTRA_REGISTRY@@ -42,6 +43,7 @@ class UpscaleModelLoader(io.ComfyNode):if not isinstance(out, ImageModelDescriptor):raise Exception("Upscale model must be a single-image model.")+ out.patcher = comfy.model_patcher.CoreModelPatcher(out.model, load_device=model_management.get_torch_device(), offload_device=model_management.unet_offload_device())return io.NodeOutput(out)load_model = execute # TODO: remove@@ -66,14 +68,12 @@ class ImageUpscaleWithModel(io.ComfyNode):@classmethoddef execute(cls, upscale_model, image) -> io.NodeOutput:- device = model_management.get_torch_device()+ device = upscale_model.patcher.load_device- memory_required = model_management.module_size(upscale_model.model)- memory_required += (512 * 512 * 3) * image.element_size() * max(upscale_model.scale, 1.0) * 384.0 #The 384.0 is an estimate of how much some of these models take, TODO: make it more accurate+ memory_required = (512 * 512 * 3) * image.element_size() * max(upscale_model.scale, 1.0) * 384.0 #The 384.0 is an estimate of how much some of these models take, TODO: make it more accuratememory_required += image.nelement() * image.element_size()- model_management.free_memory(memory_required, device)+ model_management.load_models_gpu([upscale_model.patcher], memory_required=memory_required)- upscale_model.to(device)in_img = image.movedim(-1,-3).to(device)tile = 512@@ -82,20 +82,17 @@ class ImageUpscaleWithModel(io.ComfyNode):output_device = comfy.model_management.intermediate_device()oom = True- try:- while oom:- try:- steps = in_img.shape[0] * comfy.utils.get_tiled_scale_steps(in_img.shape[3], in_img.shape[2], tile_x=tile, tile_y=tile, overlap=overlap)- pbar = comfy.utils.ProgressBar(steps)- s = comfy.utils.tiled_scale(in_img, lambda a: upscale_model(a.float()), tile_x=tile, tile_y=tile, overlap=overlap, upscale_amount=upscale_model.scale, pbar=pbar, output_device=output_device)- oom = False- except Exception as e:- model_management.raise_non_oom(e)- tile //= 2- if tile < 128:- raise e- finally:- upscale_model.to("cpu")+ while oom:+ try:+ steps = in_img.shape[0] * comfy.utils.get_tiled_scale_steps(in_img.shape[3], in_img.shape[2], tile_x=tile, tile_y=tile, overlap=overlap)+ pbar = comfy.utils.ProgressBar(steps)+ s = comfy.utils.tiled_scale(in_img, lambda a: upscale_model(a.float()), tile_x=tile, tile_y=tile, overlap=overlap, upscale_amount=upscale_model.scale, pbar=pbar, output_device=output_device)+ oom = False+ except Exception as e:+ model_management.raise_non_oom(e)+ tile //= 2+ if tile < 128:+ raise es = torch.clamp(s.movedim(-3,-1), min=0, max=1.0).to(comfy.model_management.intermediate_dtype())return io.NodeOutput(s)