The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support RelayVSR (CORE-701) (#17511)
Jukka Seppänen2f8943324e1e
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.pyindex 0e7fba912..1367c5214 100644--- a/comfy/latent_formats.py+++ b/comfy/latent_formats.py@@ -880,6 +880,14 @@ class Wan22(Wan21):0.3971, 1.0600, 0.3943, 0.5537, 0.5444, 0.4089, 0.7468, 0.7744]).view(1, self.latent_channels, 1, 1, 1)+class RelayVSR(Wan22):+ # the keyframe Wan and the Flash decoder both work on the normalized latent+ def process_in(self, latent):+ return latent++ def process_out(self, latent):+ return latent+class HunyuanImage21(LatentFormat):latent_channels = 64latent_dimensions = 2diff --git a/comfy/ldm/seedvr/vae.py b/comfy/ldm/seedvr/vae.pyindex 4437caee4..c73ee1c97 100644--- a/comfy/ldm/seedvr/vae.py+++ b/comfy/ldm/seedvr/vae.py@@ -233,7 +233,7 @@ class CausalMemoryCache:"""The one stream this cache copies on, so copies reusing a host buffer stay ordered;it starts after the compute queued so far. None copies on the compute stream."""if self._stream is None:- self._stream = comfy.model_management.get_offload_stream(device)+ self._stream = comfy.model_management.get_data_stream(device)if self._stream is not None:self._stream.wait_stream(comfy.model_management.current_stream(device))return self._streamdiff --git a/comfy/ldm/wan/model_animate2.py b/comfy/ldm/wan/model_animate2.pyindex 45629af4d..744401ecb 100644--- a/comfy/ldm/wan/model_animate2.py+++ b/comfy/ldm/wan/model_animate2.py@@ -210,7 +210,7 @@ class PoseBranchCache:with comfy.model_prefetch.pause_malloc_graph():self._staging[buf_key] = [torch.empty(t.shape, dtype=buf_key[1], device=device) for _ in range(2)]r = self._staging[buf_key][i % 2]- stream = comfy.model_management.get_offload_stream(device)+ stream = comfy.model_management.get_data_stream(device)cs = comfy.model_management.current_stream(device)if stream is not None and cs is not None:# Wait for staging allocation and the previous consumer's reads before overwriting the buffer.diff --git a/comfy/ldm/wan/model_vsr.py b/comfy/ldm/wan/model_vsr.pynew file mode 100644index 000000000..9fd4a72bc--- /dev/null+++ b/comfy/ldm/wan/model_vsr.py@@ -0,0 +1,372 @@+# Wan video super-resolution+# RelayVSR https://github.com/kopperx/RelayVSR (Apache-2.0): one-step Wan keyframes guiding a small per-frame transformer+import math++import comfy_kitchen+import torch+import torch.nn as nn+import torch.nn.functional as F++from comfy.ldm.flux.layers import EmbedND+from comfy.ldm.flux.math import apply_rope1+from comfy.ldm.wan.model import WanAttentionBlock, WanModel, WanSelfAttention, modulate, sinusoidal_embedding_1d+import comfy.ldm.common_dit+import comfy.model_management+import comfy.model_prefetch+import comfy.ops+import comfy.utils++ops = comfy.ops.disable_weight_init++RELAY_KEYFRAME_INTERVAL = 15+RELAY_WAN_CONTEXT = 4+RELAY_WAN_WINDOW = (22, 40)+RELAY_WAN_CHUNK_TOKENS = 32768+# the per-layer cache needs about as much VRAM as this many keyframes of a single pass+RELAY_WAN_CACHE_KEYFRAMES = 7+RELAY_FLASH_CONDITION_WINDOW = (48, 48)+RELAY_FLASH_VIDEO_WINDOW = (24, 24)+++def relay_keyframes(frames):+ return list(range(0, frames - 1, RELAY_KEYFRAME_INTERVAL)) + [frames - 1]+++def relay_latent_size(height, width, latent_stride):+ # whole 2x2 patches of the model's pixel stride+ stride = latent_stride * 2+ return math.ceil(height / stride) * 2, math.ceil(width / stride) * 2+++class RelayLQProjection(nn.Module):+ def __init__(self, upscale_factor, hidden_dim=768, hidden_dim2=3072, out_dim=3072, eps=1e-6, device=None, dtype=None, operations=None):+ super().__init__()+ self.upscale_factor = upscale_factor+ self.conv1 = operations.Conv2d(3 * (4 * upscale_factor) ** 2, hidden_dim, 1, device=device, dtype=dtype)+ self.norm1 = operations.RMSNorm(hidden_dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)+ self.conv2 = operations.Conv2d(hidden_dim, hidden_dim2, 3, stride=2, padding=1, device=device, dtype=dtype)+ self.norm2 = operations.RMSNorm(hidden_dim2, eps=eps, elementwise_affine=True, device=device, dtype=dtype)+ self.linear = operations.Linear(hidden_dim2, out_dim, device=device, dtype=dtype)++ def forward(self, video):+ b, c, f, h, w = video.shape+ x = video.movedim(2, 1).reshape(b * f, c, h, w)+ if self.upscale_factor == 4:+ x = F.interpolate(x, scale_factor=4, mode="bilinear", align_corners=False)+ x = self.conv1(F.pixel_unshuffle(x, 4 * self.upscale_factor))+ x = F.silu(self.norm1(x.movedim(1, -1))).movedim(-1, 1)+ x = F.silu(self.norm2(self.conv2(x).flatten(2).transpose(1, 2)))+ return self.linear(x).view(b, -1, self.linear.out_features)+++class RelayVSRSelfAttention(WanSelfAttention):+ def forward(self, x, freqs, cache, start, store, transformer_options={}):+ # causal over the last keyframes, per-query windows slid to stay inside the frame+ b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim+ frames, height, width = transformer_options["grid_sizes"]+ grid = (b, frames, height, width, n, d)+ q = apply_rope1(self.norm_q(self.q(x)).view(b, s, n, d), freqs).view(grid)+ k = apply_rope1(self.norm_k(self.k(x)).view(b, s, n, d), freqs).view(grid)+ v = self.v(x).view(grid)+ # keyframes before this chunk come from the cache, which holds the last ones in order as int8 rows with a scale each+ past = min(start, RELAY_WAN_CONTEXT - 1)+ keys, values = k, v+ if past > 0:+ cached, scales = cache+ k_past, v_past = (cached[:, :, :past].to(k.dtype) * scales[:, :, :past]).unbind(0)+ keys, values = torch.cat((k_past, k), dim=1), torch.cat((v_past, v), dim=1)+ x = comfy_kitchen.window_attention(q, keys, values, RELAY_WAN_WINDOW, temporal_window=RELAY_WAN_CONTEXT)+ if store:+ # the cache moves on to the last keyframes for the next chunk, older cached ones first when this one is short+ cached, scales = cache+ new = min(frames, RELAY_WAN_CONTEXT - 1)+ old = min(past, RELAY_WAN_CONTEXT - 1 - new)+ if old > 0:+ cached[:, :, :old] = cached[:, :, past - old:past].clone()+ scales[:, :, :old] = scales[:, :, past - old:past].clone()+ for i, rows in enumerate((k[:, frames - new:], v[:, frames - new:])):+ scale = (rows.abs().amax(dim=-1, keepdim=True) / 127.0).clamp_(min=torch.finfo(rows.dtype).tiny)+ scales[i, :, old:old + new] = scale+ cached[i, :, old:old + new] = (rows.float() / scale.float()).round_().clamp_(-127.0, 127.0)+ return self.o(x.view(b, s, n * d))+++class RelayVSRAttentionBlock(WanAttentionBlock):+ def forward(self, x, e, freqs, context, cache, start, store, transformer_options={}):+ # residual updated in place, nothing allocated in the block outlives its memory compiler scope+ e = (comfy.model_management.cast_to(self.modulation, dtype=x.dtype, device=x.device).unsqueeze(0) + e).unbind(2)+ x.addcmul_(self.self_attn(modulate(x, self.norm1, e[0], e[1]), freqs, cache, start, store, transformer_options=transformer_options), e[2])+ x.add_(self.cross_attn(self.norm3(x), context, transformer_options=transformer_options))+ x.addcmul_(self.ffn(modulate(x, self.norm2, e[3], e[4])), e[5])+ return x+++class RelayVSRWanModel(WanModel):+ def __init__(self, model_type=None, upscale_factor=4, operations=None, device=None, dtype=None, **kwargs):+ super().__init__(model_type="t2v", wan_attn_block_class=RelayVSRAttentionBlock, operations=operations, device=device, dtype=dtype, **kwargs)+ self.register_buffer("positive_conditioning", torch.empty((self.text_len, self.text_dim), device=device, dtype=dtype))+ operation_settings = {"operations": operations, "device": device, "dtype": dtype}+ for block in self.blocks:+ block.self_attn = RelayVSRSelfAttention(self.dim, self.num_heads, eps=self.eps, operation_settings=operation_settings)+ self.lq_proj_in = RelayLQProjection(upscale_factor, out_dim=self.dim, device=device, dtype=dtype, operations=operations)+ self.latent_stride = 16 // upscale_factor++ def forward_orig(self, x, t, context, freqs=None, lq_pixels=None, transformer_options={}, **kwargs):+ b, frames = x.shape[0], x.shape[2]+ height, width = x.shape[3] // self.patch_size[1], x.shape[4] // self.patch_size[2]+ tokens = height * width+ e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t.flatten()).to(dtype=x.dtype))+ e = e.reshape(t.shape[0], -1, e.shape[-1])+ e0 = self.time_projection(e).unflatten(2, (6, self.dim))+ context = self.text_embedding(context)++ # keyframes go through in chunks, every layer's cache carries the keys and values of the last ones across+ chunk = max(1, RELAY_WAN_CHUNK_TOKENS // tokens)+ if frames <= chunk + RELAY_WAN_CACHE_KEYFRAMES:+ chunk = frames+ shape = (2, b, RELAY_WAN_CONTEXT - 1, height, width, self.num_heads, self.dim // self.num_heads)+ caches = [(torch.empty(shape, device=x.device, dtype=torch.int8), torch.empty((*shape[:-1], 1), device=x.device, dtype=x.dtype)) if frames > chunk else None for _ in self.blocks]+ out = torch.empty((b, self.out_dim, *x.shape[2:]), device=x.device, dtype=x.dtype)+ blocks_replace = transformer_options.get("patches_replace", {}).get("dit", {})+ transformer_options["total_blocks"] = len(self.blocks)+ transformer_options["block_type"] = "double"+ starts = range(0, frames, chunk)+ # the sampler's bars only count the one step+ progress = comfy.utils.ProgressBar(len(starts) * len(self.blocks))+ with comfy.utils.model_trange(len(starts) * len(self.blocks), desc="RelayVSR", unit="block") as pbar:+ for start in starts:+ end = min(start + chunk, frames)+ # only a following chunk reads the cache+ store = end < frames+ grid_sizes = (end - start, height, width)+ transformer_options["grid_sizes"] = grid_sizes+ # the projected LQ frames go first so the sum is row-major, the blocks update it in place+ hidden = self.lq_proj_in(lq_pixels[:, :, start:end]) + self.patch_embedding(x[:, :, start:end].float()).to(x.dtype).flatten(2).transpose(1, 2)+ chunk_freqs = freqs[:, start * tokens:end * tokens]+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.blocks), x.device, transformer_options)+ comfy.model_prefetch.malloc_graph_begin(x.device)+ for i, block in enumerate(self.blocks):+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, x.device, block, x.dtype, malloc_scope="block")+ transformer_options["block_index"] = i+ if ("double_block", i) in blocks_replace:+ def block_wrap(args):+ return {"img": block(args["img"], args["vec"], args["pe"], args["txt"], caches[i], start, store, transformer_options=args["transformer_options"])}+ hidden = blocks_replace[("double_block", i)]({"img": hidden, "txt": context, "vec": e0, "pe": chunk_freqs, "transformer_options": transformer_options}, {"original_block": block_wrap})["img"]+ else:+ hidden = block(hidden, e0, chunk_freqs, context, caches[i], start, store, transformer_options=transformer_options)+ pbar.update(1)+ progress.update(1)+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, x.device, None, malloc_scope="block")+ comfy.model_prefetch.malloc_graph_end()+ out[:, :, start:end] = self.unpatchify(self.head(hidden, e), grid_sizes)+ return out++ def _forward(self, x, timestep, context, clip_fea=None, time_dim_concat=None, transformer_options={}, lq_pixels=None, **kwargs):+ if lq_pixels is None:+ raise ValueError("RelayVSR needs the low-quality video, connect the model through the RelayVSR Conditioning node.")+ t, h, w = x.shape[2:]+ x = comfy.ldm.common_dit.pad_to_patch_size(x, self.patch_size, padding_mode="replicate")+ tp, hp, wp = x.shape[2:]+ lq_height, lq_width = hp * self.latent_stride, wp * self.latent_stride+ lq_pixels = lq_pixels[..., :lq_height, :lq_width]+ lq_pixels = F.pad(lq_pixels, (0, lq_width - lq_pixels.shape[-1], 0, lq_height - lq_pixels.shape[-2], 0, 0), mode="replicate")+ freqs = self.rope_encode(tp, hp, wp, device=x.device, dtype=x.dtype, transformer_options=transformer_options)+ return self.forward_orig(x, timestep, context, freqs=freqs, lq_pixels=lq_pixels, transformer_options=transformer_options, **kwargs)[:, :, :t, :h, :w]+++class RelayFlashAttention(nn.Module):+ def __init__(self, dim, num_heads, kv_groups, eps, device=None, dtype=None):+ super().__init__()+ self.num_heads = num_heads+ self.kv_groups = kv_groups+ self.head_dim = dim // num_heads+ self.query = ops.Linear(dim, dim, device=device, dtype=dtype)+ self.key = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype)+ self.value = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype)+ self.condition_key = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype)+ self.condition_value = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype)+ self.key_norm = ops.RMSNorm(self.head_dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)+ self.value_norm = ops.RMSNorm(self.head_dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)+ self.output = ops.Linear(dim, dim, device=device, dtype=dtype)++ def project(self, x, key, value, freqs):+ b, s, _ = x.shape+ q = apply_rope1(self.query(x).view(b, s, self.num_heads, self.head_dim), freqs)+ k = self.key_norm(key(x).view(b, s, self.kv_groups, self.head_dim))+ v = self.value_norm(value(x).view(b, s, self.kv_groups, self.head_dim))+ return q, k, v++ def condition(self, x, freqs, height, width):+ # the rotated condition keys are reused as they are by every frame of the window+ b, s, c = x.shape+ q, k, v = self.project(x, self.condition_key, self.condition_value, freqs)+ k = apply_rope1(k, freqs).view(b, 1, height, width, self.kv_groups, self.head_dim)+ v = v.view(k.shape)+ x = comfy_kitchen.window_attention(q.view(b, 1, height, width, self.num_heads, self.head_dim), k, v, RELAY_FLASH_CONDITION_WINDOW, clamp=False)+ return self.output(x.view(b, s, c)), (k, v)++ def video(self, x, freqs, freqs_past, height, width, condition, past):+ b, s, c = x.shape+ plane = (b, 1, height, width, self.kv_groups, self.head_dim)+ q, k, v = self.project(x, self.key, self.value, freqs)+ # both condition keyframes, then the previous frame and this one+ (k_start, v_start), (k_end, v_end) = condition+ keys, values = [k_start, k_end], [v_start, v_end]+ if past is not None:+ keys.append(apply_rope1(past[0], freqs_past).view(plane))+ values.append(past[1].view(plane))+ keys.append(apply_rope1(k, freqs).view(plane))+ values.append(v.view(plane))+ windows = [RELAY_FLASH_CONDITION_WINDOW] * 2 + [RELAY_FLASH_VIDEO_WINDOW] * (len(keys) - 2)+ x = comfy_kitchen.window_attention(q.view(b, 1, height, width, self.num_heads, self.head_dim), torch.cat(keys, dim=1), torch.cat(values, dim=1), windows, clamp=False)+ return self.output(x.view(b, s, c)), (k, v)+++class RelayFlashFeedForward(nn.Module):+ def __init__(self, dim, hidden_dim, device=None, dtype=None):+ super().__init__()+ self.gate_up_proj = ops.Linear(dim, 2 * hidden_dim, bias=False, device=device, dtype=dtype)+ self.down = ops.Linear(hidden_dim, dim, bias=False, device=device, dtype=dtype)++ def forward(self, x):+ return comfy.ops.linear_input_act(self.down, self.gate_up_proj(x), "swiglu")+++class RelayFlashBlock(nn.Module):+ def __init__(self, dim, num_heads, kv_groups, hidden_dim, eps, device=None, dtype=None):+ super().__init__()+ self.attention_norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)+ self.attention = RelayFlashAttention(dim, num_heads, kv_groups, eps, device=device, dtype=dtype)+ self.feed_forward_norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)+ self.feed_forward = RelayFlashFeedForward(dim, hidden_dim, device=device, dtype=dtype)++ def condition(self, x, *args):+ y, kv = self.attention.condition(self.attention_norm(x), *args)+ x = x + y+ return x + self.feed_forward(self.feed_forward_norm(x)), kv++ def video(self, x, *args):+ y, kv = self.attention.video(self.attention_norm(x), *args)+ x = x + y+ return x + self.feed_forward(self.feed_forward_norm(x)), kv+++class RelayFlashEncoder(nn.Module):+ def __init__(self, last_stride, dim, eps, device=None, dtype=None):+ super().__init__()+ self.layers = nn.Sequential(+ ops.Conv2d(3, 64, 3, stride=2, padding=1, device=device, dtype=dtype), nn.SiLU(),+ ops.Conv2d(64, 128, 3, stride=2, padding=1, device=device, dtype=dtype), nn.SiLU(),+ ops.Conv2d(128, 128, 3, stride=last_stride, padding=1, device=device, dtype=dtype), nn.SiLU(),+ ops.Conv2d(128, dim, 1, device=device, dtype=dtype))+ self.norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)++ def forward(self, x):+ return self.norm(self.layers(x).flatten(2).transpose(1, 2))+++class RelayFlashConditionEncoder(nn.Module):+ def __init__(self, in_channels, dim, eps, device=None, dtype=None):+ super().__init__()+ self.projection = ops.Conv2d(in_channels, dim, 1, device=device, dtype=dtype)+ self.norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)++ def forward(self, x):+ return self.norm(self.projection(x).flatten(2).transpose(1, 2))+++class RelayFlashBackbone(nn.Module):+ def __init__(self, depth, dim, num_heads, kv_groups, hidden_dim, eps, device=None, dtype=None):+ super().__init__()+ self.blocks = nn.ModuleList([RelayFlashBlock(dim, num_heads, kv_groups, hidden_dim, eps, device=device, dtype=dtype) for _ in range(depth)])+++class RelayFlash(nn.Module):+ """Per-frame decoder conditioned on the keyframe latents on either side of the frame, VAE slot of RelayVSR."""++ comfy_has_chunked_io = True++ def __init__(self, upscale_factor, latent_channels=48, dim=512, depth=10, num_heads=8, kv_groups=2, hidden_dim=1408, head_dim=1024, eps=1e-5, device=None, dtype=None):+ super().__init__()+ self.upscale_factor = upscale_factor+ self.latent_stride = 16 // upscale_factor+ self.lr_encoder = RelayFlashEncoder(self.latent_stride // 4, dim, eps, device=device, dtype=dtype)+ self.condition_encoder = RelayFlashConditionEncoder(latent_channels, dim, eps, device=device, dtype=dtype)+ self.video_type_embedding = nn.Parameter(torch.empty(1, 1, dim, device=device, dtype=dtype))+ self.condition_type_embedding = nn.Parameter(torch.empty(1, 1, dim, device=device, dtype=dtype))+ self.backbone = RelayFlashBackbone(depth, dim, num_heads, kv_groups, hidden_dim, eps, device=device, dtype=dtype)+ self.spatial_head = nn.Sequential(ops.Linear(dim, head_dim, device=device, dtype=dtype), nn.SiLU())+ self.residual_projection = ops.Linear(head_dim, 768, device=device, dtype=dtype)+ self.rope_embedder = EmbedND(dim=dim // num_heads, theta=10000.0, axes_dim=[16, 24, 24])++ def grid_rope(self, t, height, width, device):+ ids = torch.zeros((height, width, 3), device=device, dtype=torch.float32)+ ids[..., 0] = t+ ids[..., 1] = torch.arange(height, device=device, dtype=torch.float32)[:, None]+ ids[..., 2] = torch.arange(width, device=device, dtype=torch.float32)[None]+ return self.rope_embedder(ids.reshape(1, -1, 3)).movedim(1, 2)++ def prepare_condition(self, latent, freqs):+ height, width = latent.shape[-2:]+ x = self.condition_encoder(latent) + comfy.ops.cast_to_input(self.condition_type_embedding, latent)+ out = []+ for block in self.backbone.blocks:+ x, kv = block.condition(x, freqs, height, width)+ out.append(kv)+ return out++ def decode_frame(self, lr, dtype, freqs, freqs_past, condition, history, detail):+ height, width = lr.shape[-2] // self.latent_stride, lr.shape[-1] // self.latent_stride+ x = self.lr_encoder(lr.to(dtype))+ x = x + comfy.ops.cast_to_input(self.video_type_embedding, x)+ next_history = []+ for block, cond, past in zip(self.backbone.blocks, condition, history):+ x, kv = block.video(x, freqs, freqs_past, height, width, cond, past)+ next_history.append(kv)+ residual = F.pixel_shuffle(self.residual_projection(self.spatial_head(x)).transpose(1, 2).reshape(lr.shape[0], 768, height, width), 16)+ if detail != 1.0:+ residual = residual * detail+ return F.interpolate(lr, scale_factor=self.upscale_factor, mode="bicubic", align_corners=False) + residual, next_history++ def decode_output_shape(self, input_shape, lq_images=None, detail=1.0):+ if lq_images is None:+ raise ValueError("The RelayVSR Flash decoder needs the low-quality frames, decode with the VSR Decode node.")+ frames, height, width = lq_images.shape[:3]+ # the x2 and x4 decoders load from the same weight shapes, a latent of the other factor would decode garbage or fail on the copy+ if tuple(input_shape[2:]) != (len(relay_keyframes(frames)), *relay_latent_size(height, width, self.latent_stride)):+ raise ValueError(f"The latent {tuple(input_shape[2:])} doesn't fit the x{self.upscale_factor} RelayVSR Flash decoder and these {frames} {width}x{height} frames: use the RelayVSR model and Flash decoder of the same factor, and the same video in RelayVSR Conditioning and VSR Decode.")+ return (input_shape[0], 3, frames, height * self.upscale_factor, width * self.upscale_factor)++ def decode(self, x, lq_images=None, output_buffer=None, detail=1.0):+ # detail scales what the model adds to the bicubic upscale of each frame+ out = output_buffer+ if out is None:+ out = torch.empty(self.decode_output_shape(x.shape, lq_images), device=comfy.model_management.intermediate_device())+ b, _, _, height, width = x.shape+ frames, lq_height, lq_width = lq_images.shape[:3]+ depth = len(self.backbone.blocks)+ keyframes = relay_keyframes(frames)+ device = x.device+ freqs, freqs_past = self.grid_rope(0, height, width, device), self.grid_rope(-1, height, width, device)+ pbar = comfy.utils.ProgressBar(frames)+ # finished frames go to the output without the GPU waiting on each copy+ with comfy.model_management.HostWriter(device, out, (b, 3, *out.shape[-2:])) as writer, comfy.utils.model_trange(frames, desc="RelayVSR Flash decode", unit="frame") as console:+ start = self.prepare_condition(x[:, :, 0], freqs)+ for window, (first, last) in enumerate(zip(keyframes, keyframes[1:] or keyframes)):+ comfy.model_management.throw_exception_if_processing_interrupted()+ end = self.prepare_condition(x[:, :, min(window + 1, len(keyframes) - 1)], freqs)+ condition = list(zip(start, end))+ history = [None] * depth+ # one upload per window, an upload from pageable memory waits for the whole stream+ lr_window = lq_images[first:last + 1].to(device=device)[..., :3].movedim(-1, 1) * 2.0 - 1.0+ lr_window = F.pad(lr_window, (0, width * self.latent_stride - lq_width, 0, height * self.latent_stride - lq_height), mode="replicate")+ for i, frame in enumerate(range(first, last + 1)):+ decoded, history = self.decode_frame(lr_window[i:i + 1].expand(b, -1, -1, -1), x.dtype, freqs, freqs_past, condition, history, detail)+ # the shared keyframe was already output by the previous window, here it only seeds the history+ if window == 0 or frame > first:+ # finished [0, 1] pixels on the GPU, the VAE's process_output is the identity+ writer.write(out[:, :, frame], decoded[..., :out.shape[-2], :out.shape[-1]].add_(1.0).div_(2.0).clamp_(0.0, 1.0).to(out.dtype))+ pbar.update(1)+ console.update(1)+ start = end+ return outdiff --git a/comfy/model_base.py b/comfy/model_base.pyindex 0d4cfb114..39db511fc 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -52,6 +52,7 @@ import comfy.ldm.wan.model_animateimport comfy.ldm.wan.model_animate2import comfy.ldm.wan.ar_modelimport comfy.ldm.wan.model_wandancer+import comfy.ldm.wan.model_vsrimport comfy.ldm.hunyuan3d.modelimport comfy.ldm.triposplat.modelimport comfy.ldm.hidream.model@@ -2045,6 +2046,19 @@ class WAN21_SCAIL(WAN21):return out+class WAN22_RelayVSR(WAN21):+ def __init__(self, model_config, model_type=ModelType.FLOW, device=None):+ super(WAN21, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.wan.model_vsr.RelayVSRWanModel)+ self.image_to_video = False++ def extra_conds(self, **kwargs):+ out = super().extra_conds(**kwargs)+ lq_images = kwargs.get("lq_images", None)+ if lq_images is not None:+ out['lq_pixels'] = comfy.conds.CONDRegular(lq_images[..., :3].movedim(-1, 0).unsqueeze(0) * 2.0 - 1.0)+ return out++class WAN21_SCAIL2(WAN21_SCAIL):"""SCAIL-2: SCAIL-Preview + an additive binary multi-identity mask stream."""diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 04d76db2a..f7e412a2b 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -787,6 +787,10 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):dit_config["model_type"] = "scail"elif '{}patch_embedding_global.weight'.format(key_prefix) in state_dict_keys:dit_config["model_type"] = "wandancer"+ elif '{}lq_proj_in.conv1.weight'.format(key_prefix) in state_dict_keys:+ dit_config["model_type"] = "relayvsr"+ # the LQ frames go in pixel-unshuffled by 4 * upscale_factor+ dit_config["upscale_factor"] = math.isqrt(state_dict['{}lq_proj_in.conv1.weight'.format(key_prefix)].shape[1] // 3) // 4else:if '{}img_emb.proj.0.bias'.format(key_prefix) in state_dict_keys:dit_config["model_type"] = "i2v"diff --git a/comfy/model_management.py b/comfy/model_management.pyindex e33147e74..405f15012 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -1490,6 +1490,21 @@ def reset_cast_buffers():STREAM_AIMDO_CAST_BUFFERS.clear()soft_empty_cache()+def new_stream(device):+ if is_device_cuda(device):+ s = torch.cuda.Stream(device=device, priority=0)+ s.as_context = torch.cuda.stream+ elif is_device_xpu(device):+ s = torch.xpu.Stream(device=device, priority=0)+ s.as_context = torch.xpu.stream+ elif is_device_npu(device):+ # torch.npu.stream provides the context manager for torch-npu streams.+ s = torch.npu.Stream(device=device, priority=0)+ s.as_context = torch.npu.stream+ else:+ return None+ return s+def get_offload_stream(device):stream_counter = stream_counters.get(device, 0)if NUM_STREAMS == 0:@@ -1505,39 +1520,24 @@ def get_offload_stream(device):stream_counter = (stream_counter + 1) % len(ss)stream_counters[device] = stream_counterreturn ss[stream_counter]- elif is_device_cuda(device):- ss = []- for k in range(NUM_STREAMS):- s1 = torch.cuda.Stream(device=device, priority=0)- s1.as_context = torch.cuda.stream- ss.append(s1)- STREAMS[device] = ss- s = ss[stream_counter]- stream_counters[device] = stream_counter- return s- elif is_device_xpu(device):- ss = []- for k in range(NUM_STREAMS):- s1 = torch.xpu.Stream(device=device, priority=0)- s1.as_context = torch.xpu.stream- ss.append(s1)- STREAMS[device] = ss- s = ss[stream_counter]- stream_counters[device] = stream_counter- return s- elif is_device_npu(device):- ss = []- # Match the CUDA/XPU stream interface used by the shared offload path.- # torch.npu.stream provides the context manager for torch-npu streams.- for k in range(NUM_STREAMS):- s1 = torch.npu.Stream(device=device, priority=0)- s1.as_context = torch.npu.stream- ss.append(s1)- STREAMS[device] = ss- s = ss[stream_counter]- stream_counters[device] = stream_counter- return s- return None+ s = new_stream(device)+ if s is None:+ return None+ ss = [s] + [new_stream(device) for _ in range(NUM_STREAMS - 1)]+ STREAMS[device] = ss+ stream_counters[device] = stream_counter+ return ss[stream_counter]++DATA_STREAMS = {}++def get_data_stream(device):+ """One stream per device for data copies such as cache staging and decoder output, kept off the+ offload streams so they don't disturb the weight prefetch. Callers order on it themselves."""+ if NUM_STREAMS == 0 or torch.compiler.is_compiling():+ return None+ if device not in DATA_STREAMS:+ DATA_STREAMS[device] = new_stream(device)+ return DATA_STREAMS[device]def sync_all_offload_streams():if NUM_STREAMS == 0:@@ -1737,6 +1737,59 @@ def unpin_memory(tensor):return False+class HostWriter:+ """Copies device tensors of one shape into slices of a CPU tensor without the device waiting on each copy:+ every write goes through one of two pinned staging buffers on the data stream and lands in its slice one+ write later. Writes go straight through when the target is on the device, the device has no streams or the+ buffers can't be pinned."""++ def __init__(self, device, target, shape):+ self.device = device+ self.staging = []+ self.pending = [None, None]+ self.slot = 0+ self.stream = get_data_stream(device) if is_device_cpu(target.device) else None+ if self.stream is not None:+ # inactive model pins make room for the staging buffers, active ones stay+ if ensure_pin_budget(2 * torch.Size(shape).numel() * target.element_size(), evict_active=False):+ for _ in range(2):+ buffer = torch.empty(shape, dtype=target.dtype)+ if pin_memory(buffer, evict_active=False):+ self.staging.append(buffer)+ if len(self.staging) < 2:+ self.stream = None++ def __enter__(self):+ return self++ def write(self, dest, src):+ if self.stream is None:+ dest.copy_(src)+ return+ if self.pending[self.slot] is not None:+ self._collect(self.slot)+ self.stream.wait_stream(current_stream(self.device))+ cast_to(src, None, torch.device("cpu"), non_blocking=True, stream=self.stream, r=self.staging[self.slot])+ # src stays referenced until its copy is done+ self.pending[self.slot] = (self.stream.record_event(), dest, src)+ self.slot ^= 1++ def _collect(self, slot):+ event, dest, _ = self.pending[slot]+ event.synchronize()+ dest.copy_(self.staging[slot])+ self.pending[slot] = None++ def __exit__(self, exc_type, exc, tb):+ if self.stream is not None:+ if exc_type is None:+ for slot in (self.slot, self.slot ^ 1):+ if self.pending[slot] is not None:+ self._collect(slot)+ self.stream.synchronize()+ for buffer in self.staging:+ unpin_memory(buffer)+def sage_attention_enabled():return args.use_sage_attentiondiff --git a/comfy/sd.py b/comfy/sd.pyindex fca0c7576..0c6168c0e 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -17,6 +17,7 @@ import comfy.ldm.cosmos.vaeimport comfy.ldm.wan.vaeimport comfy.ldm.trellis2.vaeimport comfy.ldm.wan.vae2_2+import comfy.ldm.wan.model_vsrimport comfy.ldm.hunyuan3d.vaeimport comfy.ldm.seedvr.vaeimport comfy.ldm.mage_flow.vae@@ -987,6 +988,18 @@ class VAE:self.downscale_index_formula = (4, 8, 8)self.memory_used_encode = lambda shape, dtype: (700 * (max(1, (shape[-3] ** 0.66 * 0.11)) * shape[-2] * shape[-1]) * model_management.dtype_size(dtype))self.memory_used_decode = lambda shape, dtype: (50 * (max(1, (shape[-3] ** 0.65 * 0.26)) * shape[-2] * shape[-1] * 32 * 32) * model_management.dtype_size(dtype))+ elif "lr_encoder.layers.0.weight" in sd and "condition_encoder.projection.weight" in sd: # RelayVSR Flash+ self.latent_channels = 48+ self.latent_dim = 3+ self.upscale_ratio = (lambda a: max(0, (a - 1) * comfy.ldm.wan.model_vsr.RELAY_KEYFRAME_INTERVAL + 1), 16, 16)+ # x2 and x4 decoders have the same weight shapes, only the metadata tells them apart+ config = json.loads((metadata or {}).get("config", "{}"))+ if "upscale_factor" not in config:+ raise ValueError("RelayVSR Flash decoder without its upscale_factor metadata, use the released relayvsr_flash_x2/x4 file.")+ self.first_stage_model = comfy.ldm.wan.model_vsr.RelayFlash(config["upscale_factor"])+ self.working_dtypes = [torch.float16, torch.bfloat16, torch.float32]+ self.process_input = self.process_output = lambda image: image+ self.memory_used_decode = lambda shape, dtype: (40000 * shape[-2] * shape[-1] * model_management.dtype_size(dtype))elif "vocoder.resblocks.0.convs1.0.weight" in sd or "vocoder.vocoder.resblocks.0.convs1.0.weight" in sd: # LTX Audiosd = comfy.utils.state_dict_prefix_replace(sd, {"audio_vae.": "autoencoder."})self.first_stage_model = comfy.ldm.lightricks.vae.audio_vae.AudioVAE(metadata=metadata)@@ -1366,7 +1379,7 @@ class VAE:preallocated = Falseif getattr(self.first_stage_model, 'comfy_has_chunked_io', False):with comfy.model_prefetch.pause_malloc_graph():- pixel_samples = torch.empty(self.first_stage_model.decode_output_shape(samples_in.shape), device=self.output_device, dtype=self.vae_output_dtype())+ pixel_samples = torch.empty(self.first_stage_model.decode_output_shape(samples_in.shape, **vae_options), device=self.output_device, dtype=self.vae_output_dtype())preallocated = Truefor x in range(0, samples_in.shape[0], batch_number):diff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex 3d2015bbd..66c823e6c 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -1578,6 +1578,19 @@ class WAN21_SCAIL2(WAN21_T2V):out = model_base.WAN21_SCAIL2(self, image_to_video=False, device=device)return out+class WAN22_RelayVSR(WAN21_T2V):+ unet_config = {+ "image_model": "wan2.1",+ "model_type": "relayvsr",+ }++ latent_format = latent_formats.RelayVSR++ def get_model(self, state_dict, prefix="", device=None):+ out = model_base.WAN22_RelayVSR(self, device=device)+ return out++class WAN22_WanDancer(WAN21_T2V):unet_config = {"image_model": "wan2.1",@@ -2724,6 +2737,7 @@ models = [WAN21_FlowRVS,WAN21_SCAIL,WAN21_SCAIL2,+ WAN22_RelayVSR,WAN22_WanDancer,Hunyuan3Dv2mini,Hunyuan3Dv2,diff --git a/comfy_extras/nodes_wan.py b/comfy_extras/nodes_wan.pyindex bb41f7ddb..02ec00ad3 100644--- a/comfy_extras/nodes_wan.py+++ b/comfy_extras/nodes_wan.py@@ -7,6 +7,7 @@ import comfy.utilsimport comfy.latent_formatsimport comfy.clip_visionimport comfy.ldm.wan.model_animate2+import comfy.ldm.wan.model_vsrimport comfy.patcher_extensionimport jsonimport numpy as np@@ -1459,6 +1460,58 @@ class Wan22ImageToVideoLatent(io.ComfyNode):return io.NodeOutput(out_latent)+class RelayVSRConditioning(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="RelayVSRConditioning",+ category="model/conditioning/wan",+ description="Conditioning and empty latent for RelayVSR video upscaling, sample with 1 step at cfg 1. Only every 15th frame goes through the model, VSR Decode restores all of them.",+ inputs=[+ io.Model.Input("model"),+ io.Image.Input("images", tooltip="The low-quality video at its native resolution, the model decides the upscale factor."),+ ],+ outputs=[+ io.Conditioning.Output(display_name="positive"),+ io.Latent.Output(display_name="latent"),+ ],+ )++ @classmethod+ def execute(cls, model, images) -> io.NodeOutput:+ diffusion_model = model.model.diffusion_model+ keyframes = comfy.ldm.wan.model_vsr.relay_keyframes(images.shape[0])+ positive = [[diffusion_model.positive_conditioning.unsqueeze(0).clone(), {"lq_images": images[keyframes]}]]+ latent_size = comfy.ldm.wan.model_vsr.relay_latent_size(images.shape[1], images.shape[2], diffusion_model.latent_stride)+ latent = torch.zeros([1, model.model.latent_format.latent_channels, len(keyframes), *latent_size], device=comfy.model_management.intermediate_device())+ return io.NodeOutput(positive, {"samples": latent})+++class VSRDecode(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="VSRDecode",+ display_name="VSR Decode",+ category="latent",+ description="Decode RelayVSR latents with the RelayVSR Flash decoder, which also reads the low-quality frames.",+ inputs=[+ io.Vae.Input("vae"),+ io.Latent.Input("samples"),+ io.Image.Input("images", tooltip="The same low-quality video given to the conditioning node."),+ io.Float.Input("detail", default=1.0, min=0.0, max=1.0, step=0.01, tooltip="How much of the detail the decoder adds to the bicubic upscale of each frame is kept, 1 is the full model output."),+ ],+ outputs=[+ io.Image.Output(),+ ],+ )++ @classmethod+ def execute(cls, vae, samples, images, detail=1.0) -> io.NodeOutput:+ decoded = vae.decode(samples["samples"], vae_options={"lq_images": images, "detail": detail})+ return io.NodeOutput(decoded.reshape(-1, *decoded.shape[-3:]))++from comfy.ldm.wan.model_multitalk import InfiniteTalkOuterSampleWrapper, MultiTalkCrossAttnPatch, MultiTalkGetAttnMapPatch, project_audio_featuresclass WanInfiniteTalkToVideo(io.ComfyNode):class DCValues(TypedDict):@@ -1646,6 +1699,8 @@ class WanExtension(ComfyExtension):WanAnimate2Cache,Wan22ImageToVideoLatent,WanInfiniteTalkToVideo,+ RelayVSRConditioning,+ VSRDecode,]async def comfy_entrypoint() -> WanExtension: