Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Support RelayVSR (CORE-701) (#17511)

Jukka Seppänen2f8943324e1e
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py
index 0e7fba912..1367c5214 100644
--- a/comfy/latent_formats.py
+++ b/comfy/latent_formats.py
@@ -880,6 +880,14 @@ class Wan22(Wan21):
0.3971, 1.0600, 0.3943, 0.5537, 0.5444, 0.4089, 0.7468, 0.7744
]).view(1, self.latent_channels, 1, 1, 1)
+class RelayVSR(Wan22):
+ # the keyframe Wan and the Flash decoder both work on the normalized latent
+ def process_in(self, latent):
+ return latent
+
+ def process_out(self, latent):
+ return latent
+
class HunyuanImage21(LatentFormat):
latent_channels = 64
latent_dimensions = 2
diff --git a/comfy/ldm/seedvr/vae.py b/comfy/ldm/seedvr/vae.py
index 4437caee4..c73ee1c97 100644
--- a/comfy/ldm/seedvr/vae.py
+++ b/comfy/ldm/seedvr/vae.py
@@ -233,7 +233,7 @@ class CausalMemoryCache:
"""The one stream this cache copies on, so copies reusing a host buffer stay ordered;
it starts after the compute queued so far. None copies on the compute stream."""
if self._stream is None:
- self._stream = comfy.model_management.get_offload_stream(device)
+ self._stream = comfy.model_management.get_data_stream(device)
if self._stream is not None:
self._stream.wait_stream(comfy.model_management.current_stream(device))
return self._stream
diff --git a/comfy/ldm/wan/model_animate2.py b/comfy/ldm/wan/model_animate2.py
index 45629af4d..744401ecb 100644
--- a/comfy/ldm/wan/model_animate2.py
+++ b/comfy/ldm/wan/model_animate2.py
@@ -210,7 +210,7 @@ class PoseBranchCache:
with comfy.model_prefetch.pause_malloc_graph():
self._staging[buf_key] = [torch.empty(t.shape, dtype=buf_key[1], device=device) for _ in range(2)]
r = self._staging[buf_key][i % 2]
- stream = comfy.model_management.get_offload_stream(device)
+ stream = comfy.model_management.get_data_stream(device)
cs = comfy.model_management.current_stream(device)
if stream is not None and cs is not None:
# Wait for staging allocation and the previous consumer's reads before overwriting the buffer.
diff --git a/comfy/ldm/wan/model_vsr.py b/comfy/ldm/wan/model_vsr.py
new file mode 100644
index 000000000..9fd4a72bc
--- /dev/null
+++ b/comfy/ldm/wan/model_vsr.py
@@ -0,0 +1,372 @@
+# Wan video super-resolution
+# RelayVSR https://github.com/kopperx/RelayVSR (Apache-2.0): one-step Wan keyframes guiding a small per-frame transformer
+import math
+
+import comfy_kitchen
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from comfy.ldm.flux.layers import EmbedND
+from comfy.ldm.flux.math import apply_rope1
+from comfy.ldm.wan.model import WanAttentionBlock, WanModel, WanSelfAttention, modulate, sinusoidal_embedding_1d
+import comfy.ldm.common_dit
+import comfy.model_management
+import comfy.model_prefetch
+import comfy.ops
+import comfy.utils
+
+ops = comfy.ops.disable_weight_init
+
+RELAY_KEYFRAME_INTERVAL = 15
+RELAY_WAN_CONTEXT = 4
+RELAY_WAN_WINDOW = (22, 40)
+RELAY_WAN_CHUNK_TOKENS = 32768
+# the per-layer cache needs about as much VRAM as this many keyframes of a single pass
+RELAY_WAN_CACHE_KEYFRAMES = 7
+RELAY_FLASH_CONDITION_WINDOW = (48, 48)
+RELAY_FLASH_VIDEO_WINDOW = (24, 24)
+
+
+def relay_keyframes(frames):
+ return list(range(0, frames - 1, RELAY_KEYFRAME_INTERVAL)) + [frames - 1]
+
+
+def relay_latent_size(height, width, latent_stride):
+ # whole 2x2 patches of the model's pixel stride
+ stride = latent_stride * 2
+ return math.ceil(height / stride) * 2, math.ceil(width / stride) * 2
+
+
+class RelayLQProjection(nn.Module):
+ def __init__(self, upscale_factor, hidden_dim=768, hidden_dim2=3072, out_dim=3072, eps=1e-6, device=None, dtype=None, operations=None):
+ super().__init__()
+ self.upscale_factor = upscale_factor
+ self.conv1 = operations.Conv2d(3 * (4 * upscale_factor) ** 2, hidden_dim, 1, device=device, dtype=dtype)
+ self.norm1 = operations.RMSNorm(hidden_dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
+ self.conv2 = operations.Conv2d(hidden_dim, hidden_dim2, 3, stride=2, padding=1, device=device, dtype=dtype)
+ self.norm2 = operations.RMSNorm(hidden_dim2, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
+ self.linear = operations.Linear(hidden_dim2, out_dim, device=device, dtype=dtype)
+
+ def forward(self, video):
+ b, c, f, h, w = video.shape
+ x = video.movedim(2, 1).reshape(b * f, c, h, w)
+ if self.upscale_factor == 4:
+ x = F.interpolate(x, scale_factor=4, mode="bilinear", align_corners=False)
+ x = self.conv1(F.pixel_unshuffle(x, 4 * self.upscale_factor))
+ x = F.silu(self.norm1(x.movedim(1, -1))).movedim(-1, 1)
+ x = F.silu(self.norm2(self.conv2(x).flatten(2).transpose(1, 2)))
+ return self.linear(x).view(b, -1, self.linear.out_features)
+
+
+class RelayVSRSelfAttention(WanSelfAttention):
+ def forward(self, x, freqs, cache, start, store, transformer_options={}):
+ # causal over the last keyframes, per-query windows slid to stay inside the frame
+ b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim
+ frames, height, width = transformer_options["grid_sizes"]
+ grid = (b, frames, height, width, n, d)
+ q = apply_rope1(self.norm_q(self.q(x)).view(b, s, n, d), freqs).view(grid)
+ k = apply_rope1(self.norm_k(self.k(x)).view(b, s, n, d), freqs).view(grid)
+ v = self.v(x).view(grid)
+ # keyframes before this chunk come from the cache, which holds the last ones in order as int8 rows with a scale each
+ past = min(start, RELAY_WAN_CONTEXT - 1)
+ keys, values = k, v
+ if past > 0:
+ cached, scales = cache
+ k_past, v_past = (cached[:, :, :past].to(k.dtype) * scales[:, :, :past]).unbind(0)
+ keys, values = torch.cat((k_past, k), dim=1), torch.cat((v_past, v), dim=1)
+ x = comfy_kitchen.window_attention(q, keys, values, RELAY_WAN_WINDOW, temporal_window=RELAY_WAN_CONTEXT)
+ if store:
+ # the cache moves on to the last keyframes for the next chunk, older cached ones first when this one is short
+ cached, scales = cache
+ new = min(frames, RELAY_WAN_CONTEXT - 1)
+ old = min(past, RELAY_WAN_CONTEXT - 1 - new)
+ if old > 0:
+ cached[:, :, :old] = cached[:, :, past - old:past].clone()
+ scales[:, :, :old] = scales[:, :, past - old:past].clone()
+ for i, rows in enumerate((k[:, frames - new:], v[:, frames - new:])):
+ scale = (rows.abs().amax(dim=-1, keepdim=True) / 127.0).clamp_(min=torch.finfo(rows.dtype).tiny)
+ scales[i, :, old:old + new] = scale
+ cached[i, :, old:old + new] = (rows.float() / scale.float()).round_().clamp_(-127.0, 127.0)
+ return self.o(x.view(b, s, n * d))
+
+
+class RelayVSRAttentionBlock(WanAttentionBlock):
+ def forward(self, x, e, freqs, context, cache, start, store, transformer_options={}):
+ # residual updated in place, nothing allocated in the block outlives its memory compiler scope
+ e = (comfy.model_management.cast_to(self.modulation, dtype=x.dtype, device=x.device).unsqueeze(0) + e).unbind(2)
+ x.addcmul_(self.self_attn(modulate(x, self.norm1, e[0], e[1]), freqs, cache, start, store, transformer_options=transformer_options), e[2])
+ x.add_(self.cross_attn(self.norm3(x), context, transformer_options=transformer_options))
+ x.addcmul_(self.ffn(modulate(x, self.norm2, e[3], e[4])), e[5])
+ return x
+
+
+class RelayVSRWanModel(WanModel):
+ def __init__(self, model_type=None, upscale_factor=4, operations=None, device=None, dtype=None, **kwargs):
+ super().__init__(model_type="t2v", wan_attn_block_class=RelayVSRAttentionBlock, operations=operations, device=device, dtype=dtype, **kwargs)
+ self.register_buffer("positive_conditioning", torch.empty((self.text_len, self.text_dim), device=device, dtype=dtype))
+ operation_settings = {"operations": operations, "device": device, "dtype": dtype}
+ for block in self.blocks:
+ block.self_attn = RelayVSRSelfAttention(self.dim, self.num_heads, eps=self.eps, operation_settings=operation_settings)
+ self.lq_proj_in = RelayLQProjection(upscale_factor, out_dim=self.dim, device=device, dtype=dtype, operations=operations)
+ self.latent_stride = 16 // upscale_factor
+
+ def forward_orig(self, x, t, context, freqs=None, lq_pixels=None, transformer_options={}, **kwargs):
+ b, frames = x.shape[0], x.shape[2]
+ height, width = x.shape[3] // self.patch_size[1], x.shape[4] // self.patch_size[2]
+ tokens = height * width
+ e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t.flatten()).to(dtype=x.dtype))
+ e = e.reshape(t.shape[0], -1, e.shape[-1])
+ e0 = self.time_projection(e).unflatten(2, (6, self.dim))
+ context = self.text_embedding(context)
+
+ # keyframes go through in chunks, every layer's cache carries the keys and values of the last ones across
+ chunk = max(1, RELAY_WAN_CHUNK_TOKENS // tokens)
+ if frames <= chunk + RELAY_WAN_CACHE_KEYFRAMES:
+ chunk = frames
+ shape = (2, b, RELAY_WAN_CONTEXT - 1, height, width, self.num_heads, self.dim // self.num_heads)
+ caches = [(torch.empty(shape, device=x.device, dtype=torch.int8), torch.empty((*shape[:-1], 1), device=x.device, dtype=x.dtype)) if frames > chunk else None for _ in self.blocks]
+ out = torch.empty((b, self.out_dim, *x.shape[2:]), device=x.device, dtype=x.dtype)
+ blocks_replace = transformer_options.get("patches_replace", {}).get("dit", {})
+ transformer_options["total_blocks"] = len(self.blocks)
+ transformer_options["block_type"] = "double"
+ starts = range(0, frames, chunk)
+ # the sampler's bars only count the one step
+ progress = comfy.utils.ProgressBar(len(starts) * len(self.blocks))
+ with comfy.utils.model_trange(len(starts) * len(self.blocks), desc="RelayVSR", unit="block") as pbar:
+ for start in starts:
+ end = min(start + chunk, frames)
+ # only a following chunk reads the cache
+ store = end < frames
+ grid_sizes = (end - start, height, width)
+ transformer_options["grid_sizes"] = grid_sizes
+ # the projected LQ frames go first so the sum is row-major, the blocks update it in place
+ hidden = self.lq_proj_in(lq_pixels[:, :, start:end]) + self.patch_embedding(x[:, :, start:end].float()).to(x.dtype).flatten(2).transpose(1, 2)
+ chunk_freqs = freqs[:, start * tokens:end * tokens]
+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.blocks), x.device, transformer_options)
+ comfy.model_prefetch.malloc_graph_begin(x.device)
+ for i, block in enumerate(self.blocks):
+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, x.device, block, x.dtype, malloc_scope="block")
+ transformer_options["block_index"] = i
+ if ("double_block", i) in blocks_replace:
+ def block_wrap(args):
+ return {"img": block(args["img"], args["vec"], args["pe"], args["txt"], caches[i], start, store, transformer_options=args["transformer_options"])}
+ hidden = blocks_replace[("double_block", i)]({"img": hidden, "txt": context, "vec": e0, "pe": chunk_freqs, "transformer_options": transformer_options}, {"original_block": block_wrap})["img"]
+ else:
+ hidden = block(hidden, e0, chunk_freqs, context, caches[i], start, store, transformer_options=transformer_options)
+ pbar.update(1)
+ progress.update(1)
+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, x.device, None, malloc_scope="block")
+ comfy.model_prefetch.malloc_graph_end()
+ out[:, :, start:end] = self.unpatchify(self.head(hidden, e), grid_sizes)
+ return out
+
+ def _forward(self, x, timestep, context, clip_fea=None, time_dim_concat=None, transformer_options={}, lq_pixels=None, **kwargs):
+ if lq_pixels is None:
+ raise ValueError("RelayVSR needs the low-quality video, connect the model through the RelayVSR Conditioning node.")
+ t, h, w = x.shape[2:]
+ x = comfy.ldm.common_dit.pad_to_patch_size(x, self.patch_size, padding_mode="replicate")
+ tp, hp, wp = x.shape[2:]
+ lq_height, lq_width = hp * self.latent_stride, wp * self.latent_stride
+ lq_pixels = lq_pixels[..., :lq_height, :lq_width]
+ lq_pixels = F.pad(lq_pixels, (0, lq_width - lq_pixels.shape[-1], 0, lq_height - lq_pixels.shape[-2], 0, 0), mode="replicate")
+ freqs = self.rope_encode(tp, hp, wp, device=x.device, dtype=x.dtype, transformer_options=transformer_options)
+ return self.forward_orig(x, timestep, context, freqs=freqs, lq_pixels=lq_pixels, transformer_options=transformer_options, **kwargs)[:, :, :t, :h, :w]
+
+
+class RelayFlashAttention(nn.Module):
+ def __init__(self, dim, num_heads, kv_groups, eps, device=None, dtype=None):
+ super().__init__()
+ self.num_heads = num_heads
+ self.kv_groups = kv_groups
+ self.head_dim = dim // num_heads
+ self.query = ops.Linear(dim, dim, device=device, dtype=dtype)
+ self.key = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype)
+ self.value = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype)
+ self.condition_key = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype)
+ self.condition_value = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype)
+ self.key_norm = ops.RMSNorm(self.head_dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
+ self.value_norm = ops.RMSNorm(self.head_dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
+ self.output = ops.Linear(dim, dim, device=device, dtype=dtype)
+
+ def project(self, x, key, value, freqs):
+ b, s, _ = x.shape
+ q = apply_rope1(self.query(x).view(b, s, self.num_heads, self.head_dim), freqs)
+ k = self.key_norm(key(x).view(b, s, self.kv_groups, self.head_dim))
+ v = self.value_norm(value(x).view(b, s, self.kv_groups, self.head_dim))
+ return q, k, v
+
+ def condition(self, x, freqs, height, width):
+ # the rotated condition keys are reused as they are by every frame of the window
+ b, s, c = x.shape
+ q, k, v = self.project(x, self.condition_key, self.condition_value, freqs)
+ k = apply_rope1(k, freqs).view(b, 1, height, width, self.kv_groups, self.head_dim)
+ v = v.view(k.shape)
+ x = comfy_kitchen.window_attention(q.view(b, 1, height, width, self.num_heads, self.head_dim), k, v, RELAY_FLASH_CONDITION_WINDOW, clamp=False)
+ return self.output(x.view(b, s, c)), (k, v)
+
+ def video(self, x, freqs, freqs_past, height, width, condition, past):
+ b, s, c = x.shape
+ plane = (b, 1, height, width, self.kv_groups, self.head_dim)
+ q, k, v = self.project(x, self.key, self.value, freqs)
+ # both condition keyframes, then the previous frame and this one
+ (k_start, v_start), (k_end, v_end) = condition
+ keys, values = [k_start, k_end], [v_start, v_end]
+ if past is not None:
+ keys.append(apply_rope1(past[0], freqs_past).view(plane))
+ values.append(past[1].view(plane))
+ keys.append(apply_rope1(k, freqs).view(plane))
+ values.append(v.view(plane))
+ windows = [RELAY_FLASH_CONDITION_WINDOW] * 2 + [RELAY_FLASH_VIDEO_WINDOW] * (len(keys) - 2)
+ x = comfy_kitchen.window_attention(q.view(b, 1, height, width, self.num_heads, self.head_dim), torch.cat(keys, dim=1), torch.cat(values, dim=1), windows, clamp=False)
+ return self.output(x.view(b, s, c)), (k, v)
+
+
+class RelayFlashFeedForward(nn.Module):
+ def __init__(self, dim, hidden_dim, device=None, dtype=None):
+ super().__init__()
+ self.gate_up_proj = ops.Linear(dim, 2 * hidden_dim, bias=False, device=device, dtype=dtype)
+ self.down = ops.Linear(hidden_dim, dim, bias=False, device=device, dtype=dtype)
+
+ def forward(self, x):
+ return comfy.ops.linear_input_act(self.down, self.gate_up_proj(x), "swiglu")
+
+
+class RelayFlashBlock(nn.Module):
+ def __init__(self, dim, num_heads, kv_groups, hidden_dim, eps, device=None, dtype=None):
+ super().__init__()
+ self.attention_norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
+ self.attention = RelayFlashAttention(dim, num_heads, kv_groups, eps, device=device, dtype=dtype)
+ self.feed_forward_norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
+ self.feed_forward = RelayFlashFeedForward(dim, hidden_dim, device=device, dtype=dtype)
+
+ def condition(self, x, *args):
+ y, kv = self.attention.condition(self.attention_norm(x), *args)
+ x = x + y
+ return x + self.feed_forward(self.feed_forward_norm(x)), kv
+
+ def video(self, x, *args):
+ y, kv = self.attention.video(self.attention_norm(x), *args)
+ x = x + y
+ return x + self.feed_forward(self.feed_forward_norm(x)), kv
+
+
+class RelayFlashEncoder(nn.Module):
+ def __init__(self, last_stride, dim, eps, device=None, dtype=None):
+ super().__init__()
+ self.layers = nn.Sequential(
+ ops.Conv2d(3, 64, 3, stride=2, padding=1, device=device, dtype=dtype), nn.SiLU(),
+ ops.Conv2d(64, 128, 3, stride=2, padding=1, device=device, dtype=dtype), nn.SiLU(),
+ ops.Conv2d(128, 128, 3, stride=last_stride, padding=1, device=device, dtype=dtype), nn.SiLU(),
+ ops.Conv2d(128, dim, 1, device=device, dtype=dtype))
+ self.norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
+
+ def forward(self, x):
+ return self.norm(self.layers(x).flatten(2).transpose(1, 2))
+
+
+class RelayFlashConditionEncoder(nn.Module):
+ def __init__(self, in_channels, dim, eps, device=None, dtype=None):
+ super().__init__()
+ self.projection = ops.Conv2d(in_channels, dim, 1, device=device, dtype=dtype)
+ self.norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
+
+ def forward(self, x):
+ return self.norm(self.projection(x).flatten(2).transpose(1, 2))
+
+
+class RelayFlashBackbone(nn.Module):
+ def __init__(self, depth, dim, num_heads, kv_groups, hidden_dim, eps, device=None, dtype=None):
+ super().__init__()
+ self.blocks = nn.ModuleList([RelayFlashBlock(dim, num_heads, kv_groups, hidden_dim, eps, device=device, dtype=dtype) for _ in range(depth)])
+
+
+class RelayFlash(nn.Module):
+ """Per-frame decoder conditioned on the keyframe latents on either side of the frame, VAE slot of RelayVSR."""
+
+ comfy_has_chunked_io = True
+
+ def __init__(self, upscale_factor, latent_channels=48, dim=512, depth=10, num_heads=8, kv_groups=2, hidden_dim=1408, head_dim=1024, eps=1e-5, device=None, dtype=None):
+ super().__init__()
+ self.upscale_factor = upscale_factor
+ self.latent_stride = 16 // upscale_factor
+ self.lr_encoder = RelayFlashEncoder(self.latent_stride // 4, dim, eps, device=device, dtype=dtype)
+ self.condition_encoder = RelayFlashConditionEncoder(latent_channels, dim, eps, device=device, dtype=dtype)
+ self.video_type_embedding = nn.Parameter(torch.empty(1, 1, dim, device=device, dtype=dtype))
+ self.condition_type_embedding = nn.Parameter(torch.empty(1, 1, dim, device=device, dtype=dtype))
+ self.backbone = RelayFlashBackbone(depth, dim, num_heads, kv_groups, hidden_dim, eps, device=device, dtype=dtype)
+ self.spatial_head = nn.Sequential(ops.Linear(dim, head_dim, device=device, dtype=dtype), nn.SiLU())
+ self.residual_projection = ops.Linear(head_dim, 768, device=device, dtype=dtype)
+ self.rope_embedder = EmbedND(dim=dim // num_heads, theta=10000.0, axes_dim=[16, 24, 24])
+
+ def grid_rope(self, t, height, width, device):
+ ids = torch.zeros((height, width, 3), device=device, dtype=torch.float32)
+ ids[..., 0] = t
+ ids[..., 1] = torch.arange(height, device=device, dtype=torch.float32)[:, None]
+ ids[..., 2] = torch.arange(width, device=device, dtype=torch.float32)[None]
+ return self.rope_embedder(ids.reshape(1, -1, 3)).movedim(1, 2)
+
+ def prepare_condition(self, latent, freqs):
+ height, width = latent.shape[-2:]
+ x = self.condition_encoder(latent) + comfy.ops.cast_to_input(self.condition_type_embedding, latent)
+ out = []
+ for block in self.backbone.blocks:
+ x, kv = block.condition(x, freqs, height, width)
+ out.append(kv)
+ return out
+
+ def decode_frame(self, lr, dtype, freqs, freqs_past, condition, history, detail):
+ height, width = lr.shape[-2] // self.latent_stride, lr.shape[-1] // self.latent_stride
+ x = self.lr_encoder(lr.to(dtype))
+ x = x + comfy.ops.cast_to_input(self.video_type_embedding, x)
+ next_history = []
+ for block, cond, past in zip(self.backbone.blocks, condition, history):
+ x, kv = block.video(x, freqs, freqs_past, height, width, cond, past)
+ next_history.append(kv)
+ residual = F.pixel_shuffle(self.residual_projection(self.spatial_head(x)).transpose(1, 2).reshape(lr.shape[0], 768, height, width), 16)
+ if detail != 1.0:
+ residual = residual * detail
+ return F.interpolate(lr, scale_factor=self.upscale_factor, mode="bicubic", align_corners=False) + residual, next_history
+
+ def decode_output_shape(self, input_shape, lq_images=None, detail=1.0):
+ if lq_images is None:
+ raise ValueError("The RelayVSR Flash decoder needs the low-quality frames, decode with the VSR Decode node.")
+ frames, height, width = lq_images.shape[:3]
+ # the x2 and x4 decoders load from the same weight shapes, a latent of the other factor would decode garbage or fail on the copy
+ if tuple(input_shape[2:]) != (len(relay_keyframes(frames)), *relay_latent_size(height, width, self.latent_stride)):
+ raise ValueError(f"The latent {tuple(input_shape[2:])} doesn't fit the x{self.upscale_factor} RelayVSR Flash decoder and these {frames} {width}x{height} frames: use the RelayVSR model and Flash decoder of the same factor, and the same video in RelayVSR Conditioning and VSR Decode.")
+ return (input_shape[0], 3, frames, height * self.upscale_factor, width * self.upscale_factor)
+
+ def decode(self, x, lq_images=None, output_buffer=None, detail=1.0):
+ # detail scales what the model adds to the bicubic upscale of each frame
+ out = output_buffer
+ if out is None:
+ out = torch.empty(self.decode_output_shape(x.shape, lq_images), device=comfy.model_management.intermediate_device())
+ b, _, _, height, width = x.shape
+ frames, lq_height, lq_width = lq_images.shape[:3]
+ depth = len(self.backbone.blocks)
+ keyframes = relay_keyframes(frames)
+ device = x.device
+ freqs, freqs_past = self.grid_rope(0, height, width, device), self.grid_rope(-1, height, width, device)
+ pbar = comfy.utils.ProgressBar(frames)
+ # finished frames go to the output without the GPU waiting on each copy
+ with comfy.model_management.HostWriter(device, out, (b, 3, *out.shape[-2:])) as writer, comfy.utils.model_trange(frames, desc="RelayVSR Flash decode", unit="frame") as console:
+ start = self.prepare_condition(x[:, :, 0], freqs)
+ for window, (first, last) in enumerate(zip(keyframes, keyframes[1:] or keyframes)):
+ comfy.model_management.throw_exception_if_processing_interrupted()
+ end = self.prepare_condition(x[:, :, min(window + 1, len(keyframes) - 1)], freqs)
+ condition = list(zip(start, end))
+ history = [None] * depth
+ # one upload per window, an upload from pageable memory waits for the whole stream
+ lr_window = lq_images[first:last + 1].to(device=device)[..., :3].movedim(-1, 1) * 2.0 - 1.0
+ lr_window = F.pad(lr_window, (0, width * self.latent_stride - lq_width, 0, height * self.latent_stride - lq_height), mode="replicate")
+ for i, frame in enumerate(range(first, last + 1)):
+ decoded, history = self.decode_frame(lr_window[i:i + 1].expand(b, -1, -1, -1), x.dtype, freqs, freqs_past, condition, history, detail)
+ # the shared keyframe was already output by the previous window, here it only seeds the history
+ if window == 0 or frame > first:
+ # finished [0, 1] pixels on the GPU, the VAE's process_output is the identity
+ writer.write(out[:, :, frame], decoded[..., :out.shape[-2], :out.shape[-1]].add_(1.0).div_(2.0).clamp_(0.0, 1.0).to(out.dtype))
+ pbar.update(1)
+ console.update(1)
+ start = end
+ return out
diff --git a/comfy/model_base.py b/comfy/model_base.py
index 0d4cfb114..39db511fc 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -52,6 +52,7 @@ import comfy.ldm.wan.model_animate
import comfy.ldm.wan.model_animate2
import comfy.ldm.wan.ar_model
import comfy.ldm.wan.model_wandancer
+import comfy.ldm.wan.model_vsr
import comfy.ldm.hunyuan3d.model
import comfy.ldm.triposplat.model
import comfy.ldm.hidream.model
@@ -2045,6 +2046,19 @@ class WAN21_SCAIL(WAN21):
return out
+class WAN22_RelayVSR(WAN21):
+ def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
+ super(WAN21, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.wan.model_vsr.RelayVSRWanModel)
+ self.image_to_video = False
+
+ def extra_conds(self, **kwargs):
+ out = super().extra_conds(**kwargs)
+ lq_images = kwargs.get("lq_images", None)
+ if lq_images is not None:
+ out['lq_pixels'] = comfy.conds.CONDRegular(lq_images[..., :3].movedim(-1, 0).unsqueeze(0) * 2.0 - 1.0)
+ return out
+
+
class WAN21_SCAIL2(WAN21_SCAIL):
"""SCAIL-2: SCAIL-Preview + an additive binary multi-identity mask stream."""
diff --git a/comfy/model_detection.py b/comfy/model_detection.py
index 04d76db2a..f7e412a2b 100644
--- a/comfy/model_detection.py
+++ b/comfy/model_detection.py
@@ -787,6 +787,10 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
dit_config["model_type"] = "scail"
elif '{}patch_embedding_global.weight'.format(key_prefix) in state_dict_keys:
dit_config["model_type"] = "wandancer"
+ elif '{}lq_proj_in.conv1.weight'.format(key_prefix) in state_dict_keys:
+ dit_config["model_type"] = "relayvsr"
+ # the LQ frames go in pixel-unshuffled by 4 * upscale_factor
+ dit_config["upscale_factor"] = math.isqrt(state_dict['{}lq_proj_in.conv1.weight'.format(key_prefix)].shape[1] // 3) // 4
else:
if '{}img_emb.proj.0.bias'.format(key_prefix) in state_dict_keys:
dit_config["model_type"] = "i2v"
diff --git a/comfy/model_management.py b/comfy/model_management.py
index e33147e74..405f15012 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -1490,6 +1490,21 @@ def reset_cast_buffers():
STREAM_AIMDO_CAST_BUFFERS.clear()
soft_empty_cache()
+def new_stream(device):
+ if is_device_cuda(device):
+ s = torch.cuda.Stream(device=device, priority=0)
+ s.as_context = torch.cuda.stream
+ elif is_device_xpu(device):
+ s = torch.xpu.Stream(device=device, priority=0)
+ s.as_context = torch.xpu.stream
+ elif is_device_npu(device):
+ # torch.npu.stream provides the context manager for torch-npu streams.
+ s = torch.npu.Stream(device=device, priority=0)
+ s.as_context = torch.npu.stream
+ else:
+ return None
+ return s
+
def get_offload_stream(device):
stream_counter = stream_counters.get(device, 0)
if NUM_STREAMS == 0:
@@ -1505,39 +1520,24 @@ def get_offload_stream(device):
stream_counter = (stream_counter + 1) % len(ss)
stream_counters[device] = stream_counter
return ss[stream_counter]
- elif is_device_cuda(device):
- ss = []
- for k in range(NUM_STREAMS):
- s1 = torch.cuda.Stream(device=device, priority=0)
- s1.as_context = torch.cuda.stream
- ss.append(s1)
- STREAMS[device] = ss
- s = ss[stream_counter]
- stream_counters[device] = stream_counter
- return s
- elif is_device_xpu(device):
- ss = []
- for k in range(NUM_STREAMS):
- s1 = torch.xpu.Stream(device=device, priority=0)
- s1.as_context = torch.xpu.stream
- ss.append(s1)
- STREAMS[device] = ss
- s = ss[stream_counter]
- stream_counters[device] = stream_counter
- return s
- elif is_device_npu(device):
- ss = []
- # Match the CUDA/XPU stream interface used by the shared offload path.
- # torch.npu.stream provides the context manager for torch-npu streams.
- for k in range(NUM_STREAMS):
- s1 = torch.npu.Stream(device=device, priority=0)
- s1.as_context = torch.npu.stream
- ss.append(s1)
- STREAMS[device] = ss
- s = ss[stream_counter]
- stream_counters[device] = stream_counter
- return s
- return None
+ s = new_stream(device)
+ if s is None:
+ return None
+ ss = [s] + [new_stream(device) for _ in range(NUM_STREAMS - 1)]
+ STREAMS[device] = ss
+ stream_counters[device] = stream_counter
+ return ss[stream_counter]
+
+DATA_STREAMS = {}
+
+def get_data_stream(device):
+ """One stream per device for data copies such as cache staging and decoder output, kept off the
+ offload streams so they don't disturb the weight prefetch. Callers order on it themselves."""
+ if NUM_STREAMS == 0 or torch.compiler.is_compiling():
+ return None
+ if device not in DATA_STREAMS:
+ DATA_STREAMS[device] = new_stream(device)
+ return DATA_STREAMS[device]
def sync_all_offload_streams():
if NUM_STREAMS == 0:
@@ -1737,6 +1737,59 @@ def unpin_memory(tensor):
return False
+class HostWriter:
+ """Copies device tensors of one shape into slices of a CPU tensor without the device waiting on each copy:
+ every write goes through one of two pinned staging buffers on the data stream and lands in its slice one
+ write later. Writes go straight through when the target is on the device, the device has no streams or the
+ buffers can't be pinned."""
+
+ def __init__(self, device, target, shape):
+ self.device = device
+ self.staging = []
+ self.pending = [None, None]
+ self.slot = 0
+ self.stream = get_data_stream(device) if is_device_cpu(target.device) else None
+ if self.stream is not None:
+ # inactive model pins make room for the staging buffers, active ones stay
+ if ensure_pin_budget(2 * torch.Size(shape).numel() * target.element_size(), evict_active=False):
+ for _ in range(2):
+ buffer = torch.empty(shape, dtype=target.dtype)
+ if pin_memory(buffer, evict_active=False):
+ self.staging.append(buffer)
+ if len(self.staging) < 2:
+ self.stream = None
+
+ def __enter__(self):
+ return self
+
+ def write(self, dest, src):
+ if self.stream is None:
+ dest.copy_(src)
+ return
+ if self.pending[self.slot] is not None:
+ self._collect(self.slot)
+ self.stream.wait_stream(current_stream(self.device))
+ cast_to(src, None, torch.device("cpu"), non_blocking=True, stream=self.stream, r=self.staging[self.slot])
+ # src stays referenced until its copy is done
+ self.pending[self.slot] = (self.stream.record_event(), dest, src)
+ self.slot ^= 1
+
+ def _collect(self, slot):
+ event, dest, _ = self.pending[slot]
+ event.synchronize()
+ dest.copy_(self.staging[slot])
+ self.pending[slot] = None
+
+ def __exit__(self, exc_type, exc, tb):
+ if self.stream is not None:
+ if exc_type is None:
+ for slot in (self.slot, self.slot ^ 1):
+ if self.pending[slot] is not None:
+ self._collect(slot)
+ self.stream.synchronize()
+ for buffer in self.staging:
+ unpin_memory(buffer)
+
def sage_attention_enabled():
return args.use_sage_attention
diff --git a/comfy/sd.py b/comfy/sd.py
index fca0c7576..0c6168c0e 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -17,6 +17,7 @@ import comfy.ldm.cosmos.vae
import comfy.ldm.wan.vae
import comfy.ldm.trellis2.vae
import comfy.ldm.wan.vae2_2
+import comfy.ldm.wan.model_vsr
import comfy.ldm.hunyuan3d.vae
import comfy.ldm.seedvr.vae
import comfy.ldm.mage_flow.vae
@@ -987,6 +988,18 @@ class VAE:
self.downscale_index_formula = (4, 8, 8)
self.memory_used_encode = lambda shape, dtype: (700 * (max(1, (shape[-3] ** 0.66 * 0.11)) * shape[-2] * shape[-1]) * model_management.dtype_size(dtype))
self.memory_used_decode = lambda shape, dtype: (50 * (max(1, (shape[-3] ** 0.65 * 0.26)) * shape[-2] * shape[-1] * 32 * 32) * model_management.dtype_size(dtype))
+ elif "lr_encoder.layers.0.weight" in sd and "condition_encoder.projection.weight" in sd: # RelayVSR Flash
+ self.latent_channels = 48
+ self.latent_dim = 3
+ self.upscale_ratio = (lambda a: max(0, (a - 1) * comfy.ldm.wan.model_vsr.RELAY_KEYFRAME_INTERVAL + 1), 16, 16)
+ # x2 and x4 decoders have the same weight shapes, only the metadata tells them apart
+ config = json.loads((metadata or {}).get("config", "{}"))
+ if "upscale_factor" not in config:
+ raise ValueError("RelayVSR Flash decoder without its upscale_factor metadata, use the released relayvsr_flash_x2/x4 file.")
+ self.first_stage_model = comfy.ldm.wan.model_vsr.RelayFlash(config["upscale_factor"])
+ self.working_dtypes = [torch.float16, torch.bfloat16, torch.float32]
+ self.process_input = self.process_output = lambda image: image
+ self.memory_used_decode = lambda shape, dtype: (40000 * shape[-2] * shape[-1] * model_management.dtype_size(dtype))
elif "vocoder.resblocks.0.convs1.0.weight" in sd or "vocoder.vocoder.resblocks.0.convs1.0.weight" in sd: # LTX Audio
sd = comfy.utils.state_dict_prefix_replace(sd, {"audio_vae.": "autoencoder."})
self.first_stage_model = comfy.ldm.lightricks.vae.audio_vae.AudioVAE(metadata=metadata)
@@ -1366,7 +1379,7 @@ class VAE:
preallocated = False
if getattr(self.first_stage_model, 'comfy_has_chunked_io', False):
with comfy.model_prefetch.pause_malloc_graph():
- pixel_samples = torch.empty(self.first_stage_model.decode_output_shape(samples_in.shape), device=self.output_device, dtype=self.vae_output_dtype())
+ pixel_samples = torch.empty(self.first_stage_model.decode_output_shape(samples_in.shape, **vae_options), device=self.output_device, dtype=self.vae_output_dtype())
preallocated = True
for x in range(0, samples_in.shape[0], batch_number):
diff --git a/comfy/supported_models.py b/comfy/supported_models.py
index 3d2015bbd..66c823e6c 100644
--- a/comfy/supported_models.py
+++ b/comfy/supported_models.py
@@ -1578,6 +1578,19 @@ class WAN21_SCAIL2(WAN21_T2V):
out = model_base.WAN21_SCAIL2(self, image_to_video=False, device=device)
return out
+class WAN22_RelayVSR(WAN21_T2V):
+ unet_config = {
+ "image_model": "wan2.1",
+ "model_type": "relayvsr",
+ }
+
+ latent_format = latent_formats.RelayVSR
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.WAN22_RelayVSR(self, device=device)
+ return out
+
+
class WAN22_WanDancer(WAN21_T2V):
unet_config = {
"image_model": "wan2.1",
@@ -2724,6 +2737,7 @@ models = [
WAN21_FlowRVS,
WAN21_SCAIL,
WAN21_SCAIL2,
+ WAN22_RelayVSR,
WAN22_WanDancer,
Hunyuan3Dv2mini,
Hunyuan3Dv2,
diff --git a/comfy_extras/nodes_wan.py b/comfy_extras/nodes_wan.py
index bb41f7ddb..02ec00ad3 100644
--- a/comfy_extras/nodes_wan.py
+++ b/comfy_extras/nodes_wan.py
@@ -7,6 +7,7 @@ import comfy.utils
import comfy.latent_formats
import comfy.clip_vision
import comfy.ldm.wan.model_animate2
+import comfy.ldm.wan.model_vsr
import comfy.patcher_extension
import json
import numpy as np
@@ -1459,6 +1460,58 @@ class Wan22ImageToVideoLatent(io.ComfyNode):
return io.NodeOutput(out_latent)
+class RelayVSRConditioning(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="RelayVSRConditioning",
+ category="model/conditioning/wan",
+ description="Conditioning and empty latent for RelayVSR video upscaling, sample with 1 step at cfg 1. Only every 15th frame goes through the model, VSR Decode restores all of them.",
+ inputs=[
+ io.Model.Input("model"),
+ io.Image.Input("images", tooltip="The low-quality video at its native resolution, the model decides the upscale factor."),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, model, images) -> io.NodeOutput:
+ diffusion_model = model.model.diffusion_model
+ keyframes = comfy.ldm.wan.model_vsr.relay_keyframes(images.shape[0])
+ positive = [[diffusion_model.positive_conditioning.unsqueeze(0).clone(), {"lq_images": images[keyframes]}]]
+ latent_size = comfy.ldm.wan.model_vsr.relay_latent_size(images.shape[1], images.shape[2], diffusion_model.latent_stride)
+ latent = torch.zeros([1, model.model.latent_format.latent_channels, len(keyframes), *latent_size], device=comfy.model_management.intermediate_device())
+ return io.NodeOutput(positive, {"samples": latent})
+
+
+class VSRDecode(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="VSRDecode",
+ display_name="VSR Decode",
+ category="latent",
+ description="Decode RelayVSR latents with the RelayVSR Flash decoder, which also reads the low-quality frames.",
+ inputs=[
+ io.Vae.Input("vae"),
+ io.Latent.Input("samples"),
+ io.Image.Input("images", tooltip="The same low-quality video given to the conditioning node."),
+ io.Float.Input("detail", default=1.0, min=0.0, max=1.0, step=0.01, tooltip="How much of the detail the decoder adds to the bicubic upscale of each frame is kept, 1 is the full model output."),
+ ],
+ outputs=[
+ io.Image.Output(),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, vae, samples, images, detail=1.0) -> io.NodeOutput:
+ decoded = vae.decode(samples["samples"], vae_options={"lq_images": images, "detail": detail})
+ return io.NodeOutput(decoded.reshape(-1, *decoded.shape[-3:]))
+
+
from comfy.ldm.wan.model_multitalk import InfiniteTalkOuterSampleWrapper, MultiTalkCrossAttnPatch, MultiTalkGetAttnMapPatch, project_audio_features
class WanInfiniteTalkToVideo(io.ComfyNode):
class DCValues(TypedDict):
@@ -1646,6 +1699,8 @@ class WanExtension(ComfyExtension):
WanAnimate2Cache,
Wan22ImageToVideoLatent,
WanInfiniteTalkToVideo,
+ RelayVSRConditioning,
+ VSRDecode,
]
async def comfy_entrypoint() -> WanExtension: