The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Make NAF neighborhood attention work per LR cell (#17155)
Jukka Seppänen446ef4e005fc
diff --git a/comfy/image_encoders/naf.py b/comfy/image_encoders/naf.pyindex 9d18c87df..d942daa2e 100644--- a/comfy/image_encoders/naf.py+++ b/comfy/image_encoders/naf.py@@ -18,112 +18,44 @@ import comfy.ops# Pure-torch neighborhood attention (replaces natten.na2d / na2d_qk + na2d_av).-def upsample_lr_slice(src_lr: torch.Tensor, lr_dh: int, lr_dw: int,- hr_h_range: Tuple[int, int], hr_w_range: Tuple[int, int]) -> torch.Tensor:- """Slice a LR-layout tensor [B, h_lr, w_lr, n, C], permute to BCHW, and- nearest-exact upsample only the region covering [hr_h_range, hr_w_range].- Returns BCHW at hr_h_end-hr_h_start x hr_w_end-hr_w_start (no padding for- out-of-bounds regions)."""- B = src_lr.shape[0]- n = src_lr.shape[-2]- C = src_lr.shape[-1]- h_hr_start, h_hr_end = hr_h_range- w_hr_start, w_hr_end = hr_w_range- # LR positions covering [h_hr_start, h_hr_end). Nearest-exact maps HR p → p // D.- lr_h_start = h_hr_start // lr_dh- lr_h_end = (h_hr_end - 1) // lr_dh + 1- lr_w_start = w_hr_start // lr_dw- lr_w_end = (w_hr_end - 1) // lr_dw + 1- lr_slice = src_lr[:, lr_h_start:lr_h_end, lr_w_start:lr_w_end]- lh, lw = lr_slice.shape[1], lr_slice.shape[2]- lr_bcd = lr_slice.permute(0, 3, 4, 1, 2).reshape(B * n, C, lh, lw).contiguous()- up = F.interpolate(lr_bcd, scale_factor=(lr_dh, lr_dw), mode="nearest-exact")- offset_h = h_hr_start - lr_h_start * lr_dh- offset_w = w_hr_start - lr_w_start * lr_dw- return up[:, :, offset_h:offset_h + (h_hr_end - h_hr_start),- offset_w:offset_w + (w_hr_end - w_hr_start)]-+_CHUNK_BYTES = 2 ** 29def na2d_pure(- q: torch.Tensor, # [B, H, W, n_heads, d_qk] at HR.- k_lr: torch.Tensor, # [B, h_lr, w_lr, n_heads, d_qk] at LR- v_lr: torch.Tensor, # [B, h_lr, w_lr, n_heads, d_v] at LR- kernel_size: Tuple[int, int], # (Kh, Kw) attention window.- dilation: Tuple[int, int], # (Dh, Dw) stride within the unrolled K/V grid; also the LR→HR upsample factor.+ q: torch.Tensor, # [B, n_heads, d_qk, H, W] at HR.+ k: torch.Tensor, # [B, n_heads * d_qk, h, w] at LR; H and W are multiples of h and w.+ v: torch.Tensor, # [B, n_heads * d_v, h, w] at LR.+ kernel_size: Tuple[int, int], # (Kh, Kw) attention window over the LR grid.scale: float, # 1 / sqrt(d_qk) scaling for the Q·K scores.- tile: int = 128, # Spatial tile size (output positions per tile)- v_chunk: int = 64, # Sub-divide d_v into chunks of this size when computing attn·V. None disables chunking.output: torch.Tensor = None, # Pre-allocated [B, n_heads, d_v, H, W] buffer (may be on CPU).) -> torch.Tensor: # [B, n_heads, d_v, H, W] (caller views as BCHW).- """Neighborhood attention in pure torch via F.unfold + per-tile slicing.-- K and V are passed at LR resolution and upsampled (nearest-exact) per-tile only- for the slice the unfold needs. Avoids the [B, n*d, H, W] HR allocations for K- (512 MB) and V (2 GB) at tex_1024 fp16. Spatial tiling bounds the per-tile- F.unfold blob; `v_chunk` further slices d_v so attn·V is computed in C-sized- chunks (attn is reused, computed once from Q/K).+ """natten.na2d on K/V nearest-upsampled to the HR grid, with the upsample factor as dilation.+ All HR queries of one LR cell see the same Kh x Kw window of the zero-padded LR grid, so each cell is one small+ attention of its queries against that window instead of one unfolded window per HR position."""- B, H, W, n, d_qk = q.shape- d_v = v_lr.shape[-1]+ B, n, d_qk, H, W = q.shape+ h, w = k.shape[-2:]+ d_v = v.shape[1] // nKh, Kw = kernel_size- Dh, Dw = dilation- pad_h, pad_w = (Kh // 2) * Dh, (Kw // 2) * Dw-+ Dh, Dw = H // h, W // w+ KK, DD = Kh * Kw, Dh * Dwout = output if output is not None else torch.empty((B, n, d_v, H, W), device=q.device, dtype=q.dtype)-- th = min(tile, H) if tile else H- tw = min(tile, W) if tile else W- chunk = v_chunk if (v_chunk and v_chunk < d_v) else d_v-- for h0 in range(0, H, th):- for w0 in range(0, W, tw):- h1, w1 = min(h0 + th, H), min(w0 + tw, W)- t_h, t_w = h1 - h0, w1 - w0-- # Padded HR region the unfold needs (kernel span = (K-1)*D + 1).- h_src_start = max(0, h0 - pad_h)- h_src_end = min(H, h1 + pad_h)- w_src_start = max(0, w0 - pad_w)- w_src_end = min(W, w1 + pad_w)- pad_top = max(0, pad_h - h0)- pad_bot = max(0, (h1 + pad_h) - H)- pad_lft = max(0, pad_w - w0)- pad_rgt = max(0, (w1 + pad_w) - W)-- # Upsample only the tile region from k_lr / v_lr.- k_tile = upsample_lr_slice(k_lr, Dh, Dw,- (h_src_start, h_src_end),- (w_src_start, w_src_end))- v_tile = upsample_lr_slice(v_lr, Dh, Dw,- (h_src_start, h_src_end),- (w_src_start, w_src_end))- if pad_top or pad_bot or pad_lft or pad_rgt:- k_tile = F.pad(k_tile, [pad_lft, pad_rgt, pad_top, pad_bot])- v_tile = F.pad(v_tile, [pad_lft, pad_rgt, pad_top, pad_bot])-- # Q·K → attention weights (small: KK=81 per output position).- KK = Kh * Kw- k_w = F.unfold(k_tile, kernel_size=(Kh, Kw), dilation=(Dh, Dw), padding=0)- k_w = k_w.view(B, n, d_qk, KK, t_h * t_w).permute(0, 1, 4, 3, 2) # [B, n, t, KK, d_qk]- # q is [B, H, W, n, d_qk]; per-tile slice + permute -> [B, n, t_h*t_w, 1, d_qk].- q_tile = q[:, h0:h1, w0:w1].permute(0, 3, 1, 2, 4).reshape(B, n, t_h * t_w, 1, d_qk)- scores = torch.matmul(q_tile, k_w.transpose(-1, -2)) * scale- attn = scores.softmax(dim=-1)- del k_w, scores, q_tile, k_tile-- # attn · V, chunked over d_v.- for c0 in range(0, d_v, chunk):- c1 = min(c0 + chunk, d_v)- v_w = F.unfold(v_tile[:, c0:c1], kernel_size=(Kh, Kw),dilation=(Dh, Dw), padding=0) # [B*n, (c1-c0)*KK, t]- v_w = v_w.view(B, n, c1 - c0, KK, t_h * t_w).permute(0, 1, 4, 3, 2)- out_chunk = torch.matmul(attn, v_w).squeeze(-2) # [B, n, t, c1-c0]- out_chunk = out_chunk.view(B, n, t_h, t_w, c1 - c0).permute(0, 1, 4, 2, 3)- out[:, :, c0:c1, h0:h1, w0:w1] = out_chunk- del v_w, out_chunk- del attn, v_tile-- return out # [B, n, d_v, H, W] — sole caller (CrossAttention) views it as BCHW directly.+ pad = (Kw // 2, Kw // 2, Kh // 2, Kh // 2)+ k, v = F.pad(k, pad), F.pad(v, pad)+ # LR rows per chunk, from one row's live temporaries: K/V windows, Q, fp32 copies and scores, outputs+ es = q.element_size()+ row_bytes = B * n * w * (KK * (d_qk * (es + 4) + d_v * es) + DD * (d_qk * (es + 4) + KK * (8 + es) + 2 * d_v * es))+ rows = max(1, _CHUNK_BYTES // row_bytes)+ for r0 in range(0, h, rows):+ r1 = min(r0 + rows, h)+ cells = (r1 - r0) * w+ k_w = F.unfold(k[:, :, r0:r1 + Kh - 1], (Kh, Kw)).view(B, n, d_qk, KK, cells).transpose(2, 4) # [B, n, cells, KK, d_qk]+ v_w = F.unfold(v[:, :, r0:r1 + Kh - 1], (Kh, Kw)).view(B, n, d_v, KK, cells).transpose(2, 4) # [B, n, cells, KK, d_v]+ q_c = q[..., r0 * Dh:r1 * Dh, :].reshape(B, n, d_qk, r1 - r0, Dh, w, Dw).permute(0, 1, 3, 5, 4, 6, 2).reshape(B, n, cells, DD, d_qk)+ attn = (torch.matmul(q_c.float(), k_w.float().transpose(-1, -2)) * scale).softmax(dim=-1).to(v_w.dtype)+ o = torch.matmul(attn, v_w).view(B, n, r1 - r0, w, Dh, Dw, d_v)+ out[..., r0 * Dh:r1 * Dh, :] = o.permute(0, 1, 6, 2, 4, 3, 5).reshape(B, n, d_v, (r1 - r0) * Dh, W)+ return outclass CrossAttention(nn.Module):@@ -137,23 +69,12 @@ class CrossAttention(nn.Module):self.kernel_size = kernel_sizeself.scale = (dim // num_heads) ** -0.5- @staticmethod- def _split_heads_lr(x: torch.Tensor, num_heads: int) -> torch.Tensor:- """[B, n*d, h, w] -> [B, h, w, n, d] at the input resolution (no upsample)."""- B, C, H, W = x.shape- return x.view(B, num_heads, C // num_heads, H, W).permute(0, 3, 4, 1, 2).contiguous()-def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,output=None) -> torch.Tensor:- hq, wq = q.shape[-2:]- hk, wk = k.shape[-2:]- dilation = (hq // hk, wq // wk)- B, C, _, _ = q.shape- q = q.view(B, self.num_heads, C // self.num_heads, hq, wq).permute(0, 3, 4, 1, 2).contiguous()- k_lr = self._split_heads_lr(k, self.num_heads).to(q.dtype)- v_lr = self._split_heads_lr(v, self.num_heads).to(q.dtype)- out_buf = output.view(B, self.num_heads, v.shape[1] // self.num_heads, hq, wq) if output is not None else None- out = na2d_pure(q, k_lr, v_lr, self.kernel_size, dilation, self.scale, output=out_buf)+ B, C, hq, wq = q.shape+ n = self.num_heads+ out_buf = output.view(B, n, v.shape[1] // n, hq, wq) if output is not None else None+ out = na2d_pure(q.view(B, n, C // n, hq, wq), k.to(q.dtype), v.to(q.dtype), self.kernel_size, self.scale, output=out_buf)return out.view(B, -1, hq, wq)